From 26a45a832e2d9b58854fdb79e90991d53c81c737 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Fri, 2 Dec 2022 22:04:57 +0800 Subject: [PATCH 01/73] refactor: delete all files for easy beginning Signed-off-by: DCCooper <1866858@gmail.com> --- Dockerfile | 3 - License/LICENSE | 127 -- ...Third_Party_Open_Source_Software_Notice.md | 361 ------ Makefile | 90 -- README.en.md | 52 - README.md | 116 -- VERSION | 1 - api/api.go | 42 - docs/api.md | 74 -- docs/config.md | 90 -- docs/limitation.md | 49 - docs/modules.md | 372 ------ hack/rubik-daemonset.yaml | 127 -- hack/rubik.service | 26 - hack/static_check.sh | 55 - hack/unit_test.sh | 36 - pkg/autoconfig/autoconfig.go | 93 -- pkg/autoconfig/autoconfig_test.go | 38 - pkg/blkio/blkio.go | 172 --- pkg/blkio/blkio_test.go | 276 ----- pkg/cachelimit/cachelimit.go | 149 --- pkg/cachelimit/cachelimit_init.go | 376 ------ pkg/cachelimit/cachelimit_init_test.go | 1019 ----------------- pkg/cachelimit/cachelimit_test.go | 158 --- pkg/checkpoint/checkpoint.go | 272 ----- pkg/checkpoint/checkpoint_test.go | 397 ------- pkg/config/config.go | 171 --- pkg/config/config_test.go | 127 -- pkg/constant/constant.go | 113 -- pkg/iocost/iocost.go | 253 ---- pkg/iocost/iocost_test.go | 571 --------- pkg/memory/dynlevel.go | 205 ---- pkg/memory/fssr.go | 203 ---- pkg/memory/memory.go | 205 ---- pkg/memory/status.go | 110 -- pkg/memory/status_test.go | 85 -- pkg/perf/perf.go | 285 ----- pkg/perf/perf_test.go | 71 -- pkg/qos/qos.go | 232 ---- pkg/qos/qos_test.go | 392 ------- pkg/quota/quota_burst.go | 91 -- pkg/quota/quota_burst_test.go | 245 ---- pkg/rubik/rubik.go | 361 ------ pkg/rubik/rubik_test.go | 275 ----- pkg/sync/sync.go | 46 - pkg/tinylog/tinylog.go | 306 ----- pkg/tinylog/tinylog_test.go | 273 ----- pkg/try/try.go | 127 -- pkg/try/try_test.go | 43 - pkg/typedef/convert.go | 60 - pkg/typedef/convert_test.go | 156 --- pkg/typedef/types.go | 98 -- pkg/typedef/types_test.go | 142 --- pkg/util/file.go | 101 -- pkg/util/file_test.go | 169 --- pkg/util/pod.go | 77 -- pkg/util/pod_test.go | 145 --- pkg/version/version.go | 49 - rubik.go | 26 - tests/data/fuzz-test/README.md | 26 - tests/data/fuzz-test/fuzz-test-newconfig/Fuzz | 31 - .../fuzz-test-newconfig/corpus/case1 | 24 - .../fuzz-test-newconfig/corpus/case2 | 7 - .../fuzz-test-newconfig/corpus/case3 | 7 - .../fuzz-test-newconfig/corpus/case4 | 7 - .../fuzz-test-newconfig/corpus/case5 | 7 - .../fuzz-test-newconfig/corpus/case6 | 7 - .../fuzz-test-newconfig/corpus/case7 | 18 - tests/data/fuzz-test/fuzz-test-newconfig/path | 1 - .../data/fuzz-test/fuzz-test-pinghandler/Fuzz | 23 - .../fuzz-test-pinghandler/corpus/case1 | 1 - .../data/fuzz-test/fuzz-test-pinghandler/path | 1 - .../data/fuzz-test/fuzz-test-roothandler/Fuzz | 23 - .../fuzz-test-roothandler/corpus/case1 | 1 - .../data/fuzz-test/fuzz-test-roothandler/path | 1 - .../fuzz-test/fuzz-test-versionhandler/Fuzz | 23 - .../fuzz-test-versionhandler/corpus/case1 | 1 - .../fuzz-test/fuzz-test-versionhandler/path | 1 - tests/lib/commonlib.sh | 282 ----- tests/lib/fuzz_commonlib.sh | 103 -- tests/src/TEMPLATE | 67 -- tests/src/fuzz_test_newconfig.sh | 58 - tests/src/fuzz_test_pinghandler.sh | 58 - tests/src/fuzz_test_roothandler.sh | 58 - tests/src/fuzz_test_versionhandler.sh | 58 - tests/src/test_rubik_flags_0001.sh | 41 - tests/src/test_rubik_offline_0001.sh | 56 - tests/src/test_rubik_offline_0002.sh | 69 -- tests/src/test_rubik_offline_0003.sh | 63 - tests/src/test_rubik_online_0001.sh | 58 - tests/src/test_rubik_reply_cachelimit_abn.sh | 52 - tests/src/test_rubik_reply_cachelimit_fun.sh | 61 - .../src/test_rubik_reply_healthcheck_0001.sh | 39 - .../src/test_rubik_reply_healthcheck_0002.sh | 41 - .../test_rubik_reply_http_parameter_0001.sh | 60 - .../test_rubik_reply_http_parameter_0002.sh | 72 -- .../test_rubik_reply_http_parameter_0003.sh | 108 -- tests/src/test_rubik_reply_version_0001.sh | 47 - tests/test.sh | 71 -- 99 files changed, 12116 deletions(-) delete mode 100644 Dockerfile delete mode 100644 License/LICENSE delete mode 100644 License/Third_Party_Open_Source_Software_Notice.md delete mode 100644 Makefile delete mode 100644 README.en.md delete mode 100644 README.md delete mode 100644 VERSION delete mode 100644 api/api.go delete mode 100644 docs/api.md delete mode 100644 docs/config.md delete mode 100644 docs/limitation.md delete mode 100644 docs/modules.md delete mode 100644 hack/rubik-daemonset.yaml delete mode 100644 hack/rubik.service delete mode 100755 hack/static_check.sh delete mode 100755 hack/unit_test.sh delete mode 100644 pkg/autoconfig/autoconfig.go delete mode 100644 pkg/autoconfig/autoconfig_test.go delete mode 100644 pkg/blkio/blkio.go delete mode 100644 pkg/blkio/blkio_test.go delete mode 100644 pkg/cachelimit/cachelimit.go delete mode 100644 pkg/cachelimit/cachelimit_init.go delete mode 100644 pkg/cachelimit/cachelimit_init_test.go delete mode 100644 pkg/cachelimit/cachelimit_test.go delete mode 100644 pkg/checkpoint/checkpoint.go delete mode 100644 pkg/checkpoint/checkpoint_test.go delete mode 100644 pkg/config/config.go delete mode 100644 pkg/config/config_test.go delete mode 100644 pkg/constant/constant.go delete mode 100644 pkg/iocost/iocost.go delete mode 100644 pkg/iocost/iocost_test.go delete mode 100644 pkg/memory/dynlevel.go delete mode 100644 pkg/memory/fssr.go delete mode 100644 pkg/memory/memory.go delete mode 100644 pkg/memory/status.go delete mode 100644 pkg/memory/status_test.go delete mode 100644 pkg/perf/perf.go delete mode 100644 pkg/perf/perf_test.go delete mode 100644 pkg/qos/qos.go delete mode 100644 pkg/qos/qos_test.go delete mode 100644 pkg/quota/quota_burst.go delete mode 100644 pkg/quota/quota_burst_test.go delete mode 100644 pkg/rubik/rubik.go delete mode 100644 pkg/rubik/rubik_test.go delete mode 100644 pkg/sync/sync.go delete mode 100644 pkg/tinylog/tinylog.go delete mode 100644 pkg/tinylog/tinylog_test.go delete mode 100644 pkg/try/try.go delete mode 100644 pkg/try/try_test.go delete mode 100644 pkg/typedef/convert.go delete mode 100644 pkg/typedef/convert_test.go delete mode 100644 pkg/typedef/types.go delete mode 100644 pkg/typedef/types_test.go delete mode 100644 pkg/util/file.go delete mode 100644 pkg/util/file_test.go delete mode 100644 pkg/util/pod.go delete mode 100644 pkg/util/pod_test.go delete mode 100644 pkg/version/version.go delete mode 100644 rubik.go delete mode 100644 tests/data/fuzz-test/README.md delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/Fuzz delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case1 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case2 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case3 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case4 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case5 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case6 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/corpus/case7 delete mode 100644 tests/data/fuzz-test/fuzz-test-newconfig/path delete mode 100644 tests/data/fuzz-test/fuzz-test-pinghandler/Fuzz delete mode 100644 tests/data/fuzz-test/fuzz-test-pinghandler/corpus/case1 delete mode 100644 tests/data/fuzz-test/fuzz-test-pinghandler/path delete mode 100644 tests/data/fuzz-test/fuzz-test-roothandler/Fuzz delete mode 100644 tests/data/fuzz-test/fuzz-test-roothandler/corpus/case1 delete mode 100644 tests/data/fuzz-test/fuzz-test-roothandler/path delete mode 100644 tests/data/fuzz-test/fuzz-test-versionhandler/Fuzz delete mode 100644 tests/data/fuzz-test/fuzz-test-versionhandler/corpus/case1 delete mode 100644 tests/data/fuzz-test/fuzz-test-versionhandler/path delete mode 100755 tests/lib/commonlib.sh delete mode 100755 tests/lib/fuzz_commonlib.sh delete mode 100755 tests/src/TEMPLATE delete mode 100755 tests/src/fuzz_test_newconfig.sh delete mode 100755 tests/src/fuzz_test_pinghandler.sh delete mode 100755 tests/src/fuzz_test_roothandler.sh delete mode 100755 tests/src/fuzz_test_versionhandler.sh delete mode 100755 tests/src/test_rubik_flags_0001.sh delete mode 100755 tests/src/test_rubik_offline_0001.sh delete mode 100755 tests/src/test_rubik_offline_0002.sh delete mode 100755 tests/src/test_rubik_offline_0003.sh delete mode 100755 tests/src/test_rubik_online_0001.sh delete mode 100755 tests/src/test_rubik_reply_cachelimit_abn.sh delete mode 100755 tests/src/test_rubik_reply_cachelimit_fun.sh delete mode 100755 tests/src/test_rubik_reply_healthcheck_0001.sh delete mode 100755 tests/src/test_rubik_reply_healthcheck_0002.sh delete mode 100755 tests/src/test_rubik_reply_http_parameter_0001.sh delete mode 100755 tests/src/test_rubik_reply_http_parameter_0002.sh delete mode 100755 tests/src/test_rubik_reply_http_parameter_0003.sh delete mode 100755 tests/src/test_rubik_reply_version_0001.sh delete mode 100755 tests/test.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 51f0e4c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM scratch -COPY ./rubik /rubik -ENTRYPOINT ["/rubik"] diff --git a/License/LICENSE b/License/LICENSE deleted file mode 100644 index 9e32cde..0000000 --- a/License/LICENSE +++ /dev/null @@ -1,127 +0,0 @@ - 木兰宽松许可证, 第2版 - - 木兰宽松许可证， 第2版 - 2020年1月 http://license.coscl.org.cn/MulanPSL2 - - - 您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第2版（“本许可证”）的如下条款的约束： - - 0. 定义 - - “软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 - - “贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 - - “贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 - - “法人实体”是指提交贡献的机构及其“关联实体”。 - - “关联实体”是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 - - 1. 授予版权许可 - - 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、分发其“贡献”，不论修改与否。 - - 2. 授予专利许可 - - 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求，不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“关联实体”直接或间接地，就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 - - 3. 无商标许可 - - “本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用除外。 - - 4. 分发限制 - - 您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。 - - 5. 免责声明与责任限制 - - “软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于何种法律理论，即使其曾被建议有此种损失的可能性。 - - 6. 语言 - “本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致，以中文版为准。 - - 条款结束 - - 如何将木兰宽松许可证，第2版，应用到您的软件 - - 如果您希望将木兰宽松许可证，第2版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步： - - 1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字； - - 2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中； - - 3， 请将如下声明文本放入每个源文件的头部注释中。 - - Copyright (c) [Year] [name of copyright holder] - [Software Name] is licensed under Mulan PSL v2. - You can use this software according to the terms and conditions of the Mulan PSL v2. - You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - See the Mulan PSL v2 for more details. - - - Mulan Permissive Software License，Version 2 - - Mulan Permissive Software License，Version 2 (Mulan PSL v2) - January 2020 http://license.coscl.org.cn/MulanPSL2 - - Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions: - - 0. Definition - - Software means the program and related documents which are licensed under this License and comprise all Contribution(s). - - Contribution means the copyrightable work licensed by a particular Contributor under this License. - - Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. - - Legal Entity means the entity making a Contribution and all its Affiliates. - - Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. - - 1. Grant of Copyright License - - Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. - - 2. Grant of Patent License - - Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. - - 3. No Trademark License - - No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4. - - 4. Distribution Restriction - - You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. - - 5. Disclaimer of Warranty and Limitation of Liability - - THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 6. Language - - THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL. - - END OF THE TERMS AND CONDITIONS - - How to Apply the Mulan Permissive Software License，Version 2 (Mulan PSL v2) to Your Software - - To apply the Mulan PSL v2 to your work, for easy identification by recipients, you are suggested to complete following three steps: - - i Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; - - ii Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; - - iii Attach the statement to the appropriate annotated syntax at the beginning of each source file. - - - Copyright (c) [Year] [name of copyright holder] - [Software Name] is licensed under Mulan PSL v2. - You can use this software according to the terms and conditions of the Mulan PSL v2. - You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - See the Mulan PSL v2 for more details. diff --git a/License/Third_Party_Open_Source_Software_Notice.md b/License/Third_Party_Open_Source_Software_Notice.md deleted file mode 100644 index 515f6f4..0000000 --- a/License/Third_Party_Open_Source_Software_Notice.md +++ /dev/null @@ -1,361 +0,0 @@ -**OPEN SOURCE SOFTWARE NOTICE** - -Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country. - -**Warranty Disclaimer** - -THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - -**Copyright Notice and License Texts** - -**Software**: github.com/cyphar/filepath-securejoin v0.2.3 - -**Copyright notice:** - -Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. -Copyright (C) 2017 SUSE LLC. All rights reserved. - -**License:** BSD 3-clause - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -**Software**: github.com/pkg/errors v0.9.1 - -**Copyright notice:** - -Copyright (c) 2015, Dave Cheney - -**License:** BSD-2-Clause - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -**Software**: github.com/stretchr/testify v1.6.1 - -**Copyright notice:** - -Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. - -**License:** MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -**Software**: golang.org/x/sys v0.0.0-20201112073958-5cba982894dd - -**Copyright notice:** - -Copyright (c) 2009 The Go Authors. All rights reserved. - -**License:** - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -**Software**: k8s.io/api v0.20.2 - -**Copyright notice:** - -Copyright The Kubernetes Authors. - -**License:** Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -**Software**: k8s.io/apimachinery v0.20.2 - -**Copyright notice:** - -Copyright The Kubernetes Authors. - -**License:** Apache License Version 2.0 - -Please see above. - -**Software**: k8s.io/client-go v0.20.2 - -**Copyright notice:** - -Copyright The Kubernetes Authors. - -**License:** Apache License Version 2.0 - -Please see above. diff --git a/Makefile b/Makefile deleted file mode 100644 index 4285721..0000000 --- a/Makefile +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: Xiang Li -# Create: 2021-04-17 -# Description: Makefile for rubik - -CWD=$(realpath .) -TMP_DIR := /tmp/rubik_tmpdir -INSTALL_DIR := /var/lib/rubik -BUILD_DIR=build -VERSION_FILE := ./VERSION -VERSION := $(shell awk -F"-" '{print $$1}' < $(VERSION_FILE)) -RELEASE :=$(if $(shell awk -F"-" '{print $$2}' < $(VERSION_FILE)),$(shell awk -F"-" '{print $$2}' < $(VERSION_FILE)),NA) -BUILD_TIME := $(shell date "+%Y-%m-%d") -GIT_COMMIT := $(if $(shell git rev-parse --short HEAD),$(shell git rev-parse --short HEAD),$(shell cat ./git-commit | head -c 7)) - -DEBUG_FLAGS := -gcflags="all=-N -l" -LD_FLAGS := -ldflags '-buildid=none -tmpdir=$(TMP_DIR) \ - -X isula.org/rubik/pkg/version.GitCommit=$(GIT_COMMIT) \ - -X isula.org/rubik/pkg/version.BuildTime=$(BUILD_TIME) \ - -X isula.org/rubik/pkg/version.Version=$(VERSION) \ - -X isula.org/rubik/pkg/version.Release=$(RELEASE) \ - -extldflags=-ftrapv \ - -extldflags=-Wl,-z,relro,-z,now -linkmode=external -extldflags=-static' - -GO_BUILD=CGO_ENABLED=1 \ - CGO_CFLAGS="-fstack-protector-strong -fPIE" \ - CGO_CPPFLAGS="-fstack-protector-strong -fPIE" \ - CGO_LDFLAGS_ALLOW='-Wl,-z,relro,-z,now' \ - CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ - go build -buildmode=pie - -all: release - -help: - @echo "Usage:" - @echo - @echo "make # build rubik with security build option" - @echo "make image # build container image" - @echo "make check # static check for latest commit" - @echo "make checkall # static check for whole project" - @echo "make tests # run all tests" - @echo "make test-unit # run unit test" - @echo "make cover # generate coverage report" - @echo "make install # install files to /var/lib/rubik" - -release: - mkdir -p $(TMP_DIR) $(BUILD_DIR) - rm -rf $(TMP_DIR) && mkdir -p $(ORG_PATH) $(TMP_DIR) - $(GO_BUILD) -o $(BUILD_DIR)/rubik $(LD_FLAGS) rubik.go - sed 's/__RUBIK_IMAGE__/rubik:$(VERSION)-$(RELEASE)/g' hack/rubik-daemonset.yaml > $(BUILD_DIR)/rubik-daemonset.yaml - cp hack/rubik.service $(BUILD_DIR) - -image: release - docker build -f Dockerfile -t rubik:$(VERSION)-$(RELEASE) . - -check: - @echo "Static check for last commit ..." - @./hack/static_check.sh last - @echo "Static check for last commit finished" - -checkall: - @echo "Static check for all ..." - @./hack/static_check.sh all - @echo "Static check for all finished" - -tests: test-unit test-integration - -test-unit: - @bash ./hack/unit_test.sh - -test-integration: - @bash ./tests/test.sh - -cover: - go test -p 1 -v ./... -coverprofile=cover.out - go tool cover -html=cover.out -o cover.html - python3 -m http.server 8080 - -install: - install -d -m 0750 $(INSTALL_DIR) - cp -f $(BUILD_DIR)/* $(INSTALL_DIR) - cp -f $(BUILD_DIR)/rubik.service /lib/systemd/system/ diff --git a/README.en.md b/README.en.md deleted file mode 100644 index 9c945f9..0000000 --- a/README.en.md +++ /dev/null @@ -1,52 +0,0 @@ -# rubik - -## Description - -The current global cloud infrastructure service expenditure is huge. However, the average CPU utilization of data center user clusters is very low, which is a huge waste of resources. Therefore, improving the utilization of data center resources is an important issue that needs to be solved urgently. - -Deployment of various types of services togather can significantly improve the utilization of cluster resources, but it also brings the problem of co-peaking, this issue can lead to partial service quality of service (QoS) compromise. How to ensure that the application's QoS is not damaged after improving the utilization of resources is a key technical challenge. - -To this end, we propose the Rubik resource utilization improvement solution, Rubik literally means Rubik's Cube, The Rubik’s Cube was invented in 1974 by Ernõ Rubik, a Hungarian architecture professor. In our solution, Rubik symbolizes being able to manage servers in an orderly manner. - -Rubik currently supports the following features: -- pod's CPU priority configure. -- pod's memory priority configure. - -## Build - -Pull the source code: -```sh -git clone https://gitee.com/openeuler/rubik.git -``` - -Enter the source code directory to compile: -```sh -cd rubik -make -``` - -Make rubik image: -```sh -make image -``` - -Install the relevant deployment files on the system: -```sh -sudo make install -``` -## Deployment - -### Prepare environment - -- OS: openeuler 21.09/22.03/22.09+ -- kubernetes: 1.17.0+ - -### Deploy rubik as daemonset - -```sh -kubectl apply -f /var/lib/rubik/rubik-daemonset.yaml -``` - -## Copyright - -Mulan PSL v2 diff --git a/README.md b/README.md deleted file mode 100644 index cb4ed8d..0000000 --- a/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# rubik - -## 概述 - -当前全球云基础设施服务支出费用庞大，然而数据中心用户集群的平均CPU利用率却很低，存在巨大的资源浪费。因此，提升数据中心资源利用率是当前急需解决的一个重要问题。 - -将多种类型业务混合部署能够显著提升集群资源利用率，但也带来了共峰问题，该问题会导致部分业务服务质量（QoS）受损。如何在提升资源利用率之后，保障业务QoS不受损是技术上的关键挑战。 - -为此我们提出了Rubik资源利用率提升解决方案，Rubik字面意思为魔方，魔方由Rubik在1974年发明，故Rubik既是人名也指代魔方，在我们的解决方案中，Rubik象征着能够将服务器管理的有条不紊。 - -Rubik当前支持如下特性： - -- [pod CPU优先级配置](./docs/modules.md/#pod-cpu优先级) -- [pod memory优先级配置](./docs/modules.md#pod-内存优先级) -- [pod 访存带宽和LLC限制](./docs/modules.md#dyncache-访存带宽和llc限制) -- [pod blkio配置](./docs/modules.md/#blkio) -- [pod memory cgroup分级](./docs/modules.md/#memory) -- [pod quota burst配置](./docs/modules.md/#quota-burst) -- [基于iocost io权重控制](./docs/modules.md/#rubik支持基于iocost的io权重控制) - -## 编译 - -拉取源代码： - -```sh -git clone https://gitee.com/openeuler/rubik.git -``` - -进入源码目录编译： - -```sh -cd rubik -make -``` - -制作rubik镜像 - -```bash -make image -``` - -将相关部署文件安装到系统中： - -```sh -sudo make install -``` - -## 部署 - -### 环境准备 - -- OS: openEuler 21.09/22.03/22.09+ -- kubernetes: 1.17.0+ - -### rubik daemonset部署 - -在master节点上使用kubectl命令部署rubik daemonset： - -```sh -kubectl apply -f /var/lib/rubik/rubik-daemonset.yaml -``` - -## 常用配置 - -通过以上方式部署的rubik将以默认配置启动，若用户需要修改rubik的配置，可通过修改rubik-daemonset.yaml文件中的config.json部分后重新部署rubik daemonset。 - -以下介绍几个常见配置，其他配置详见[配置文档](./docs/config.md) - -### Pod优先级自动配置 - -若在rubik config中配置autoConfig为true开启了Pod自动感知配置功能，用户仅需在部署业务pod时在yaml中通过annotation指定其优先级，部署后rubik会自动感知当前节点pod的创建与更新，并根据用户配置的优先级设置pod优先级。 - -### 依赖于kubelet的Pod优先级配置 - -由于自动配置依赖于来自api-server pod创建事件的通知，具有一定的延迟性，无法在进程启动之前及时完成优先级的配置，导致业务性能可能存在抖动。用户可以关闭自动配置，通过修改kubelet，向rubik发送http请求，在更早的时间点调用rubik配置pod优先级，http接口具体使用方法详见[http接口文档](./docs/http_API.md) - -### 支持自动校对Pod优先级 - -rubik支持在启动时对当前节点Pod QoS优先级配置一致性进行校对，这里的一致性是指k8s集群中的配置和rubik对pod优先级的配置之间的一致性。可以通过config选项autoCheck控制是否开启校对功能，默认关闭。若开启校对Pod优先级功能，启动或重启rubik时，rubik会自动校验并更正当前节点pod优先级配置。 - -## 在离线业务配置示例 - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: nginx - namespace: qosexample - annotations: - volcano.sh/preemptable: "true" # volcano.sh/preemptable为true代表业务为离线业务，false代表业务为在线业务，默认为false -spec: - containers: - - name: nginx - image: nginx - resources: - limits: - memory: "200Mi" - cpu: "1" - requests: - memory: "200Mi" - cpu: "1" -``` - -## 注意事项 - -约束限制详见[约束限制文档](./docs/limitation.md) - -## 如何贡献 - -我们很高兴能有新的贡献者加入！ - -在一切开始之前，请签署[CLA协议](https://openeuler.org/en/cla.html) - -## 版权 - -rubik遵从**Mulan PSL v2**版权协议 diff --git a/VERSION b/VERSION deleted file mode 100644 index ee1372d..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.2.2 diff --git a/api/api.go b/api/api.go deleted file mode 100644 index d9ecb64..0000000 --- a/api/api.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: api definition for rubik - -// Package api is for api definition -package api - -// PodQoS describe Pod QoS settings -type PodQoS struct { - CgroupPath string `json:"CgroupPath"` - QosLevel int `json:"QosLevel"` - CacheLimitLevel string `json:"CacheLimitLevel"` -} - -// SetQosRequest is request get from north end -type SetQosRequest struct { - Pods map[string]PodQoS `json:"Pods"` -} - -// SetQosResponse is response format for http responser -type SetQosResponse struct { - ErrCode int `json:"code"` - Message string `json:"msg"` -} - -// VersionResponse is version response for http responser -type VersionResponse struct { - Version string `json:"Version"` - Release string `json:"Release"` - GitCommit string `json:"Commit"` - BuildTime string `json:"BuildTime"` - Usage string `json:"Usage,omitempty"` -} diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index 14f1a88..0000000 --- a/docs/api.md +++ /dev/null @@ -1,74 +0,0 @@ -# http接口 - -rubik包含如下http接口 - -## 设置、更新Pod优先级接口 - -接口语法： - -```bash -HTTP POST /run/rubik/rubik.sock -{ - "Pods": { - "podaaa": { - "CgroupPath": "kubepods/burstable/podaaa", - "QosLevel": 0 - }, - "podbbb": { - "CgroupPath": "kubepods/burstable/podbbb", - "QosLevel": -1 - } - } -} -``` - -参数说明： - -- pods map必须提供pods。 - -- podUID map必须提供每个pod的UID。 - -- QosLevel int必须提供优先级。 - - - 0：默认值，在线业务。 - - -1：离线业务。 - - 其他：非法，不支持。 - -- CgroupPath string必须提供Pod的cgroup子路径。 - -说明： - -- 请求并发量1000QPS，并发量越界报错。 -- 单个请求pod数100个，请求数量越界报错。 - -示例如下： - -```sh -curl -v -H "Accept: application/json" -H "Content-type: application/json" -X POST --data '{"Pods": {"podaaa": {"CgroupPath": "kubepods/burstable/podaaa","QosLevel": 0},"podbbb": {"CgroupPath": "kubepods/burstable/podbbb","QosLevel": -1}}}' --unix-socket /run/rubik/rubik.sock http://localhost/ -``` - -## 探活接口 - -rubik作为HTTP服务，提供探活接口用于帮助判断rubik服务是否还在运行。 - -接口形式：HTTP/GET /ping - -示例如下： - -```sh -curl -XGET --unix-socket /run/rubik/rubik.sock http://localhost/ping -``` - -## 版本信息查询接口 - -rubik支持通过HTTP请求查询版本号。 - -接口形式：HTTP/GET /version - -示例如下： - -```sh -curl -XGET --unix-socket /run/rubik/rubik.sock http://localhost/version -{"Version":"0.0.1","Release":"1","Commit":"29910e6","BuildTime":"2021-05-12"} -``` - diff --git a/docs/config.md b/docs/config.md deleted file mode 100644 index 68b4a4a..0000000 --- a/docs/config.md +++ /dev/null @@ -1,90 +0,0 @@ -# Rubik配置说明 - -## 基本配置说明 - -Rubik执行程序由Go语言实现，并编译为静态可执行文件，以便尽可能与系统依赖解耦。 - -Rubik除支持 `-v` 参数查询版本信息之外，不支持其他参数，版本信息输出示例如下所示，该信息中的内容和格式可能随着版本发生变化。 - -``` -rubik -v -Version: 0.1.0 -Release: -Go Version: go1.15.15 -Git Commit: 9bafc90 -Built: 2022-06-24 -OS/Arch: linux/amd64 -``` - -Rubik启动时会解析配置文件，配置文件的路径固定为 `/var/lib/rubik/config.json` ，为避免配置混乱，暂不支持指定其他路径。 - -配置文件采用json格式，字段键采用驼峰命名规则，且首字母小写。 - -配置文件示例内容如下： - -```json -{ - "autoConfig": true, - "autoCheck": false, - "logDriver": "stdio", - "logDir": "/var/log/rubik", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup", - "cacheConfig": { - "enable": false, - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - }, - "blkioConfig": { - "enable": false - }, - "memoryConfig": { - "enable": true, - "strategy": "none", - "checkInterval": 5 - } -} -``` - -常用配置项说明： - -| 配置键[=默认值] | 类型 | 描述 | 示例值 | -|---------------------------|--------|-----------------------------------------------------|----------------------| -| autoConfig=false | bool | 自动配置开关，自动配置即自行拉取Pod信息并配置给系统 | false, true | -| autoCheck=false | bool | 自动检查开关，自动纠正因故障等原因导致的错误配置 | false, true | -| logDriver=stdio | string | 日志驱动，支持标准输出和文件 | stdio, file | -| logDir=/var/log/rubik | string | 日志保存目录 | /var/log/rubik | -| logSize=1024 | int | 总日志大小，单位MB，适用于logDriver=file | [10, 2**20] | -| logLevel=info | string | 日志级别 | debug, info, error | -| cgroupRoot=/sys/fs/cgroup | string | 系统cgroup挂载点路径 | /sys/fs/cgroup | -| cacheConfig | map | 动态控制CPU高速缓存模块（dynCache）的相关配置 | | -| .enable=false | bool | dynCache功能启用开关 | false, true | -| .defaultLimitMode=static | string | dynCache控制模式 | static, dynamic | -| .adjustInterval=1000 | int | dynCache动态控制间隔时间，单位ms | [10, 10000] | -| .perfDuration=1000 | int | dynCache性能perf执行时长，单位ms | [10, 10000] | -| .l3Percent | map | dynCache控制中L3各级别对应水位（%） | | -| ..low=20 | int | L3低水位组控制线 | [10, 100] | -| ..mid=30 | int | L3中水位组控制线 | [low, 100] | -| ..high=50 | int | L3高水位组控制线 | [mid, 100] | -| .memBandPercent | map | dynCache控制中MB各级别对应水位（%） | | -| ..low=10 | int | MB低水位组控制线 | [10, 100] | -| ..mid=30 | int | MB中水位组控制线 | [low, 100] | -| ..high=50 | int | MB高水位组控制线 | [mid, 100] | -| blkioConfig | map | IO控制模块相关配置 | | -| .enable=false | bool | IO控制模块使能开关 | | -| memoryConfig | map | 内存控制模块相关配置 | | -| .enable=false | bool | 内存控制模块使能开关 | | -| .strategy=none | string | 内存动态分级回收控制策略 | none, dynlevel, fssr | -| .checkInterval=5 | string | 内存动态分级回收控制策略检测间隔 | (0, 30] | diff --git a/docs/limitation.md b/docs/limitation.md deleted file mode 100644 index f6efab3..0000000 --- a/docs/limitation.md +++ /dev/null @@ -1,49 +0,0 @@ -# 约束限制 - -## 规格 - -- 磁盘：1GB+ - -- 内存：100MB+ - -- 单个请求超时时间：120s - -- 单个请求Pod上限：100个 - -- HTTP请求并发量上限：1000QPS - -## 运行时 - -- 每个k8s节点只能部署一个rubik，多个rubik会冲突。 - -- rubik不提供端口访问，只能通过sock通信。 - -- rubik只接收合法http请求路径及网络协议：http://localhost/（POST）、http://localhost/ping（GET）、http://localhost/version（GET） - -- rubik不接受任何命令行参数，若添加参数启动会报错退出。 - -- 容器挂载目录时，rubik本地套接字/run/rubik的目录权限需由业务侧保证最小权限（如700）。 - -- 如果rubik进程进入T、D状态，则服务端不可用，此时服务不会响应任何请求。为了避免此情况的发生，请在客户端设置超时时间，避免无限等待。 - -## Pod优先级设置 - -- 禁止低优先级往高优先级切换。如业务A先被设置为低优先级（-1），接着请求设置为高优先级（0），rubik报错。 - -- 用户添加注解、修改注解、修改yaml中的注解并重新apply等操作不会触发Pod重建。rubik不会监控Pod注解变化情况，因此Pod的优先级和注解一致性需要kubelet及上层组件保证。 - -- 禁止将任务从在线组迁移到离线组后再迁移回在线组，此操作会导致该任务QoS异常。 - -- 禁止将重要的系统服务和内核线程加入到离线组中，否则可能导致调度不及时，进而导致系统异常。 - -- cpu和memory的在线、离线配置需要统一，否则可能导致两个子系统的QoS冲突。 - -- kubelet创建pod时需要调用rubik并确保成功，否则不保证数据一致性。 - -- 使用混部后，原始的cpu share功能存在限制。具体表现为： - - - 若当前cpu中同时存在在线任务和离线任务，则离线任务的cpu share无法生效。 - - - 若当前cpu中只有在线任务或只有离线任务，cpu share能生效 - -- 用户态的优先级反转、smt、cache、numa负载均衡、离线任务的负载均衡，当前不支持。 diff --git a/docs/modules.md b/docs/modules.md deleted file mode 100644 index df5c02f..0000000 --- a/docs/modules.md +++ /dev/null @@ -1,372 +0,0 @@ -# 模块介绍 - -## Pod CPU优先级 - -rubik支持业务CPU优先级配置，针对在离线业务混合部署的场景，确保在线业务相对离线业务的CPU资源抢占。 - -**前置条件**： - -- 建议内核版本openEuler-22.03+。内核支持针对cgroup的cpu优先级配置，cpu子系统存在接口cpu.qos_level。 - -### CPU优先级内核接口 - -- /sys/fs/cgroup/cpu目录下容器的cgroup中，如`/sys/fs/cgroup/cpu/kubepods/burstable//`目录 - - cpu.qos_level：开启CPU优先级配置，默认值为0， 有效值为0和-1。 - - 0：标识为在线业务 - - -1：标识为离线业务 - -### CPU优先级配置详解 - -rubik会根据pod的yaml文件中的注解`volcano.sh/preemptable`自动配置cpu.qos_level, 默认为false。 - -``` -annotations: - volcano.sh/preemptable: true -``` - -- true：代表业务为离线业务， -- false：代表业务为在线业务 - ---------------------- - -## pod 内存优先级 - -rubik支持业务memory优先级配置，针对在离线业务混合部署的场景，确保OOM时优先kill离线业务。 - -**前置条件**： - -- 建议内核版本openEuler-22.03+。内核支持针对cgroup的memory优先级配置，memory子系统存在接口memory.qos_level。 -- 开启内存优先级支持: `echo 1 > /proc/sys/vm/memcg_qos_enable` - -### 内存优先级内核接口 - -- /proc/sys/vm/memcg_qos_enable：开启内存优先级特性，默认值为0，有效值为0和1。开启命令为：`echo 1 > /proc/sys/vm/memcg_qos_enable` - - 0：表示关闭特性 - - 1：表示开启特性。 - -- /sys/fs/cgroup/memory目录下容器的cgroup中，如`/sys/fs/cgroup/memory/kubepods/burstable//`目录 - - memory.qos_level：开启内存优先级配置，默认值为0，有效值为0和-1。 - - 0：标识为在线业务 - - -1：标识为离线业务 - -### 内存优先级配置详解 - -rubik会根据pod的yaml文件中的注解`volcano.sh/preemptable`自动配置memory.qos_level，参考[CPU优先级配置详解](#cpu优先级配置详解) - ---------------------- - -## dynCache 访存带宽和LLC限制 - -rubik支持业务的Pod访存带宽(memory bandwidth)和LLC(Last Level Cache)限制，通过限制离线业务的访存带宽/LLC使用，减少其对在线业务的干扰。 - -**前置条件**： - -- cache/访存限制功能仅支持物理机，不支持虚拟机。 - - X86物理机，需要OS支持且开启intel RDT的CAT和MBA功能，内核启动项cmdline需要添加`rdt=l3cat,mba` - - ARM物理机，需要OS支持且开启mpam功能，内核启动项需要添加`mpam=acpi`。 - -- 由于内核限制，RDT mode当前不支持pseudo-locksetup模式。 - -**rubik新增权限和目录**： - -- 挂载目录: `/sys/fs/resctrl`。 rubik需要读取和设置/sys/fs/resctrl目录下的文件，该目录需在rubik启动前挂载，且需保障在rubik运行过程中不被卸载。 -- 权限: SYS_ADMIN. 设置主机/sys/fs/resctrl目录下的文件需要rubik容器被赋有SYS_ADMIN权限。 -- namepsace: pid namespace. rubik需要获取业务容器进程在主机上的pid，所以rubik容器需与主机共享pid namespace。 - -**rubik rdt 控制组**： - -rubik在RDT resctrl目录(默认为 /sys/fs/resctrl)下创建5个控制组，分别为rubik_max、rubik_high、rubik_middle、rubik_low、rubik_dynamic。rubik启动后，将水位线写入对应控制组的schemata。其中，low、middle、high的水位线可在cacheConfig中配置；max控制组为默认最大值, dynamic控制组初始水位线和low控制组一致。 - -离线业务pod启动时通过注解`volcano.sh/cache-limit`设置其cache level，并被加入到指定的控制组中, 如下列配置的pod将被加入rubik_low控制组: - -``` -annotations: - volcano.sh/cache-limit: "low" -``` - -**rubik dynamic控制组**： - -当存在level为dynamic的离线pod时，rubik通过采集当前节点在线业务pod的cache miss 和 llc miss 指标，调整rubik_dynamic控制组的水位线，实现对dynamic控制组内离线应用pod的动态控制。 - -### dynCache内核接口 - -- /sys/fs/resctrl: 在该目录下创建5个控制组目录，并修改其schemata和tasks文件。 - -### dynCache配置详解 - -dynCache功能相关的配置在`cacheConfig`中: - -``` -"cacheConfig": { - "enable": false, - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - }, -``` - -- l3Percent 和 memBandPercent: - 通过 l3Percent 和 memBandPercent 配置low, mid, high控制组的水位线。 - - 比如当环境的`rdt bitmask=fffff, numa=2`时, rubik_low的控制组将根据 l3Percent low=20 和 memBandPercent low=10 两个参数, 将为/sys/fs/resctrl/rubik_low控制组配置: - - ``` - L3:0=f;1=f - MB:0=10;1=10 - ``` - -- defaultLimitMode: 如果离线pod未指定`volcano.sh/cache-limit`注解，将根据cacheConfig的defaultLimitMode来决定pod将被加入哪个控制组: - - defaultLimitMode为static时，pod将被加入到rubik_max控制组 - - defaultLimitMode为dynamic时，pod将被加入到rubik_dynamic控制组 -- adjustInterval: dynCache动态调整rubik_dynamic控制组的间隔时间，单位ms，默认1000ms -- perfDuration: dynCache性能perf执行时长，单位ms，默认1000ms - -### dynCache注意事项 - -- dynCache仅针对离线pod，对在线业务不生效。 -- 若业务容器运行过程中被手动重启（容器ID不变但容器进程PID变化），针对该容器的dynCache无法生效。 -- 业务容器启动并已设置dynCache级别后，不支持对其限制级别进行修改。 -- 动态限制组的调控灵敏度受到rubik配置文件内adjustInterval、perfDuration值以及节点在线业务pod数量的影响，每次调整（若干扰检测结果为需要调整）间隔在区间[adjustInterval+perfDuration, adjustInterval+perfDuration*pod数量]内波动，用户可根据灵敏度需求调整配置项。 - ---------------------- - -## blkio - -Pod的blkio的配置以`volcano.sh/blkio-limit`注解的形式，在pod创建的时候配置，或者在pod运行期间通过kubectl annotate进行动态的修改，支持离线和在线pod。 - -配置内容为4个列表: -| 项 | 说明 | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| device_read_bps | 用于设定设备执行“读”操作字节的上限。该配置为list，可以对多个device进行配置，device指定需要限制的块设备，value限定上限值，单位为byte | -| device_read_iops | 用于设定设备执行“读”操作次数的上限。该配置为list，可以对多个device进行配置，device指定需要限制的块设备 | -| device_write_bps | 用于设定设备执行 “写” 操作次数的上限。该配置为list，可以对多个device进行配置，device指定需要限制的块设备，value限定上限值，单位为byte | -| device_write_iops | 用于设定设备执行“写”操作字节的上限。该配置为list，可以对多个device进行配置，device指定需要限制的块设备 | - -### blkio内核接口 - -- /sys/fs/cgroup/blkio目录下容器的cgroup中，如`/sys/fs/cgroup/blkio/kubepods/burstable//`目录: - - blkio.throttle.read_bps_device - - blkio.throttle.read_iops_device - - blkio.throttle.write_bps_device - - blkio.throttle.write_iops_device - -配置的key:value和cgroup的key:value的配置规则一致: - -- 写入时会转换成环境page size的倍数 -- 只有minor为0的device配置才会生效 -- 如果取消限速，可将值设为0 - -### blkio配置详解 - -**rubik开启关闭blkio功能**: - -rubik提供blkio配置功能的开关，在`blkioConfig`中 - -``` -"blkioConfig": { - "enable": true -} -``` - -- enable: IO控制模块使能开关， 默认为false - -**pod配置样例**: - -通过pod的注解配置时可提供四个列表，分别是write_bps, write_iops, read_bps, read_iops, read_byte. - -- 创建时: 在yaml文件中 - - ``` - volcano.sh/blkio-limit: '{"device_read_bps":[{"device":"/dev/sda1","value":"10485760"}, {"device":"/dev/sda","value":"20971520"}], - "device_write_bps":[{"device":"/dev/sda1","value":"20971520"}], - "device_read_iops":[{"device":"/dev/sda1","value":"200"}], - "device_write_iops":[{"device":"/dev/sda1","value":"300"}]}' - ``` - -- 修改annotation: 可通过 kubectl annotate动态修改，如: - ```kubectl annotate --overwrite pods volcano.sh/blkio-limit='{"device_read_bps":[{"device":"/dev/vda", "value":"211715200"}]}'``` - ---------------------- - -## memory - -rubik中支持多种内存策略。针对不同场景使用不同的内存分配方案,以解决多场景内存分配。 - -dynlevel策略：基于内核cgroup的多级别控制。通过监测节点内存压力，多级别动态调整离线业务的memory cgroup，尽可能地保障在线业务服务质量。 - -fssr策略：基于内核cgroup的动态水位线控制。memory.high是内核提供的memcg级的水位线接口，rubik动态检测内存压力，动态调整离线应用的memory.high上限，实现对离线业务的内存压制，保障在线业务的服务质量。 - -### memory dynlevel策略内核接口 - -- /sys/fs/cgroup/memory目录下容器的cgroup中，如`/sys/fs/cgroup/memory/kubepods/burstable//`目录。dynlevel策略会依据当前节点的内存压力大小，依次调整节点离线应用容器的下列值: - - - memory.soft_limit_in_bytes - - memory.force_empty - - memory.limit_in_bytes - - /proc/sys/vm/drop_caches - -### memory dynlevel策略配置详解 - -rubik提供memory的指定策略和控制间隔，在`memoryConfig`中 - -``` -"memoryConfig": { - "enable": true, - "strategy": "none", - "checkInterval": 5 - } -``` - -- enable 为是否打开该配置的开关 -- strategy为memory的策略名称，现支持 dynlevel/fssr/none，默认为none。 - - none: 即不设置任何策略，不会对内存进行调整。 - - dynlevel: 动态分级调整策略。 - - fssr: 快压制慢恢复策略。1)rubik启动时，默认配置所有离线的memory.high为总内存的80%。2)当内存压力增加，可用内存freeMemory < reservedMemory(预留内存,totalMemory * 5%) 时认为内存紧张，此时压缩所有离线的memory.high, 压缩量为总内存的10%，即最新的memory.high=memory.high-totalMemory * 10%。3)当持续一段时间总内存比较富裕，即可用内存freeMemory > 3 * reservedMemory(totalMemory * 5%)时认为内存富裕，此时释放总内存的1%给离线应用，memory.high=memory.high+totalMemory * 1%, 直到memory free 介于reservedMemory与3 * reservedMemory之间。 - -- checkInterval为策略的周期性检查的时间，单位为秒, 默认为5。 - -### memory fssr策略内核接口 - -- /sys/fs/cgroup/memory目录下容器的cgroup中，如`/sys/fs/cgroup/memory/kubepods/burstable//`目录。fssr策略会依据当前节点的内存压力大小，依次调整节点离线应用容器的下列值: -- memory.high - -### memory fssr策略配置详解 - -rubik提供memory的指定策略和控制间隔，在`memoryConfig`中 -``` -"memoryConfig": { - "enable": true, - "strategy": "fssr", - "checkInterval": 5 - } -``` - -- enable 为是否打开该配置的开关 -- strategy为memory的策略名称，现支持 dynlevel/fssr/none 两个选项，默认为none。 - - none: 即不设置任何策略，不会对内存进行调整。 - - dynlevel: 动态分级调整策略。 - - fssr: 快压制慢恢复策略。1)rubik启动时，默认配置所有离线的memory.high为总内存的80%。2)当内存压力增加，可用内存freeMemory < reservedMemory(预留内存,totalMemory * 5%) 时认为内存紧张，此时压缩所有离线的memory.high, 压缩量为总内存的10%，即最新的memory.high=memory.high-totalMemory * 10%。3)当持续一段时间总内存比较富裕，即可用内存freeMemory > 3 * reservedMemory(totalMemory * 5%)时认为内存富裕，此时释放总内存的1%给离线应用，memory.high=memory.high+totalMemory * 1%, 直到memory free 介于reservedMemory与3 * reservedMemory之间。 - -- checkInterval为策略的周期性检查的时间，单位为秒, 默认为5。 - ---------------------- - -## quota burst - -Pod的quota burst的配置以`volcano.sh/quota-burst-time`注解的形式，在pod创建的时候配置，或者在pod运行期间通过kubectl annotate进行动态的修改，支持离线和在线pod。 - -Pod的quota burst默认单位是microseconds, 其允许容器的cpu使用率低于quota时累积cpu资源，并在cpu利用率超过quota时，使用容器累积的cpu资源。 - -### quota burst内核接口 - -- /sys/fs/cgroup/cpu目录下容器的cgroup中，如`/sys/fs/cgroup/cpu/kubepods/burstable//`目录，注解的值将被写入下列文件中： - - cpu.cfs_burst_us - -- 注解`volcano.sh/quota-burst-time`的值和cpu.cfs_burst_us的约束一致： - - 当cpu.cfs_quota_us不为-1，需满足cpu.cfs_burst_us + cpu.cfs_quota_us <= 2^44-1 且 cpu.cfs_burst_us <= cpu.cfs_quota_us - - 当cpu.cfs_quota_us为-1，cpu.cfs_burst_us最大没有限制，取决于系统最大可设置的值 - - -**pod配置样例** - -- 创建时: 在yaml文件中 - - ``` - metadata: - annotations: - volcano.sh/quota-burst-time : "2000" - ``` - -- 修改annotation: 可通过 kubectl annotate动态修改，如: - - ```kubectl annotate --overwrite pods volcano.sh/quota-burst-time='3000'``` - ---------------------- - -## rubik支持基于iocost的io权重控制 - -### 依赖说明 -rubik支持通过在cgroup v1下的iocost控制不同pod的io权重分配。因此需要内核支持如下特性： -- 内核支持cgroup v1 blkcg iocost -- 内核支持cgroup v1 writeback - -### rubik实现说明 - -```mermaid -sequenceDiagram -actor user as actor -participant apiserver -participant rubik -participant kernel; -participant cgroup; -user->>kernel: use iocost_coef_gen.py to get iocost parameter -user->>rubik: deploy rubik and enable rubik iocost feature and take iocost parameter -activate rubik; -rubik->>apiserver: listen and watch -rubik->>rubik: parsing iocost parameters -rubik->>cgroup : configure iocost parameters -user->>apiserver : deploy pod -apiserver->>rubik : send pod configure -rubik->>cgroup : parse pod's configure and bind memcg with blkcg and config pod's iocost.weight -``` - -步骤如下 -- 部署rubik时，rubik解析配置并设置iocost相关参数 -- rubik注册监听事件到k8s api-server -- pod被部署时将pod配置信息等回调到rubik -- rubik解析pod配置信息，并根据qos level配置pod iocost权重 - -### rubik协议说明 -```json -"nodeConfig": [ - { - "nodeName": "slaver01", - "iocostEnable": true, - "iocostConfig": [ - { - "dev": "sda", - "enable": false, - "model": "linear", - "param": { - "rbps": 174610612, - "rseqiops": 41788, - "rrandiops": 371, - "wbps": 178587889, - "wseqiops": 42792, - "wrandiops": 379 - } - } - ] - } - ] -``` - -| 配置项 | 类型 | 说明 | -| ----------- | ----------- | ------ | -| nodeConfig | 数组 | node节点配置信息 | -| nodeName | string | 要配置的节点名称 | -| iocostEnable | bool | 该node节点是否使用iocost | -| iocostConfig | 数组 | 针对不同物理磁盘的配置数组,当iocostEnable为true时会被读取 | -| dev | string | 物理磁盘名称 | -| enable | bool | 该物理磁盘是否启用iocost | -| model | string | iocost的模型名称,linear为内核自带线性模型 | -| param | object | 该参数针对model参数配置,当model为linear时,下面的参数都是linear相关参数 | -| r(w)bps | int64 | 该物理块设备最大读(写)带宽 | -| r(w)seqiops | int64 | 该物理块设备最大顺序读(写)iops | -| r(w)randiops | int64 | 该物理块设备最大随机读(写)iops | - - -### 其他 -- iocost linear模型相关参数可以通过iocost_coef_gen.py脚本获取，可以从[link](https://github.com/torvalds/linux/blob/master/tools/cgroup/iocost_coef_gen.py)获得。 - -- 在blkcg根系统文件下存在`blkio.cost.qos`和`blkio.cost.model`两个文件接口。实现方式和接口说明可以访问openEuler内核文档。 \ No newline at end of file diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml deleted file mode 100644 index df166d0..0000000 --- a/hack/rubik-daemonset.yaml +++ /dev/null @@ -1,127 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rubik -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rubik -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rubik -subjects: - - kind: ServiceAccount - name: rubik - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rubik - namespace: kube-system ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rubik-config - namespace: kube-system -data: - config.json: | - { - "autoCheck": false, - "logDriver": "stdio", - "logDir": "/var/log/rubik", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup", - "cacheConfig": { - "enable": false, - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: rubik-agent - namespace: kube-system - labels: - k8s-app: rubik-agent -spec: - selector: - matchLabels: - name: rubik-agent - template: - metadata: - namespace: kube-system - labels: - name: rubik-agent - spec: - serviceAccountName: rubik - hostPID: true - containers: - - name: rubik-agent - image: rubik_image_name_and_tag - imagePullPolicy: IfNotPresent - env: - - name: RUBIK_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - capabilities: - add: - - SYS_ADMIN - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 200Mi - volumeMounts: - - name: rubiklog - mountPath: /var/log/rubik - readOnly: false - - name: runrubik - mountPath: /run/rubik - readOnly: false - - name: sysfs - mountPath: /sys/fs - readOnly: false - - name: config-volume - mountPath: /var/lib/rubik - terminationGracePeriodSeconds: 30 - volumes: - - name: rubiklog - hostPath: - path: /var/log/rubik - - name: runrubik - hostPath: - path: /run/rubik - - name: sysfs - hostPath: - path: /sys/fs - - name: config-volume - configMap: - name: rubik-config - items: - - key: config.json - path: config.json diff --git a/hack/rubik.service b/hack/rubik.service deleted file mode 100644 index 6a885cf..0000000 --- a/hack/rubik.service +++ /dev/null @@ -1,26 +0,0 @@ -[Unit] -Description=Rubik multi-priority colocation engine -Documentation=https://gitee.com/openeuler/rubik -After=network-online.target firewalld.service -Wants=network-online.target - -[Service] -Type=notify -ExecStart=/usr/local/bin/rubik -ExecReload=/bin/kill -s HUP $MAINPID -TimeoutSec=0 -RestartSec=2 -Restart=always - -StartLimitBurst=3 -StartLimitInterval=60s -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -Delegate=yes -KillMode=process -OOMScoreAdjust=-500 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/hack/static_check.sh b/hack/static_check.sh deleted file mode 100755 index 0d15c2d..0000000 --- a/hack/static_check.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-04-15 -# Description: shell script for static checking - -workspace=$(cd "$(dirname "$0")" && cd .. && pwd) -config_file=".golangci.yml" -check_type=$1 -export GO111MODULE=off - -# check binary file golangci-lint and it's config exist -function pre() { - # check golangci-lint exist - lint=$(command -v golangci-lint) > /dev/null 2>&1 - if [ -z "${lint}" ]; then - echo "Could not find binary golangci-lint" - exit 1 - fi - - # check config exist - config_path=${workspace}/${config_file} - if [[ ! -f ${config_path} ]]; then - echo "Could not find config file for golangci-lint" - exit 1 - fi -} - -# last: only do static check for the very latest commit -# all : do static check for the whole project -function run() { - case ${check_type} in - last) - # ${lint} run --modules-download-mode vendor - ${lint} run - ;; - all) - ${lint} run --new=false --new-from-rev=false - ;; - *) - return - ;; - esac -} - -pre -run diff --git a/hack/unit_test.sh b/hack/unit_test.sh deleted file mode 100755 index 50aee5f..0000000 --- a/hack/unit_test.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-04-15 -# Description: go test script - -export GO111MODULE=off -export GOPATH="${PWD}"/build - -mkdir -p "${GOPATH}"/src/isula.org -ln -sfn "${PWD}" build/src/isula.org/rubik -cd $GOPATH/src/isula.org/rubik - -test_log=unit_test.log -rm -rf "${test_log}" -touch "${test_log}" - -go_list=$(go list ./...) -for path in ${go_list}; do - echo "Start to test: ${path}" - go test -race -cover -count=1 -timeout 300s -v "${path}" >> "${test_log}" - cat "${test_log}" | grep -E -- "--- FAIL:|^FAIL" - if [ $? -eq 0 ]; then - echo "Testing failed... Please check ${test_log}" - exit 1 - fi - tail -n 1 "${test_log}" -done diff --git a/pkg/autoconfig/autoconfig.go b/pkg/autoconfig/autoconfig.go deleted file mode 100644 index 5a494e0..0000000 --- a/pkg/autoconfig/autoconfig.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-07-22 -// Description: qos auto config - -// Package autoconfig is for qos auto config -package autoconfig - -import ( - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "isula.org/rubik/pkg/config" - log "isula.org/rubik/pkg/tinylog" -) - -const invalidErr = "Auto config error: invalid pod type" - -// EventHandler is used to process pod events pushed by Kubernetes APIServer. -type EventHandler interface { - AddEvent(pod *corev1.Pod) - UpdateEvent(oldPod *corev1.Pod, newPod *corev1.Pod) - DeleteEvent(pod *corev1.Pod) -} - -// Backend is Rubik struct. -var Backend EventHandler - -// Init initializes the callback function for the pod event. -func Init(kubeClient *kubernetes.Clientset, nodeName string) error { - const ( - reSyncTime = 30 - specNodeNameField = "spec.nodeName" - ) - kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, - time.Duration(reSyncTime)*time.Second, - informers.WithTweakListOptions(func(options *metav1.ListOptions) { - // set Options to return only pods on the current node. - options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, nodeName).String() - })) - kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: addHandler, - UpdateFunc: updateHandler, - DeleteFunc: deleteHandler, - }) - kubeInformerFactory.Start(config.ShutdownChan) - return nil -} - -func addHandler(obj interface{}) { - pod, ok := obj.(*corev1.Pod) - if !ok { - log.Errorf(invalidErr) - return - } - - Backend.AddEvent(pod) -} - -func updateHandler(old, new interface{}) { - oldPod, ok1 := old.(*corev1.Pod) - newPod, ok2 := new.(*corev1.Pod) - if !ok1 || !ok2 { - log.Errorf(invalidErr) - return - } - - Backend.UpdateEvent(oldPod, newPod) -} - -func deleteHandler(obj interface{}) { - pod, ok := obj.(*corev1.Pod) - if !ok { - log.Errorf(invalidErr) - return - } - - Backend.DeleteEvent(pod) -} diff --git a/pkg/autoconfig/autoconfig_test.go b/pkg/autoconfig/autoconfig_test.go deleted file mode 100644 index 3abd1b4..0000000 --- a/pkg/autoconfig/autoconfig_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2022-07-10 -// Description: autoconfig test - -package autoconfig - -import ( - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" -) - -// TestInit test Init -func TestInit(t *testing.T) { - kb := &kubernetes.Clientset{} - err := Init(kb, "nodeName") - assert.NoError(t, err) -} - -// TestAddUpdateDelHandler test Handler -func TestAddUpdateDelHandler(t *testing.T) { - oldObj := corev1.Pod{} - newObj := corev1.Pod{} - addHandler(oldObj) - updateHandler(oldObj, newObj) - deleteHandler(oldObj) -} diff --git a/pkg/blkio/blkio.go b/pkg/blkio/blkio.go deleted file mode 100644 index 573b3bf..0000000 --- a/pkg/blkio/blkio.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Song Yanting -// Create: 2022-6-7 -// Description: blkio setting for pods - -// Package blkio now only support byte unit. -// For example, limit read operation maximum 10 MBps, set value 10485760 -// More units will be supported like MB, KB... -package blkio - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "syscall" - - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/util" -) - -const ( - deviceReadBpsFile = "blkio.throttle.read_bps_device" - deviceWriteBpsFile = "blkio.throttle.write_bps_device" - deviceReadIopsFile = "blkio.throttle.read_iops_device" - deviceWriteIopsFile = "blkio.throttle.write_iops_device" -) - -// DeviceConfig defines blkio device configurations -type DeviceConfig struct { - DeviceName string `json:"device,omitempty"` - DeviceValue string `json:"value,omitempty"` -} - -// BlkConfig defines blkio device configurations -type BlkConfig struct { - DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` - DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` - DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` - DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` -} - -// SetBlkio set blkio limtis according to annotation -func SetBlkio(pod *corev1.Pod) { - cfg := decodeBlkioCfg(pod.Annotations[constant.BlkioKey]) - if cfg == nil { - return - } - blkioLimit(pod, cfg, false) -} - -// WriteBlkio updates blkio limtis according to annotation -func WriteBlkio(old *corev1.Pod, new *corev1.Pod) { - if new.Status.Phase != corev1.PodRunning { - return - } - - if old.Annotations[constant.BlkioKey] == new.Annotations[constant.BlkioKey] { - return - } - - // empty old blkio limits - if oldCfg := decodeBlkioCfg(old.Annotations[constant.BlkioKey]); oldCfg != nil { - blkioLimit(old, oldCfg, true) - } - - // set new blkio limits - if newCfg := decodeBlkioCfg(new.Annotations[constant.BlkioKey]); newCfg != nil { - blkioLimit(new, newCfg, false) - } -} - -func blkioLimit(pod *corev1.Pod, cfg *BlkConfig, empty bool) { - if len(cfg.DeviceReadBps) > 0 { - tryWriteBlkioLimit(pod, cfg.DeviceReadBps, deviceReadBpsFile, empty) - } - if len(cfg.DeviceWriteBps) > 0 { - tryWriteBlkioLimit(pod, cfg.DeviceWriteBps, deviceWriteBpsFile, empty) - } - if len(cfg.DeviceReadIops) > 0 { - tryWriteBlkioLimit(pod, cfg.DeviceReadIops, deviceReadIopsFile, empty) - } - if len(cfg.DeviceWriteIops) > 0 { - tryWriteBlkioLimit(pod, cfg.DeviceWriteIops, deviceWriteIopsFile, empty) - } -} - -func decodeBlkioCfg(blkioCfg string) *BlkConfig { - if len(blkioCfg) == 0 { - return nil - } - log.Infof("blkioCfg is %v", blkioCfg) - cfg := &BlkConfig{ - DeviceReadBps: []DeviceConfig{}, - DeviceWriteBps: []DeviceConfig{}, - DeviceReadIops: []DeviceConfig{}, - DeviceWriteIops: []DeviceConfig{}, - } - reader := bytes.NewReader([]byte(blkioCfg)) - if err := json.NewDecoder(reader).Decode(cfg); err != nil { - log.Errorf("decode blkioCfg failed with error: %v", err) - return nil - } - return cfg -} - -func tryWriteBlkioLimit(pod *corev1.Pod, devCfgs []DeviceConfig, deviceFilePath string, empty bool) { - for _, devCfg := range devCfgs { - devName, devLimit := devCfg.DeviceName, devCfg.DeviceValue - - fi, err := os.Stat(devName) - if err != nil { - log.Errorf("stat %s failed with error %v", devName, err) - continue - } - if fi.Mode()&os.ModeDevice == 0 { - log.Errorf("%s is not a device", devName) - continue - } - - if st, ok := fi.Sys().(*syscall.Stat_t); ok { - devno := st.Rdev - major, minor := devno/256, devno%256 - var limit string - if empty == true { - limit = fmt.Sprintf("%v:%v 0", major, minor) - } else { - limit = fmt.Sprintf("%v:%v %s", major, minor, devLimit) - } - writeBlkioLimit(pod, limit, deviceFilePath) - } else { - log.Errorf("failed to get Sys(), %v has type %v", devName, st) - } - } -} - -func writeBlkioLimit(pod *corev1.Pod, limit, deviceFilePath string) { - const ( - dockerPrefix = "docker://" - containerdPrefix = "containerd://" - blkioPath = "blkio" - ) - podCgroupPath := util.GetPodCgroupPath(pod) - for _, container := range pod.Status.ContainerStatuses { - containerID := strings.TrimPrefix(container.ContainerID, dockerPrefix) - containerID = strings.TrimPrefix(containerID, containerdPrefix) - containerPath := filepath.Join(podCgroupPath, containerID) - containerBlkFilePath := filepath.Join(config.CgroupRoot, blkioPath, containerPath, deviceFilePath) - - err := ioutil.WriteFile(containerBlkFilePath, []byte(limit), constant.DefaultFileMode) - if err != nil { - log.Errorf("writeBlkioLimit write %v to %v failed with error: %v", limit, containerBlkFilePath, err) - continue - } - log.Infof("writeBlkioLimit write %s to %v success", limit, containerBlkFilePath) - } -} diff --git a/pkg/blkio/blkio_test.go b/pkg/blkio/blkio_test.go deleted file mode 100644 index 96ef3d8..0000000 --- a/pkg/blkio/blkio_test.go +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Song Yanting -// Create: 2022-6-7 -// Description: blkio test -package blkio - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -const ( - blkioAnnotation = `{"device_read_bps":[{"device":"/dev/sda1","value":"52428800"}, {"device":"/dev/sda","value":"105857600"}], -"device_write_bps":[{"device":"/dev/sda1","value":"105857600"}], -"device_read_iops":[{"device":"/dev/sda1","value":"200"}], -"device_write_iops":[{"device":"/dev/sda1","value":"300"}]}` - invBlkioAnnotation = `{"device_read_bps":[{"device":"/dev/sda1","value":"52428800"}, {"device":"/dev/sda","value":"105857600"}}` -) - -var ( - devicePaths = map[string]string{ - "device_read_bps": deviceReadBpsFile, - "device_write_bps": deviceWriteBpsFile, - "device_read_iops": deviceReadIopsFile, - "device_write_iops": deviceWriteIopsFile, - } - status = corev1.PodStatus{ - ContainerStatuses: []corev1.ContainerStatus{ - {ContainerID: "docker://aaa"}, - }, - QOSClass: corev1.PodQOSBurstable, - Phase: corev1.PodRunning, - } - containerDir = filepath.Join(constant.TmpTestDir, "blkio/kubepods/burstable/podaaa/aaa") -) - -func getMajor(devName string) (major int, err error) { - cmd := fmt.Sprintf("ls -l %v | awk -F ' ' '{print $5}'", devName) - out, err := exec.Command("/bin/bash", "-c", cmd).Output() - if err != nil { - return -1, err - } - o := strings.TrimSuffix(strings.TrimSpace(string(out)), ",") - return strconv.Atoi(o) -} - -func getMinor(devName string) (minor int, err error) { - cmd := fmt.Sprintf("ls -l %v | awk -F ' ' '{print $6}'", devName) - out, err := exec.Command("sh", "-c", cmd).Output() - if err != nil { - return -1, err - } - o := strings.TrimSuffix(strings.TrimSpace(string(out)), "\n") - return strconv.Atoi(o) -} - -func TestBlkioAnnotation1(t *testing.T) { - // valid blkiocfg format - cfg := decodeBlkioCfg(blkioAnnotation) - assert.True(t, len(cfg.DeviceReadBps) > 0) - assert.True(t, len(cfg.DeviceWriteBps) > 0) - assert.True(t, len(cfg.DeviceReadIops) > 0) - assert.True(t, len(cfg.DeviceWriteIops) > 0) - - assert.Equal(t, "/dev/sda1", cfg.DeviceReadBps[0].DeviceName) - assert.Equal(t, "52428800", cfg.DeviceReadBps[0].DeviceValue) - assert.Equal(t, "/dev/sda", cfg.DeviceReadBps[1].DeviceName) - assert.Equal(t, "105857600", cfg.DeviceReadBps[1].DeviceValue) - - assert.Equal(t, "/dev/sda1", cfg.DeviceWriteBps[0].DeviceName) - assert.Equal(t, "105857600", cfg.DeviceWriteBps[0].DeviceValue) - - assert.Equal(t, "/dev/sda1", cfg.DeviceReadIops[0].DeviceName) - assert.Equal(t, "200", cfg.DeviceReadIops[0].DeviceValue) - - assert.Equal(t, "/dev/sda1", cfg.DeviceWriteIops[0].DeviceName) - assert.Equal(t, "300", cfg.DeviceWriteIops[0].DeviceValue) - - // invalid blkiocfg format - cfg = decodeBlkioCfg(invBlkioAnnotation) - assert.Equal(t, (*BlkConfig)(nil), cfg) -} - -func TestBlkioAnnotation2(t *testing.T) { - // valid blkiocfg format, valid + invalid device name - s1 := `{"device_read_bps":[{"device":"/dev/sda1","value":"10485760"}, {"device":"/dev/sda","value":"10485760"}], - "device_read_iops":[{"device":"/dev/sda1","value":"200"}, {"device":"/dev/123","value":"123"}]}` - cfg := decodeBlkioCfg(s1) - assert.True(t, len(cfg.DeviceReadBps) == 2) - assert.True(t, len(cfg.DeviceWriteBps) == 0) - assert.True(t, len(cfg.DeviceReadIops) == 2) - assert.True(t, len(cfg.DeviceWriteIops) == 0) - - assert.Equal(t, "/dev/sda1", cfg.DeviceReadBps[0].DeviceName) - assert.Equal(t, "10485760", cfg.DeviceReadBps[0].DeviceValue) - assert.Equal(t, "/dev/sda", cfg.DeviceReadBps[1].DeviceName) - assert.Equal(t, "10485760", cfg.DeviceReadBps[1].DeviceValue) - - assert.Equal(t, "/dev/sda1", cfg.DeviceReadIops[0].DeviceName) - assert.Equal(t, "200", cfg.DeviceReadIops[0].DeviceValue) - assert.Equal(t, "/dev/123", cfg.DeviceReadIops[1].DeviceName) - assert.Equal(t, "123", cfg.DeviceReadIops[1].DeviceValue) -} - -func listDevices() []string { - dir, _ := ioutil.ReadDir("/sys/block") - devices := make([]string, 0, len(dir)) - for _, f := range dir { - devices = append(devices, f.Name()) - } - return devices -} - -func genarateRandDev(n int, devices []string) string { - const bytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-" - b := make([]byte, n) - valid := true - for valid { - valid = false - for i := range b { - b[i] = bytes[typedef.RandInt(len(bytes))] - } - for _, device := range devices { - if device == string(b) { - valid = true - } - } - } - return string(b) -} - -func mkdirHelper(t *testing.T) { - assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) - for _, fname := range []string{deviceReadBpsFile, deviceWriteBpsFile, deviceReadIopsFile, deviceWriteIopsFile} { - assert.NoError(t, util.CreateFile(filepath.Join(containerDir, fname))) - } -} - -func TestSetBlkio(t *testing.T) { - testFunc := func(s1, deviceType, devicePath string, devices []string) { - mkdirHelper(t) - defer os.RemoveAll(constant.TmpTestDir) - - config.CgroupRoot = constant.TmpTestDir - pod := &corev1.Pod{ - ObjectMeta: v1.ObjectMeta{ - UID: "aaa", - Annotations: map[string]string{ - constant.BlkioKey: s1, - }, - }, - Status: status, - } - - SetBlkio(pod) - - expected := "" - for _, device := range devices { - major, err := getMajor("/dev/" + device) - if major == 0 || err != nil { - continue - } - minor, err := getMinor("/dev/" + device) - if err != nil { - continue - } - expected += fmt.Sprintf("%v:%v %v", major, minor, 10485760) - } - actual, _ := ioutil.ReadFile(filepath.Join(containerDir, devicePath)) - - assert.Equal(t, expected, strings.TrimSuffix(string(actual), "\n")) - } - - // test valid devices names from /sys/block - devices := listDevices() - for deviceType, devicePath := range devicePaths { - for _, device := range devices { - cfg := `{"` + deviceType + `":[{"device":"/dev/` + device + `","value":"10485760"}]}` - testFunc(cfg, deviceType, devicePath, []string{device}) - } - } - - // test invalid device names from random generated characters - for deviceType, devicePath := range devicePaths { - invalidDeviceName := genarateRandDev(3, devices) - cfg := `{"` + deviceType + `":[{"device":"/dev/` + invalidDeviceName + `","value":"10485760"}]}` - testFunc(cfg, deviceType, devicePath, []string{invalidDeviceName}) - - } - - // test valid devices names + invalid devices names - for deviceType, devicePath := range devicePaths { - for _, device := range devices { - invalidDeviceName := genarateRandDev(3, devices) - cfg := `{"` + deviceType + `":[{"device":"/dev/` + device + `","value":"10485760"}, {"device":"/dev/` + invalidDeviceName + `","value":"10485760"}]}` - testFunc(cfg, deviceType, devicePath, []string{device, invalidDeviceName}) - } - } -} - -func TestWriteBlkio(t *testing.T) { - mkdirHelper(t) - defer os.RemoveAll(constant.TmpTestDir) - - config.CgroupRoot = constant.TmpTestDir - old := corev1.Pod{ - ObjectMeta: v1.ObjectMeta{ - UID: "aaa", - Annotations: map[string]string{ - constant.BlkioKey: "", - }, - }, - Status: status, - } - SetBlkio(&old) - - testFunc := func(newCfg, deviceType, devicePath string, devices []string) { - new := corev1.Pod{ - ObjectMeta: v1.ObjectMeta{ - UID: "aaa", - Annotations: map[string]string{ - constant.BlkioKey: newCfg, - }, - }, - Status: status, - } - WriteBlkio(&old, &new) - - old = new - expected := "" - for _, device := range devices { - major, _ := getMajor("/dev/" + device) - if major == 0 { - continue - } - minor, _ := getMinor("/dev/" + device) - expected += fmt.Sprintf("%v:%v %v", major, minor, 10485760) - } - file := filepath.Join(containerDir, devicePath) - actual, _ := ioutil.ReadFile(file) - - assert.Equal(t, expected, strings.TrimSuffix(string(actual), "\n")) - } - - // test valid devices names from /sys/block - devices := listDevices() - for deviceType, devicePath := range devicePaths { - for _, device := range devices { - newCfg := `{"` + deviceType + `":[{"device":"/dev/` + device + `","value":"10485760"}]}` - testFunc(newCfg, deviceType, devicePath, []string{device}) - } - } -} diff --git a/pkg/cachelimit/cachelimit.go b/pkg/cachelimit/cachelimit.go deleted file mode 100644 index 9b668bb..0000000 --- a/pkg/cachelimit/cachelimit.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2022-01-18 -// Description: offline pod cache limit function - -// Package cachelimit is for cache limiting -package cachelimit - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - - securejoin "github.com/cyphar/filepath-securejoin" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/types" - - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -const ( - resctrlDir = "/sys/fs/resctrl" - noProErr = "no such process" -) - -// SyncLevel sync cache limit level -func SyncLevel(pi *typedef.PodInfo) error { - level := pi.CacheLimitLevel - if level == "" { - if defaultLimitMode == staticMode { - pi.CacheLimitLevel = maxLevel - } else { - pi.CacheLimitLevel = dynamicLevel - } - } - if !levelValid(pi.CacheLimitLevel) { - return errors.Errorf("invalid cache limit level %v for pod: %v", level, pi.UID) - } - return nil -} - -// syncCacheLimit sync cache limit for offline pods, as new processes may generate during pod running, -// they should be moved to resctrl directory -func syncCacheLimit() { - offlinePods := cpm.ListOfflinePods() - for _, p := range offlinePods { - if err := SyncLevel(p); err != nil { - log.Errorf("sync cache limit level err: %v", err) - continue - } - if err := writeTasksToResctrl(p, resctrlDir); err != nil { - log.Errorf("set cache limit for pod %v err: %v", p.UID, err) - } - } -} - -// SetCacheLimit set cache limit for offline pods -func SetCacheLimit(pi *typedef.PodInfo) error { - log.Logf("setting cache limit level=%v for pod %s", pi.CacheLimitLevel, pi.UID) - - return writeTasksToResctrl(pi, resctrlDir) -} - -func writeTasksToResctrl(pi *typedef.PodInfo, resctrlRoot string) error { - taskRootPath := filepath.Join(config.CgroupRoot, "cpu", pi.CgroupPath) - if !util.PathExist(taskRootPath) { - log.Infof("path %v not exist, maybe pod %v is deleted", taskRootPath, pi.UID) - return nil - } - - tasks, _, err := getTasks(pi, taskRootPath) - if err != nil { - return err - } - if len(tasks) == 0 { - return nil - } - - resctrlTaskFile := filepath.Join(resctrlRoot, dirPrefix+pi.CacheLimitLevel, "tasks") - for _, task := range tasks { - if err := ioutil.WriteFile(resctrlTaskFile, []byte(task), constant.DefaultFileMode); err != nil { - if strings.Contains(err.Error(), noProErr) { - log.Errorf("pod %s task %s not exist", pi.UID, task) - continue - } - return errors.Errorf("add task %v to file %v error: %v", task, resctrlTaskFile, err) - } - } - - return nil -} - -func getTasks(pi *typedef.PodInfo, taskRootPath string) ([]string, []string, error) { - file := "cgroup.procs" - var taskList, containers []string - err := filepath.Walk(taskRootPath, func(path string, f os.FileInfo, err error) error { - if f != nil && f.IsDir() { - containerID := filepath.Base(f.Name()) - if cpm.ContainerExist(types.UID(pi.UID), containerID) { - return nil - } - cgFilePath, err := securejoin.SecureJoin(path, file) - if err != nil { - return errors.Errorf("join path failed for %s and %s: %v", path, file, err) - } - tasks, err := ioutil.ReadFile(filepath.Clean(cgFilePath)) - if err != nil { - return errors.Errorf("read task file %v err: %v", cgFilePath, err) - } - if strings.TrimSpace(string(tasks)) == "" { - return nil - } - if containerID != filepath.Base(taskRootPath) { - containers = append(containers, containerID) - } - taskList = append(taskList, strings.Split(strings.TrimSpace(string(tasks)), "\n")...) - } - return nil - }) - - return taskList, containers, err -} - -func levelValid(level string) bool { - switch level { - case lowLevel: - case middleLevel: - case highLevel: - case maxLevel: - case dynamicLevel: - default: - return false - } - - return true -} diff --git a/pkg/cachelimit/cachelimit_init.go b/pkg/cachelimit/cachelimit_init.go deleted file mode 100644 index 2abfc6f..0000000 --- a/pkg/cachelimit/cachelimit_init.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2022-01-18 -// Description: offline pod cache limit directory init function - -package cachelimit - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/wait" - - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/perf" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -const ( - schemataFile = "schemata" - numaNodeDir = "/sys/devices/system/node" - dirPrefix = "rubik_" - perfEvent = "perf_event" - - lowLevel = "low" - middleLevel = "middle" - highLevel = "high" - maxLevel = "max" - dynamicLevel = "dynamic" - - staticMode = "static" - dynamicMode = "dynamic" - - defaultL3PercentMax = 100 - defaultMbPercentMax = 100 - minAdjustInterval = 10 - maxAdjustInterval = 10000 - // minimum perf duration, unit ms - minPerfDur = 10 - // maximum perf duration, unit ms - maxPerfDur = 10000 - minPercent = 10 - maxPercent = 100 - - base2, base16, bitSize = 2, 16, 32 -) - -var ( - numaNum, l3PercentDynamic, mbPercentDynamic int - defaultLimitMode string - enable bool - cpm *checkpoint.Manager -) - -type cacheLimitSet struct { - level string - clDir string - L3Percent int - MbPercent int -} - -// Init init and starts cache limit -func Init(m *checkpoint.Manager, cfg *config.CacheConfig) error { - enable = true - if !isHostPidns("/proc/self/ns/pid") { - return errors.New("share pid namespace with host is needed for cache limit") - } - if !perf.HwSupport() { - return errors.New("hardware event perf not supported") - } - if err := checkCacheCfg(cfg); err != nil { - return err - } - if err := checkResctrlExist(cfg); err != nil { - return err - } - if err := initCacheLimitDir(cfg); err != nil { - return errors.Errorf("cache limit directory create failed: %v", err) - } - defaultLimitMode = cfg.DefaultLimitMode - cpm = m - - go wait.Until(syncCacheLimit, time.Second, config.ShutdownChan) - missMax, missMin := 20, 10 - dynamicFunc := func() { startDynamic(cfg, missMax, missMin) } - go wait.Until(dynamicFunc, time.Duration(cfg.AdjustInterval)*time.Millisecond, config.ShutdownChan) - return nil -} - -func isHostPidns(path string) bool { - ns, err := os.Readlink(path) - if err != nil { - log.Errorf("get pid namespace inode error: %v", err) - return false - } - hostPidInode := "4026531836" - return strings.Trim(ns, "pid:[]") == hostPidInode -} - -func checkCacheCfg(cfg *config.CacheConfig) error { - defaultLimitMode = cfg.DefaultLimitMode - if defaultLimitMode != staticMode && defaultLimitMode != dynamicMode { - return errors.Errorf("invalid cache limit mode: %s, should be %s or %s", - cfg.DefaultLimitMode, staticMode, dynamicMode) - } - if cfg.AdjustInterval < minAdjustInterval || cfg.AdjustInterval > maxAdjustInterval { - return errors.Errorf("adjust interval %d out of range [%d,%d]", - cfg.AdjustInterval, minAdjustInterval, maxAdjustInterval) - } - if cfg.PerfDuration < minPerfDur || cfg.PerfDuration > maxPerfDur { - return errors.Errorf("perf duration %d out of range [%d,%d]", cfg.PerfDuration, minPerfDur, maxPerfDur) - } - for _, per := range []int{cfg.L3Percent.Low, cfg.L3Percent.Mid, cfg.L3Percent.High, cfg.MemBandPercent.Low, - cfg.MemBandPercent.Mid, cfg.MemBandPercent.High} { - if per < minPercent || per > maxPercent { - return errors.Errorf("cache limit percentage %d out of range [%d,%d]", per, minPercent, maxPercent) - } - } - if cfg.L3Percent.Low > cfg.L3Percent.Mid || cfg.L3Percent.Mid > cfg.L3Percent.High { - return errors.Errorf("cache limit config L3Percent does not satisfy constraint low<=mid<=high") - } - if cfg.MemBandPercent.Low > cfg.MemBandPercent.Mid || cfg.MemBandPercent.Mid > cfg.MemBandPercent.High { - return errors.Errorf("cache limit config MemBandPercent does not satisfy constraint low<=mid<=high") - } - - return nil -} - -// initCacheLimitDir init multi-level cache limit directories -func initCacheLimitDir(cfg *config.CacheConfig) error { - log.Infof("init cache limit directory") - - var err error - if numaNum, err = getNUMANum(numaNodeDir); err != nil { - return errors.Errorf("get NUMA nodes number error: %v", err) - } - - l3PercentDynamic = cfg.L3Percent.Low - mbPercentDynamic = cfg.MemBandPercent.Low - cacheLimitList := []*cacheLimitSet{ - newCacheLimitSet(cfg.DefaultResctrlDir, dynamicLevel, l3PercentDynamic, mbPercentDynamic), - newCacheLimitSet(cfg.DefaultResctrlDir, lowLevel, cfg.L3Percent.Low, cfg.MemBandPercent.Low), - newCacheLimitSet(cfg.DefaultResctrlDir, middleLevel, cfg.L3Percent.Mid, cfg.MemBandPercent.Mid), - newCacheLimitSet(cfg.DefaultResctrlDir, highLevel, cfg.L3Percent.High, cfg.MemBandPercent.High), - newCacheLimitSet(cfg.DefaultResctrlDir, maxLevel, defaultL3PercentMax, defaultMbPercentMax), - } - - for _, cl := range cacheLimitList { - if err = cl.writeResctrlSchemata(numaNum); err != nil { - return err - } - } - - log.Infof("init cache limit directory success") - return nil -} - -func newCacheLimitSet(basePath, level string, l3Per, mbPer int) *cacheLimitSet { - return &cacheLimitSet{ - level: level, - L3Percent: l3Per, - MbPercent: mbPer, - clDir: filepath.Join(filepath.Clean(basePath), dirPrefix+level), - } -} - -// calcLimitedCacheValue calculate number of cache way could be used according to L3 limit percent -func calcLimitedCacheValue(path string, l3Percent int) (string, error) { - l3BinaryMask, err := getBinaryMask(path) - if err != nil { - return "", err - } - ten, hundred, binValue := 10, 100, 0 - binLen := l3BinaryMask * l3Percent / hundred - if binLen == 0 { - binLen = 1 - } - for i := 0; i < binLen; i++ { - binValue = binValue*ten + 1 - } - decValue, err := strconv.ParseInt(strconv.Itoa(binValue), base2, bitSize) - if err != nil { - return "", errors.Errorf("transfer %v to decimal format error: %v", binValue, err) - } - - return strconv.FormatInt(decValue, base16), nil -} - -func (cl *cacheLimitSet) setClDir() error { - if len(cl.clDir) == 0 { - return errors.Errorf("cache limit path empty") - } - if err := os.Mkdir(cl.clDir, constant.DefaultDirMode); err != nil && !os.IsExist(err) { - return errors.Errorf("create cache limit directory error: %v", err) - } - return nil -} - -func (cl *cacheLimitSet) writeResctrlSchemata(numaNum int) error { - // get cbm mask like "fffff" means 20 cache way - maskFile := filepath.Join(filepath.Dir(cl.clDir), "info", "L3", "cbm_mask") - llc, err := calcLimitedCacheValue(maskFile, cl.L3Percent) - if err != nil { - return errors.Errorf("get limited cache value from L3 percent error: %v", err) - } - - if err := cl.setClDir(); err != nil { - return err - } - schemetaFile := filepath.Join(cl.clDir, schemataFile) - var content string - for i := 0; i < numaNum; i++ { - content = content + fmt.Sprintf("L3:%d=%s\n", i, llc) + fmt.Sprintf("MB:%d=%d\n", i, cl.MbPercent) - } - if err := ioutil.WriteFile(schemetaFile, []byte(content), constant.DefaultFileMode); err != nil { - return errors.Errorf("write %s to file %s error: %v", content, schemetaFile, err) - } - - return nil -} - -func (cl *cacheLimitSet) doFlush() error { - if err := cl.writeResctrlSchemata(numaNum); err != nil { - return errors.Errorf("adjust dynamic cache limit to l3:%v mb:%v error: %v", - cl.L3Percent, cl.MbPercent, err) - } - l3PercentDynamic = cl.L3Percent - mbPercentDynamic = cl.MbPercent - - return nil -} - -func (cl *cacheLimitSet) flush(cfg *config.CacheConfig, step int) error { - l3 := nextPercent(l3PercentDynamic, cfg.L3Percent.Low, cfg.L3Percent.High, step) - mb := nextPercent(mbPercentDynamic, cfg.MemBandPercent.Low, cfg.MemBandPercent.High, step) - if l3PercentDynamic == l3 && mbPercentDynamic == mb { - return nil - } - log.Infof("flush L3 from %v to %v, Mb from %v to %v", cl.L3Percent, l3, cl.MbPercent, mb) - cl.L3Percent, cl.MbPercent = l3, mb - return cl.doFlush() -} - -func nextPercent(value, min, max, step int) int { - value += step - if value < min { - return min - } - if value > max { - return max - } - return value -} - -// startDynamic start monitor online pod qos and adjust dynamic cache limit value -func startDynamic(cfg *config.CacheConfig, missMax, missMin int) { - if !dynamicExist() { - return - } - - stepMore, stepLess := 5, -50 - needMore := true - limiter := newCacheLimitSet(cfg.DefaultResctrlDir, dynamicLevel, l3PercentDynamic, mbPercentDynamic) - - onlinePods := cpm.ListOnlinePods() - for _, p := range onlinePods { - cacheMiss, LLCMiss := getPodCacheMiss(p, cfg.PerfDuration) - if cacheMiss >= missMax || LLCMiss >= missMax { - log.Infof("online pod %v cache miss: %v LLC miss: %v exceeds maxmiss, lower offline cache limit", - p.UID, cacheMiss, LLCMiss) - - if err := limiter.flush(cfg, stepLess); err != nil { - log.Errorf(err.Error()) - } - return - } - if cacheMiss >= missMin || LLCMiss >= missMin { - needMore = false - } - } - - if !needMore { - return - } - if err := limiter.flush(cfg, stepMore); err != nil { - log.Errorf(err.Error()) - } -} - -func dynamicExist() bool { - offlinePods := cpm.ListOfflinePods() - for _, p := range offlinePods { - err := SyncLevel(p) - if err != nil { - continue - } - if p.CacheLimitLevel == dynamicLevel { - return true - } - } - return false -} - -func getPodCacheMiss(pi *typedef.PodInfo, perfDu int) (int, int) { - cgroupPath := filepath.Join(config.CgroupRoot, perfEvent, pi.CgroupPath) - if !util.PathExist(cgroupPath) { - return 0, 0 - } - - stat, err := perf.CgroupStat(cgroupPath, time.Duration(perfDu)*time.Millisecond) - if err != nil { - return 0, 0 - } - - return int(100.0 * float64(stat.CacheMisses) / (1.0 + float64(stat.CacheReferences))), - int(100.0 * float64(stat.LLCMiss) / (1.0 + float64(stat.LLCAccess))) -} - -// ClEnabled return if cache limit is enabled -func ClEnabled() bool { - return enable -} - -// checkResctrlExist check if resctrl directory exists -func checkResctrlExist(cfg *config.CacheConfig) error { - if !util.PathExist(cfg.DefaultResctrlDir) { - return errors.Errorf("path %v not exist, not support cache limit", cfg.DefaultResctrlDir) - } - schemataPath := filepath.Join(cfg.DefaultResctrlDir, schemataFile) - if !util.PathExist(schemataPath) { - return errors.Errorf("path %v not exist, check if %v directory is mounted", - schemataPath, cfg.DefaultResctrlDir) - } - return nil -} - -func getNUMANum(path string) (int, error) { - files, err := filepath.Glob(filepath.Join(path, "node*")) - if err != nil { - return 0, err - } - return len(files), nil -} - -// getBinaryMask get l3 limit mask like "7ff" and transfer it to binary like "111 1111 1111", return binary length 11 -func getBinaryMask(path string) (int, error) { - maskValue, err := ioutil.ReadFile(filepath.Clean(path)) - if err != nil { - return -1, errors.Errorf("get L3 mask value error: %v", err) - } - - // transfer mask to binary format - decMask, err := strconv.ParseInt(strings.TrimSpace(string(maskValue)), base16, bitSize) - if err != nil { - return -1, errors.Errorf("transfer L3 mask value %v to decimal format error: %v", string(maskValue), err) - } - return len(strconv.FormatInt(decMask, base2)), nil -} diff --git a/pkg/cachelimit/cachelimit_init_test.go b/pkg/cachelimit/cachelimit_init_test.go deleted file mode 100644 index fe8f541..0000000 --- a/pkg/cachelimit/cachelimit_init_test.go +++ /dev/null @@ -1,1019 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2022-05-16 -// Description: offline pod cache limit directory init function - -package cachelimit - -import ( - "fmt" - "io/ioutil" - "math" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/perf" - "isula.org/rubik/pkg/try" - "isula.org/rubik/pkg/typedef" -) - -// TestGetNUMANum testcase -func TestGetNUMANum(t *testing.T) { - threeNodeDir := try.GenTestDir().String() - for i := 0; i < 3; i++ { - nodeDir := filepath.Join(threeNodeDir, fmt.Sprintf("node%d", i)) - try.MkdirAll(nodeDir, constant.DefaultDirMode) - } - - type args struct { - path string - } - tests := []struct { - name string - args args - want int - wantErr bool - compare bool - }{ - { - name: "TC-right numa folder", - args: args{path: numaNodeDir}, - wantErr: false, - compare: false, - }, - { - name: "TC-three numa foler", - args: args{path: threeNodeDir}, - want: 3, - wantErr: false, - compare: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getNUMANum(tt.args.path) - if (err != nil) != tt.wantErr { - t.Errorf("getNUMANum() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if tt.compare { - if got != tt.want { - t.Errorf("getNUMANum() = %v, want %v", got, tt.want) - } - } - }) - } -} - -// TestGetBinaryMask testcase -func TestGetBinaryMask(t *testing.T) { - file7ff := filepath.Join(try.GenTestDir().String(), "7ff") - file3ff := filepath.Join(try.GenTestDir().String(), "3ff") - fileNotHex := filepath.Join(try.GenTestDir().String(), "nohex") - - tests := []struct { - preHook func(t *testing.T) - name string - path string - want int - wantErr bool - }{ - { - name: "TC-7ff", - path: file7ff, - want: 11, - wantErr: false, - preHook: func(t *testing.T) { - try.WriteFile(file7ff, []byte("7ff"), constant.DefaultFileMode) - }, - }, - { - name: "TC-3ff", - path: file3ff, - want: 10, - wantErr: false, - preHook: func(t *testing.T) { - try.WriteFile(file3ff, []byte("3ff"), constant.DefaultFileMode) - }, - }, - { - name: "TC-not hex format", - path: fileNotHex, - wantErr: true, - preHook: func(t *testing.T) { - try.WriteFile(fileNotHex, []byte("ghi"), constant.DefaultFileMode) - }, - }, - { - name: "TC-file not exist", - path: "/file/not/exist", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.preHook != nil { - tt.preHook(t) - } - got, err := getBinaryMask(tt.path) - if (err != nil) != tt.wantErr { - t.Errorf("getBinaryMask() error = %v, wantErr %v, file = %v", err, tt.wantErr, tt.path) - return - } - if err == nil { - if got != tt.want { - t.Errorf("getBinaryMask() = %v, want %v", got, tt.want) - } - } - }) - } -} - -// TestCalcLimitedCacheValue testcase -func TestCalcLimitedCacheValue(t *testing.T) { - testFile := filepath.Join(try.GenTestDir().String(), "testFile") - type fields struct { - level string - L3Percent int - MbPercent int - } - type args struct { - path string - } - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - fields fields - args args - want string - wantErr bool - }{ - { - name: "TC-7ff", - args: args{testFile}, - want: "1", - fields: fields{ - L3Percent: 10, - MbPercent: 10, - }, - preHook: func(t *testing.T) { - try.WriteFile(testFile, []byte("7ff"), constant.DefaultFileMode) - }, - }, - { - name: "TC-fffff", - args: args{testFile}, - want: "3", - fields: fields{ - L3Percent: 10, - MbPercent: 10, - }, - preHook: func(t *testing.T) { - try.WriteFile(testFile, []byte("fffff"), constant.DefaultFileMode) - }, - }, - { - name: "TC-ff", - args: args{testFile}, - want: "1", - fields: fields{ - L3Percent: 10, - }, - preHook: func(t *testing.T) { - try.WriteFile(testFile, []byte("ff"), constant.DefaultFileMode) - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clSet := &cacheLimitSet{ - level: tt.fields.level, - L3Percent: tt.fields.L3Percent, - MbPercent: tt.fields.MbPercent, - } - if tt.preHook != nil { - tt.preHook(t) - } - got, err := calcLimitedCacheValue(tt.args.path, clSet.L3Percent) - if (err != nil) != tt.wantErr { - t.Errorf("cacheLimitSet.calcLimitedCacheValue() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("cacheLimitSet.calcLimitedCacheValue() = %v, want %v", got, tt.want) - } - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} - -// TestWriteResctrlSchemata testcase -func TestWriteResctrlSchemata(t *testing.T) { - testFolder := try.GenTestDir().String() - assert.NoError(t, setMaskFile(t, testFolder, "3ff")) - type fields struct { - level string - clDir string - L3Percent int - MbPercent int - } - type args struct { - llc string - numaNum int - } - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - fields fields - args args - wantErr bool - }{ - { - name: "TC-normal", - fields: fields{ - level: lowLevel, - clDir: filepath.Join(testFolder, "normal"), - L3Percent: 30, - MbPercent: 30, - }, - args: args{llc: "3ff", numaNum: 2}, - wantErr: false, - }, - { - name: "TC-cache limit dir not set", - fields: fields{ - level: lowLevel, - L3Percent: 30, - MbPercent: 30, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clSet := &cacheLimitSet{ - level: tt.fields.level, - clDir: tt.fields.clDir, - L3Percent: tt.fields.L3Percent, - MbPercent: tt.fields.MbPercent, - } - if tt.preHook != nil { - tt.preHook(t) - } - if err := clSet.writeResctrlSchemata(tt.args.numaNum); (err != nil) != tt.wantErr { - t.Errorf("cacheLimitSet.writeResctrlSchemata() error = %v, wantErr %v", err, tt.wantErr) - } - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} - -// TestCheckCacheCfg testcase -func TestCheckCacheCfg(t *testing.T) { - type args struct { - cfg config.CacheConfig - } - tests := []struct { - name string - args args - wantErr bool - wantMsg string - }{ - { - name: "TC-static mode config", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: minAdjustInterval + 1, - PerfDuration: minPerfDur + 1, - L3Percent: config.MultiLvlPercent{ - Low: minPercent + 1, - Mid: maxPercent/2 + 1, - High: maxPercent - 1, - }, - MemBandPercent: config.MultiLvlPercent{ - Low: minPercent + 1, - Mid: maxPercent/2 + 1, - High: maxPercent - 1, - }, - }}, - }, - { - name: "TC-invalid mode config", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: "invalid mode", - }}, - wantErr: true, - wantMsg: dynamicMode, - }, - { - name: "TC-invalid adjust interval less than min value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: minAdjustInterval - 1, - }}, - wantErr: true, - wantMsg: strconv.Itoa(minAdjustInterval), - }, - { - name: "TC-invalid adjust interval greater than max value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval + 1, - }}, - wantErr: true, - wantMsg: strconv.Itoa(maxAdjustInterval), - }, - { - name: "TC-invalid perf duration less than min value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: minPerfDur - 1, - }}, - wantErr: true, - wantMsg: strconv.Itoa(minPerfDur), - }, - { - name: "TC-invalid perf duration greater than max value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur + 1, - }}, - wantErr: true, - wantMsg: strconv.Itoa(maxPerfDur), - }, - { - name: "TC-invalid percent value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, - L3Percent: config.MultiLvlPercent{ - Low: minPercent - 1, - }, - }}, - wantErr: true, - wantMsg: strconv.Itoa(minPercent), - }, - { - name: "TC-invalid l3 percent low value larger than mid value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, - L3Percent: config.MultiLvlPercent{ - Low: minPercent + 2, - Mid: minPercent + 1, - High: minPercent + 1, - }, - MemBandPercent: config.MultiLvlPercent{ - Low: minPercent, - Mid: minPercent + 1, - High: minPercent + 2, - }, - }}, - wantErr: true, - wantMsg: "low<=mid<=high", - }, - { - name: "TC-invalid memband percent mid value larger than high value", - args: args{cfg: config.CacheConfig{ - DefaultLimitMode: staticMode, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, - L3Percent: config.MultiLvlPercent{ - Low: minPercent, - Mid: minPercent + 1, - High: minPercent + 2, - }, - MemBandPercent: config.MultiLvlPercent{ - Low: minPercent, - Mid: maxPercent/2 + 1, - High: maxPercent / 2, - }, - }}, - wantErr: true, - wantMsg: "low<=mid<=high", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := checkCacheCfg(&tt.args.cfg) - if (err != nil) != tt.wantErr { - t.Errorf("checkCacheCfg() error = %v, wantErr %v", err, tt.wantErr) - } - if err != nil && !strings.Contains(err.Error(), tt.wantMsg) { - t.Errorf("checkCacheCfg() error = %v, wantMsg %v", err, tt.wantMsg) - } - }) - } -} - -func setMaskFile(t *testing.T, resctrlDir string, data string) error { - maskDir := filepath.Join(resctrlDir, "info", "L3") - maskFile := filepath.Join(maskDir, "cbm_mask") - if err := os.MkdirAll(maskDir, constant.DefaultDirMode); err != nil { - return err - } - if err := ioutil.WriteFile(maskFile, []byte(data), constant.DefaultFileMode); err != nil { - return err - } - return nil -} - -// TestInitCacheLimitDir testcase -func TestInitCacheLimitDir(t *testing.T) { - resctrlDir := try.GenTestDir().String() - type args struct { - cfg config.CacheConfig - } - tests := []struct { - setMaskFile func(t *testing.T) error - name string - args args - wantErr bool - }{ - { - name: "TC-valid cache limit dir setting", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - DefaultLimitMode: staticMode, - }}, - setMaskFile: func(t *testing.T) error { - return setMaskFile(t, resctrlDir, "3ff") - }, - }, - { - name: "TC-empty resctrl dir", - args: args{config.CacheConfig{ - DefaultResctrlDir: "", - DefaultLimitMode: staticMode, - }}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.setMaskFile != nil { - assert.NoError(t, tt.setMaskFile(t)) - } - if err := initCacheLimitDir(&tt.args.cfg); (err != nil) != tt.wantErr { - t.Errorf("initCacheLimitDir() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -// TestSetClDir testcase -func TestSetClDir(t *testing.T) { - testRoot := try.GenTestDir().String() - _, err := os.Create(filepath.Join(testRoot, "test")) - assert.NoError(t, err) - type fields struct { - level string - clDir string - L3Percent int - MbPercent int - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "TC-normal cache limit dir", - fields: fields{clDir: testRoot}, - }, - { - name: "TC-empty dir", - wantErr: true, - }, - { - name: "TC-path not exist", - fields: fields{clDir: "/path/not/exist"}, - wantErr: true, - }, - { - name: "TC-path not exist", - fields: fields{clDir: filepath.Join(testRoot, "test", "test")}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clSet := &cacheLimitSet{ - level: tt.fields.level, - clDir: tt.fields.clDir, - L3Percent: tt.fields.L3Percent, - MbPercent: tt.fields.MbPercent, - } - if err := clSet.setClDir(); (err != nil) != tt.wantErr { - t.Errorf("cacheLimitSet.setClDir() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -// TestCheckResctrlExist testcase -func TestCheckResctrlExist(t *testing.T) { - resctrlDir := try.GenTestDir().String() - resctrlDirNoSchemataFile := try.GenTestDir().String() - schemataPath := filepath.Join(resctrlDir, schemataFile) - _, err := os.Create(schemataPath) - assert.NoError(t, err) - type args struct { - cfg config.CacheConfig - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "TC-resctrl exist", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - }}, - }, - { - name: "TC-resctrl exist but not schemata file", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDirNoSchemataFile, - }}, - wantErr: true, - }, - { - name: "TC-resctrl not exist", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: "/path/not/exist", - }}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := checkResctrlExist(&tt.args.cfg); (err != nil) != tt.wantErr { - t.Errorf("checkResctrlExist() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -// TestDoFlush testcase -func TestAdjustCacheLimit(t *testing.T) { - resctrlDir := try.GenTestDir().String() - assert.NoError(t, setMaskFile(t, resctrlDir, "3ff")) - - type fields struct { - level string - clDir string - L3Percent int - MbPercent int - } - type args struct { - clValue string - } - tests := []struct { - preHook func(t *testing.T) - name string - fields fields - args args - wantErr bool - }{ - { - name: "TC-adjust success", - fields: fields{ - level: lowLevel, - clDir: filepath.Join(filepath.Clean(resctrlDir), dirPrefix+lowLevel), - L3Percent: 10, - MbPercent: 10, - }, - }, - { - name: "TC-l3PercentDynamic", - fields: fields{ - level: lowLevel, - clDir: filepath.Join(filepath.Clean(resctrlDir), dirPrefix+lowLevel), - L3Percent: l3PercentDynamic, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clSet := &cacheLimitSet{ - level: tt.fields.level, - clDir: tt.fields.clDir, - L3Percent: tt.fields.L3Percent, - MbPercent: tt.fields.MbPercent, - } - if tt.preHook != nil { - tt.preHook(t) - } - if err := clSet.doFlush(); (err != nil) != tt.wantErr { - t.Errorf("clSet.doFlush() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestGetPodCacheMiss(t *testing.T) { - if !perf.HwSupport() { - t.Skipf("%s only run on physical machine", t.Name()) - } - testCGRoot := filepath.Join(config.CgroupRoot, "perf_event", t.Name()) - type fields struct { - podID string - cgroupPath string - cacheLimitLevel string - containers map[string]*typedef.ContainerInfo - } - type args struct { - cgroupRoot string - perfDu int - } - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - fields fields - args args - want int - }{ - { - name: "TC-get pod cache miss success", - fields: fields{ - podID: "abcd", - cgroupPath: t.Name(), - cacheLimitLevel: lowLevel, - containers: make(map[string]*typedef.ContainerInfo), - }, - preHook: func(t *testing.T) { - try.MkdirAll(testCGRoot, constant.DefaultDirMode) - try.WriteFile(filepath.Join(testCGRoot, "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - }, - postHook: func(t *testing.T) { - try.WriteFile(filepath.Join(config.CgroupRoot, "perf_event", "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - try.RemoveAll(testCGRoot) - }, - args: args{cgroupRoot: config.CgroupRoot, perfDu: 1}, - }, - { - name: "TC-get pod cache miss failed", - fields: fields{ - podID: "abcd", - cgroupPath: t.Name(), - cacheLimitLevel: middleLevel, - containers: make(map[string]*typedef.ContainerInfo), - }, - }, - } - for _, tt := range tests { - name := t.Name() - fmt.Println(name) - t.Run(tt.name, func(t *testing.T) { - p := &typedef.PodInfo{ - UID: tt.fields.podID, - CgroupPath: tt.fields.cgroupPath, - CacheLimitLevel: tt.fields.cacheLimitLevel, - Containers: tt.fields.containers, - } - if tt.preHook != nil { - tt.preHook(t) - } - getPodCacheMiss(p, tt.args.perfDu) - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} - -func TestStartDynamic(t *testing.T) { - if !perf.HwSupport() { - t.Skipf("%s only run on physical machine", t.Name()) - } - initCpm() - startDynamic(&config.CacheConfig{}, 0, 0) - resctrlDir := try.GenTestDir().String() - testCGRoot := filepath.Join(config.CgroupRoot, "perf_event", t.Name()) - assert.NoError(t, setMaskFile(t, resctrlDir, "3ff")) - - type args struct { - minWaterLine, maxWaterLine, wantL3, wantMb, WantFinalL3, wantFinalMb int - cfg config.CacheConfig - } - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - args args - }{ - { - name: "TC-start dynamic", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - DefaultLimitMode: dynamicMode, - PerfDuration: 10, - L3Percent: config.MultiLvlPercent{ - High: 50, - Low: 20, - Mid: 30, - }, - MemBandPercent: config.MultiLvlPercent{ - High: 50, - Low: 10, - Mid: 30, - }, - }, - minWaterLine: 0, - maxWaterLine: 0, - wantL3: 20, - wantMb: 10, - WantFinalL3: 20, - wantFinalMb: 10, - }, - preHook: func(t *testing.T) { - pi := &typedef.PodInfo{ - UID: "abcde", - CgroupPath: filepath.Base(testCGRoot), - CacheLimitLevel: lowLevel, - Containers: make(map[string]*typedef.ContainerInfo), - } - cpm.Checkpoint.Pods[pi.UID] = pi - try.MkdirAll(testCGRoot, constant.DefaultDirMode) - try.WriteFile(filepath.Join(testCGRoot, "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - }, - postHook: func(t *testing.T) { - try.WriteFile(filepath.Join(config.CgroupRoot, "perf_event", "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - try.RemoveAll(testCGRoot) - cpm.Checkpoint.Pods = make(map[string]*typedef.PodInfo) - }, - }, - { - name: "TC-start dynamic with very high water line", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - DefaultLimitMode: dynamicMode, - PerfDuration: 10, - L3Percent: config.MultiLvlPercent{ - High: 50, - Low: 20, - Mid: 30, - }, - MemBandPercent: config.MultiLvlPercent{ - High: 50, - Low: 10, - Mid: 30, - }, - }, - minWaterLine: math.MaxInt64, - maxWaterLine: math.MaxInt64, - wantL3: 25, - wantMb: 15, - WantFinalL3: 50, - wantFinalMb: 50, - }, - preHook: func(t *testing.T) { - pi := &typedef.PodInfo{ - UID: "abcde", - CgroupPath: filepath.Base(testCGRoot), - CacheLimitLevel: lowLevel, - Containers: make(map[string]*typedef.ContainerInfo), - } - cpm.Checkpoint.Pods[pi.UID] = pi - try.MkdirAll(testCGRoot, constant.DefaultDirMode) - try.WriteFile(filepath.Join(testCGRoot, "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - }, - postHook: func(t *testing.T) { - try.WriteFile(filepath.Join(config.CgroupRoot, "perf_event", "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - try.RemoveAll(testCGRoot) - cpm.Checkpoint.Pods = make(map[string]*typedef.PodInfo) - }, - }, - { - name: "TC-start dynamic with low min water line", - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - DefaultLimitMode: dynamicMode, - PerfDuration: 10, - L3Percent: config.MultiLvlPercent{ - High: 50, - Low: 20, - Mid: 30, - }, - MemBandPercent: config.MultiLvlPercent{ - High: 50, - Low: 10, - Mid: 30, - }, - }, - minWaterLine: 0, - maxWaterLine: math.MaxInt64, - wantL3: 20, - wantMb: 10, - WantFinalL3: 20, - wantFinalMb: 10, - }, - preHook: func(t *testing.T) { - pi := &typedef.PodInfo{ - UID: "abcde", - CgroupPath: filepath.Base(testCGRoot), - CacheLimitLevel: lowLevel, - Containers: make(map[string]*typedef.ContainerInfo), - } - cpm.Checkpoint.Pods[pi.UID] = pi - try.MkdirAll(testCGRoot, constant.DefaultDirMode) - try.WriteFile(filepath.Join(testCGRoot, "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - }, - postHook: func(t *testing.T) { - try.WriteFile(filepath.Join(config.CgroupRoot, "perf_event", "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - try.RemoveAll(testCGRoot) - cpm.Checkpoint.Pods = make(map[string]*typedef.PodInfo) - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.preHook != nil { - tt.preHook(t) - } - - l3PercentDynamic = tt.args.cfg.L3Percent.Low - mbPercentDynamic = tt.args.cfg.MemBandPercent.Low - startDynamic(&tt.args.cfg, tt.args.maxWaterLine, tt.args.minWaterLine) - assert.Equal(t, tt.args.wantL3, l3PercentDynamic) - assert.Equal(t, tt.args.wantMb, mbPercentDynamic) - for i := 0; i < 10; i++ { - startDynamic(&tt.args.cfg, tt.args.maxWaterLine, tt.args.minWaterLine) - } - assert.Equal(t, tt.args.WantFinalL3, l3PercentDynamic) - assert.Equal(t, tt.args.wantFinalMb, mbPercentDynamic) - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} - -func TestClEnabled(t *testing.T) { - oldEnbaled := enable - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - want bool - }{ - { - name: "TC-return enabled", - preHook: func(t *testing.T) { - enable = true - }, - postHook: func(t *testing.T) { - enable = oldEnbaled - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.preHook != nil { - tt.preHook(t) - } - if got := ClEnabled(); got != tt.want { - t.Errorf("ClEnabled() = %v, want %v", got, tt.want) - } - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} - -// TestDynamicExist test dynamicExist -func TestDynamicExist(t *testing.T) { - initCpm() - cpm.Checkpoint.Pods["podabc"].CacheLimitLevel = lowLevel - assert.Equal(t, false, dynamicExist()) - cpm.Checkpoint.Pods["podabc"].CacheLimitLevel = dynamicLevel - assert.Equal(t, true, dynamicExist()) -} - -// TestIsHostPidns test isHostPidns -func TestIsHostPidns(t *testing.T) { - assert.Equal(t, false, isHostPidns(filepath.Join(constant.TmpTestDir, "path/not/exist/pid"))) - assert.Equal(t, true, isHostPidns("/proc/self/ns/pid")) -} - -// TestInit test Init -func TestInit(t *testing.T) { - resctrlDir := try.GenTestDir().String() - schemataPath := filepath.Join(resctrlDir, schemataFile) - _, err := os.Create(schemataPath) - assert.NoError(t, err) - assert.NoError(t, setMaskFile(t, resctrlDir, "3ff")) - var TC1WantErr bool - if !perf.HwSupport() { - TC1WantErr = true - } - type args struct { - cfg config.CacheConfig - } - tests := []struct { - preHook func(t *testing.T) - postHook func(t *testing.T) - name string - args args - wantErr bool - }{ - { - name: "TC-normal testcase", - wantErr: TC1WantErr, - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: resctrlDir, - DefaultLimitMode: dynamicMode, - PerfDuration: 10, - L3Percent: config.MultiLvlPercent{ - High: 100, - Low: 10, - Mid: 50, - }, - MemBandPercent: config.MultiLvlPercent{ - High: 100, - Low: 10, - Mid: 50, - }, - AdjustInterval: 10, - }}, - }, - { - name: "TC-invalid cache config", - wantErr: true, - args: args{cfg: config.CacheConfig{ - AdjustInterval: 0, - }}, - }, - { - name: "TC-resctrl not exist", - wantErr: true, - args: args{cfg: config.CacheConfig{ - DefaultResctrlDir: "/path/not/exist", - DefaultLimitMode: dynamicMode, - PerfDuration: 10, - L3Percent: config.MultiLvlPercent{ - High: 100, - Low: 10, - Mid: 50, - }, - MemBandPercent: config.MultiLvlPercent{ - High: 100, - Low: 10, - Mid: 50, - }, - AdjustInterval: 10, - }}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.preHook != nil { - tt.preHook(t) - } - var cfg config.CacheConfig - cfg = tt.args.cfg - m := &checkpoint.Manager{ - Checkpoint: &checkpoint.Checkpoint{ - Pods: make(map[string]*typedef.PodInfo), - }, - } - if err := Init(m, &cfg); (err != nil) != tt.wantErr { - t.Errorf("Init() error = %v, wantErr %v", err, tt.wantErr) - } - if tt.postHook != nil { - tt.postHook(t) - } - }) - } -} diff --git a/pkg/cachelimit/cachelimit_test.go b/pkg/cachelimit/cachelimit_test.go deleted file mode 100644 index ec7a36f..0000000 --- a/pkg/cachelimit/cachelimit_test.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li, Danni Xia -// Create: 2022-05-16 -// Description: offline pod cache limit function - -// Package cachelimit is for cache limiting -package cachelimit - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/try" - "isula.org/rubik/pkg/typedef" - - "github.com/stretchr/testify/assert" -) - -var podInfo = typedef.PodInfo{ - CgroupPath: "kubepods/podaaa", - Offline: true, - CacheLimitLevel: "dynamic", -} - -func initCpm() { - podID := "podabc" - cpm = &checkpoint.Manager{ - Checkpoint: &checkpoint.Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podID: &podInfo, - }, - }, - } -} - -// TestLevelValid testcase -func TestLevelValid(t *testing.T) { - type args struct { - level string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "TC-normal cache limit level", - args: args{level: lowLevel}, - want: true, - }, - { - name: "TC-abnormal cache limit level", - args: args{level: "abnormal level"}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := levelValid(tt.args.level); got != tt.want { - t.Errorf("levelValid() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestNewCacheLimitPodInfo test NewCacheLimitPodInfo -func TestSyncLevel(t *testing.T) { - podInfo := typedef.PodInfo{ - CgroupPath: "kubepods/podaaa", - Offline: true, - CacheLimitLevel: "invalid", - } - err := SyncLevel(&podInfo) - assert.Equal(t, true, err != nil) - - podInfo.CacheLimitLevel = lowLevel - err = SyncLevel(&podInfo) - assert.NoError(t, err) - assert.Equal(t, podInfo.CacheLimitLevel, lowLevel) - - defaultLimitMode = staticMode - podInfo.CacheLimitLevel = "" - err = SyncLevel(&podInfo) - assert.NoError(t, err) - assert.Equal(t, podInfo.CacheLimitLevel, maxLevel) - - defaultLimitMode = dynamicMode - podInfo.CacheLimitLevel = "" - err = SyncLevel(&podInfo) - assert.NoError(t, err) - assert.Equal(t, podInfo.CacheLimitLevel, dynamicLevel) -} - -// TestWriteTasksToResctrl test writeTasksToResctrl -func TestWriteTasksToResctrl(t *testing.T) { - initCpm() - err := SyncLevel(&podInfo) - assert.NoError(t, err) - - testDir := try.GenTestDir().String() - config.CgroupRoot = testDir - - pid, procsFile, container := "12345", "cgroup.procs", "container1" - podCPUCgroupPath := filepath.Join(testDir, "cpu", podInfo.CgroupPath) - try.MkdirAll(filepath.Join(podCPUCgroupPath, container), constant.DefaultDirMode) - err = writeTasksToResctrl(&podInfo, testDir) - // pod cgroup.procs not exist, return error - assert.Equal(t, true, err != nil) - _, err = os.Create(filepath.Join(podCPUCgroupPath, procsFile)) - assert.NoError(t, err) - try.WriteFile(filepath.Join(podCPUCgroupPath, container, procsFile), []byte(pid), constant.DefaultFileMode) - - err = writeTasksToResctrl(&podInfo, testDir) - // resctrl tasks file not exist, return error - assert.Equal(t, true, err != nil) - - resctrlSubDir, taskFile := dirPrefix+podInfo.CacheLimitLevel, "tasks" - try.MkdirAll(filepath.Join(testDir, resctrlSubDir), constant.DefaultDirMode) - err = writeTasksToResctrl(&podInfo, testDir) - // write success - assert.NoError(t, err) - bytes, err := ioutil.ReadFile(filepath.Join(testDir, resctrlSubDir, taskFile)) - assert.NoError(t, err) - assert.Equal(t, pid, strings.TrimSpace(string(bytes))) - - // container pid already written - err = writeTasksToResctrl(&podInfo, testDir) - assert.NoError(t, err) - - config.CgroupRoot = constant.DefaultCgroupRoot -} - -// TestSetCacheLimit test SetCacheLimit -func TestSetCacheLimit(t *testing.T) { - initCpm() - err := SetCacheLimit(&podInfo) - assert.NoError(t, err) -} - -// TestSyncCacheLimit test syncCacheLimit -func TestSyncCacheLimit(t *testing.T) { - initCpm() - syncCacheLimit() -} diff --git a/pkg/checkpoint/checkpoint.go b/pkg/checkpoint/checkpoint.go deleted file mode 100644 index c33ef50..0000000 --- a/pkg/checkpoint/checkpoint.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-04-27 -// Description: provide pods checkpoint management - -// Package checkpoint provide pods checkpoint management. -package checkpoint - -import ( - "path/filepath" - "strings" - "sync" - - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -// Checkpoint stores the binding between the CPU and pod container. -type Checkpoint struct { - Pods map[string]*typedef.PodInfo `json:"pods,omitempty"` -} - -// Manager manage checkpoint -type Manager struct { - Checkpoint *Checkpoint - CgroupRoot string - sync.Mutex -} - -// NewManager create manager -func NewManager(cgroupRoot string) *Manager { - return &Manager{ - Checkpoint: &Checkpoint{ - Pods: make(map[string]*typedef.PodInfo, 0), - }, - CgroupRoot: cgroupRoot, - } -} - -// AddPod returns pod info from pod ID -func (cm *Manager) AddPod(pod *corev1.Pod) { - // Before adding pod to the checkpoint, ensure that the pod is in the running state. Otherwise, problems may occur. - // Only pod.Status.Phase == corev1.PodRunning - cm.Lock() - defer cm.Unlock() - if pod == nil || string(pod.UID) == "" { - return - } - if _, ok := cm.Checkpoint.Pods[string(pod.UID)]; ok { - log.Debugf("pod %v is existed", string(pod.UID)) - return - } - log.Debugf("add pod %v", string(pod.UID)) - cm.Checkpoint.Pods[string(pod.UID)] = NewPodInfo(pod, cm.CgroupRoot) -} - -// GetPod returns pod info from pod ID -func (cm *Manager) GetPod(podID types.UID) *typedef.PodInfo { - cm.Lock() - defer cm.Unlock() - return cm.Checkpoint.Pods[string(podID)].Clone() -} - -// PodExist returns true if there is a pod whose key is podID in the checkpoint -func (cm *Manager) PodExist(podID types.UID) bool { - cm.Lock() - defer cm.Unlock() - _, ok := cm.Checkpoint.Pods[string(podID)] - return ok -} - -// ContainerExist returns true if there is a pod whose key is podID in the checkpoint -func (cm *Manager) ContainerExist(podID types.UID, containerID string) bool { - cm.Lock() - defer cm.Unlock() - if _, ok := cm.Checkpoint.Pods[string(podID)]; !ok { - return false - } - _, ok := cm.Checkpoint.Pods[string(podID)].Containers[containerID] - return ok -} - -// DelPod delete pod from checkpoint -func (cm *Manager) DelPod(podID types.UID) { - cm.Lock() - defer cm.Unlock() - if _, ok := cm.Checkpoint.Pods[string(podID)]; !ok { - log.Debugf("pod %v is not existed", string(podID)) - return - } - log.Debugf("delete pod %v", podID) - delete(cm.Checkpoint.Pods, string(podID)) -} - -// UpdatePod updates pod information based on pods -func (cm *Manager) UpdatePod(pod *corev1.Pod) { - cm.Lock() - defer cm.Unlock() - old, ok := cm.Checkpoint.Pods[string(pod.UID)] - if !ok { - log.Debugf("pod %v is not existed", string(pod.UID)) - return - } - log.Debugf("update pod %v", string(pod.UID)) - updatePodInfoNoLock(old, pod) -} - -// SyncFromCluster synchronizing data from the kubernetes cluster using the list mechanism at the beginning -func (cm *Manager) SyncFromCluster(items []corev1.Pod) { - cm.Lock() - defer cm.Unlock() - for _, pod := range items { - if string(pod.UID) == "" { - continue - } - log.Debugf("add pod %v", string(pod.UID)) - cm.Checkpoint.Pods[string(pod.UID)] = NewPodInfo(&pod, cm.CgroupRoot) - } -} - -// filter filtering for list functions -type filter func(pi *typedef.PodInfo) bool - -// listContainersWithFilters filters and returns deep copy objects of all containers -func (cm *Manager) listContainersWithFilters(filters ...filter) map[string]*typedef.ContainerInfo { - cm.Lock() - defer cm.Unlock() - cc := make(map[string]*typedef.ContainerInfo, len(cm.Checkpoint.Pods)) - - for _, pod := range cm.Checkpoint.Pods { - if !mergeFilters(pod, filters) { - continue - } - for _, ci := range pod.Containers { - cc[ci.ID] = ci.Clone() - } - } - - return cc -} - -// ListPodsWithFilters filters and returns deep copy objects of all pod -func (cm *Manager) listPodsWithFilters(filters ...filter) map[string]*typedef.PodInfo { - cm.Lock() - defer cm.Unlock() - pc := make(map[string]*typedef.PodInfo, 0) - - for _, pod := range cm.Checkpoint.Pods { - if !mergeFilters(pod, filters) { - continue - } - pc[pod.UID] = pod.Clone() - } - return pc -} - -func mergeFilters(pi *typedef.PodInfo, filters []filter) bool { - for _, f := range filters { - if !f(pi) { - return false - } - } - return true -} - -// ListOfflineContainers filtering offline containers -func (cm *Manager) ListOfflineContainers() map[string]*typedef.ContainerInfo { - return cm.listContainersWithFilters(func(pi *typedef.PodInfo) bool { - return pi.Offline && pi.Namespace != v1.NamespaceSystem - }) -} - -// ListAllContainers returns all containers copies -func (cm *Manager) ListAllContainers() map[string]*typedef.ContainerInfo { - return cm.listContainersWithFilters() -} - -// ListAllPods returns all pods copies -func (cm *Manager) ListAllPods() map[string]*typedef.PodInfo { - return cm.listPodsWithFilters() -} - -// ListOfflinePods returns all pods copies -func (cm *Manager) ListOfflinePods() map[string]*typedef.PodInfo { - return cm.listPodsWithFilters(func(pi *typedef.PodInfo) bool { - return pi.Offline && pi.Namespace != v1.NamespaceSystem - }) -} - -// ListOnlinePods returns all pods copies -func (cm *Manager) ListOnlinePods() map[string]*typedef.PodInfo { - return cm.listPodsWithFilters(func(pi *typedef.PodInfo) bool { - return !pi.Offline && pi.Namespace != v1.NamespaceSystem - }) -} - -// NewPodInfo create PodInfo -func NewPodInfo(pod *corev1.Pod, cgroupRoot string) *typedef.PodInfo { - pi := &typedef.PodInfo{ - Name: pod.Name, - UID: string(pod.UID), - Containers: make(map[string]*typedef.ContainerInfo, 0), - CgroupPath: util.GetPodCgroupPath(pod), - Namespace: pod.Namespace, - CgroupRoot: cgroupRoot, - } - updatePodInfoNoLock(pi, pod) - return pi -} - -// updatePodInfoNoLock updates PodInfo from the pod of Kubernetes. -// UpdatePodInfoNoLock does not lock pods during the modification. -// Therefore, ensure that the pod is being used only by this function. -// Currently, the checkpoint manager variable is locked when this function is invoked. -func updatePodInfoNoLock(pi *typedef.PodInfo, pod *corev1.Pod) { - const ( - dockerPrefix = "docker://" - containerdPrefix = "containerd://" - ) - pi.Name = pod.Name - pi.Offline = util.IsOffline(pod) - pi.CacheLimitLevel = util.GetPodCacheLimit(pod) - pi.QuotaBurst = util.GetQuotaBurst(pod) - - nameID := make(map[string]string, len(pod.Status.ContainerStatuses)) - for _, c := range pod.Status.ContainerStatuses { - // Rubik is compatible with dockerd and containerd container engines. - cid := strings.TrimPrefix(c.ContainerID, dockerPrefix) - cid = strings.TrimPrefix(cid, containerdPrefix) - - // the container may be in the creation or deletion phase. - if len(cid) == 0 { - log.Debugf("no container id found of container %v", c.Name) - continue - } - nameID[c.Name] = cid - } - // update ContainerInfo in a PodInfo - for _, c := range pod.Spec.Containers { - ci, ok := pi.Containers[c.Name] - // add a container - if !ok { - log.Debugf("add new container %v", c.Name) - pi.AddContainerInfo(typedef.NewContainerInfo(c, string(pod.UID), nameID[c.Name], - pi.CgroupRoot, pi.CgroupPath)) - continue - } - // The container name remains unchanged, and other information about the container is updated. - ci.ID = nameID[c.Name] - ci.CgroupAddr = filepath.Join(pi.CgroupPath, ci.ID) - } - // delete a container that does not exist - for name := range pi.Containers { - if _, ok := nameID[name]; !ok { - log.Debugf("delete container %v", name) - delete(pi.Containers, name) - } - } -} diff --git a/pkg/checkpoint/checkpoint_test.go b/pkg/checkpoint/checkpoint_test.go deleted file mode 100644 index e9167a6..0000000 --- a/pkg/checkpoint/checkpoint_test.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-05-10 -// Description: checkpoint DT test - -package checkpoint - -import ( - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/typedef" -) - -var containerInfos = []*typedef.ContainerInfo{ - { - Name: "FooCon", - ID: "testCon1", - PodID: "testPod1", - CgroupRoot: constant.DefaultCgroupRoot, - CgroupAddr: "kubepods/testPod1/testCon1", - }, - { - Name: "BarCon", - ID: "testCon2", - PodID: "testPod2", - CgroupRoot: constant.DefaultCgroupRoot, - CgroupAddr: "kubepods/testPod2/testCon2", - }, - { - Name: "BiuCon", - ID: "testCon3", - PodID: "testPod3", - CgroupRoot: constant.DefaultCgroupRoot, - CgroupAddr: "kubepods/testPod3/testCon3", - }, - { - Name: "PahCon", - ID: "testCon4", - PodID: "testPod4", - CgroupRoot: constant.DefaultCgroupRoot, - CgroupAddr: "kubepods/testPod4/testCon4", - }, -} - -var podInfos = []*typedef.PodInfo{ - // allow quota adjustment - { - Name: "FooPod", - UID: containerInfos[0].PodID, - Containers: map[string]*typedef.ContainerInfo{ - containerInfos[0].Name: containerInfos[0], - }, - }, - // allow quota adjustment - { - Name: "BarPod", - UID: containerInfos[1].PodID, - Containers: map[string]*typedef.ContainerInfo{ - containerInfos[1].Name: containerInfos[1], - }, - }, - // quota adjustment is not allowed - { - Name: "BiuPod", - UID: containerInfos[2].PodID, - Containers: map[string]*typedef.ContainerInfo{ - containerInfos[2].Name: containerInfos[2], - }, - }, - // quota adjustment is not allowed - { - Name: "PahPod", - UID: containerInfos[3].PodID, - Containers: map[string]*typedef.ContainerInfo{ - containerInfos[3].Name: containerInfos[3], - }, - }, -} - -var coreV1Pods = []corev1.Pod{ - { - Status: corev1.PodStatus{ - Phase: corev1.PodFailed, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("testPod5"), - Name: "BiuPod", - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - QOSClass: corev1.PodQOSGuaranteed, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "BiuCon", - ContainerID: "docker://testCon5", - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "BiuCon", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": *resource.NewQuantity(2, resource.DecimalSI), - }, - Limits: corev1.ResourceList{ - "cpu": *resource.NewQuantity(3, resource.DecimalSI), - "memory": *resource.NewQuantity(300, resource.DecimalSI), - }, - }, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID(podInfos[1].UID), - Name: podInfos[1].Name, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - QOSClass: corev1.PodQOSGuaranteed, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "BarCon1", - ContainerID: "docker://testCon6", - }, - { - Name: "BarCon2", - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "BarCon1", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": *resource.NewQuantity(2, resource.DecimalSI), - }, - Limits: corev1.ResourceList{ - "cpu": *resource.NewQuantity(2, resource.DecimalSI), - "memory": *resource.NewQuantity(100, resource.DecimalSI), - }, - }, - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID(podInfos[0].UID), - Name: "FooPod", - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - QOSClass: corev1.PodQOSGuaranteed, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "FooCon", - ContainerID: "docker://" + containerInfos[0].ID, - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "FooCon", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": *resource.NewQuantity(2, resource.DecimalSI), - }, - Limits: corev1.ResourceList{ - "cpu": *resource.NewQuantity(3, resource.DecimalSI), - "memory": *resource.NewQuantity(300, resource.DecimalSI), - }, - }, - }, - }, - }, - }, -} - -// TestManagerAddPod tests AddPod of Manager -func TestManagerAddPod(t *testing.T) { - var ( - podNum1 = 1 - podNum2 = 2 - ) - cpm := &Manager{ - Checkpoint: &Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podInfos[0].UID: podInfos[0].Clone(), - }, - }, - } - // 1. add pods that do not exist - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum1) - var mockPahPod = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID(podInfos[3].UID), - Name: podInfos[3].Name, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - QOSClass: corev1.PodQOSBurstable, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "PahCon", - ContainerID: "docker://" + containerInfos[3].ID, - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "PahCon", - }, - }, - }, - } - cpm.AddPod(mockPahPod) - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum2) - // 2.join a joined pods - cpm.AddPod(mockPahPod) - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum2) - - // 3.add a pod whose name is empty - cpm.AddPod(&coreV1Pods[0]) - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum2) -} - -// TestManagerDelPod tests DelPod of Manager -func TestManagerDelPod(t *testing.T) { - var ( - podNum0 = 0 - podNum1 = 1 - ) - cpm := &Manager{ - Checkpoint: &Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podInfos[0].UID: podInfos[0].Clone(), - }, - }, - } - // 1. delete pods that do not exist - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum1) - cpm.DelPod(types.UID(podInfos[1].UID)) - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum1) - // 2. delete existed pods - cpm.DelPod(types.UID(podInfos[0].UID)) - assert.Equal(t, len(cpm.Checkpoint.Pods), podNum0) -} - -// TestManagerUpdatePod test UpdatePod function of Manager -func TestManagerUpdatePod(t *testing.T) { - var managerUpdatePodTests = []struct { - pod *corev1.Pod - judgement func(t *testing.T, m *Manager) - name string - }{ - { - name: "TC1 - update a non-added pod", - pod: &coreV1Pods[1], - judgement: func(t *testing.T, m *Manager) { - podNum2 := 2 - assert.Equal(t, podNum2, len(m.Checkpoint.Pods)) - }, - }, - } - cpm := &Manager{ - Checkpoint: &Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podInfos[0].UID: podInfos[0].Clone(), - podInfos[1].UID: podInfos[1].Clone(), - }, - }, - } - - for _, tt := range managerUpdatePodTests { - t.Run(tt.name, func(t *testing.T) { - cpm.UpdatePod(tt.pod) - tt.judgement(t, cpm) - }) - } -} - -// TestManagerListPodsAndContainers tests methods of list pods and containers of Manager -func TestManagerListPodsAndContainers(t *testing.T) { - var ( - podNum3 = 3 - podNum4 = 4 - ) - // The pod names in Kubernetes must be unique. The same pod cannot have the same name, but different pods can have the same name. - cpm := &Manager{ - Checkpoint: &Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podInfos[0].UID: podInfos[0].Clone(), - podInfos[1].UID: podInfos[1].Clone(), - podInfos[2].UID: podInfos[2].Clone(), - }, - }, - } - // 1. Containers with Different Pods with Different Names - assert.Equal(t, len(cpm.ListAllPods()), podNum3) - assert.Equal(t, len(cpm.ListAllContainers()), podNum3) - // 2. Containers with the same name in different pods - var podWithSameNameCon = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: types.UID("testPod5"), - Name: "FakeFooPod", - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - ContainerStatuses: []corev1.ContainerStatus{ - { - Name: "FooCon", - ContainerID: "docker://testCon5", - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "FooCon", - }, - }, - }, - } - - cpm.AddPod(podWithSameNameCon) - assert.Equal(t, len(cpm.ListAllContainers()), podNum4) -} - -// TestManagerSyncFromCluster tests SyncFromCluster of Manager -func TestManagerSyncFromCluster(t *testing.T) { - cpm := NewManager("") - cpm.Checkpoint = &Checkpoint{ - Pods: make(map[string]*typedef.PodInfo, 0), - } - - cpm.SyncFromCluster(coreV1Pods) - expPodNum := 3 - assert.Equal(t, len(cpm.Checkpoint.Pods), expPodNum) - - pi2 := cpm.GetPod(coreV1Pods[1].UID) - assert.Equal(t, "BiuPod", pi2.Name) -} - -// TestMangerPodExist tests the PodExist of Manger -func TestMangerPodExist(t *testing.T) { - tests := []struct { - name string - id types.UID - want bool - }{ - { - name: "TC1 - check a non-existed pod", - id: types.UID(podInfos[0].UID), - want: true, - }, - { - name: "TC2 - check an existed pod", - id: types.UID(podInfos[1].UID), - want: false, - }, - } - cpm := NewManager("") - cpm.Checkpoint = &Checkpoint{ - Pods: map[string]*typedef.PodInfo{ - podInfos[0].UID: podInfos[0].Clone(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, cpm.PodExist(tt.id)) - }) - } -} diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index 5938c87..0000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-04-26 -// Description: config load - -package config - -import ( - "bytes" - "encoding/json" - "path/filepath" - - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/util" -) - -var ( - // CgroupRoot is cgroup mount point - CgroupRoot = constant.DefaultCgroupRoot - // ShutdownFlag is rubik shutdown flag - ShutdownFlag int32 - // ShutdownChan is rubik shutdown channel - ShutdownChan = make(chan struct{}) -) - -// Config defines the configuration for rubik -type Config struct { - AutoCheck bool `json:"autoCheck,omitempty"` - LogDriver string `json:"logDriver,omitempty"` - LogDir string `json:"logDir,omitempty"` - LogSize int `json:"logSize,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - CgroupRoot string `json:"cgroupRoot,omitempty"` - CacheCfg CacheConfig `json:"cacheConfig,omitempty"` - BlkioCfg BlkioConfig `json:"blkioConfig,omitempty"` - MemCfg MemoryConfig `json:"memoryConfig,omitempty"` - NodeConfig []NodeConfig `json:"nodeConfig,omitempty"` -} - -// CacheConfig define cache limit related config -type CacheConfig struct { - Enable bool `json:"enable,omitempty"` - DefaultLimitMode string `json:"defaultLimitMode,omitempty"` - DefaultResctrlDir string `json:"-"` - AdjustInterval int `json:"adjustInterval,omitempty"` - PerfDuration int `json:"perfDuration,omitempty"` - L3Percent MultiLvlPercent `json:"l3Percent,omitempty"` - MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` -} - -// BlkioConfig defines blkio related configurations. -type BlkioConfig struct { - Enable bool `json:"enable,omitempty"` -} - -// MultiLvlPercent define multi level percentage -type MultiLvlPercent struct { - Low int `json:"low,omitempty"` - Mid int `json:"mid,omitempty"` - High int `json:"high,omitempty"` -} - -type MemoryConfig struct { - Enable bool `json:"enable,omitempty"` - Strategy string `json:"strategy,omitempty"` - CheckInterval int `json:"checkInterval,omitempty"` -} - -// NodeConfig define node configuration for each node -type NodeConfig struct { - NodeName string `json:"nodeName,omitempty"` - IOcostEnable bool `json:"iocostEnable,omitempty"` - IOcostConfig []IOcostConfig `json:"iocostConfig,omitempty"` -} - -// IOcostConfig define iocost for node -type IOcostConfig struct { - Dev string `json:"dev,omitempty"` - Enable bool `json:"enable,omitempty"` - Model string `json:"model,omitempty"` - Param Param `json:"param,omitempty"` -} - -// Param for linear model -type Param struct { - Rbps int64 `json:"rbps,omitempty"` - Rseqiops int64 `json:"rseqiops,omitempty"` - Rrandiops int64 `json:"rrandiops,omitempty"` - Wbps int64 `json:"wbps,omitempty"` - Wseqiops int64 `json:"wseqiops,omitempty"` - Wrandiops int64 `json:"wrandiops,omitempty"` -} - -// NewConfig returns new config load from config file -func NewConfig(path string) (*Config, error) { - if path == "" { - path = constant.ConfigFile - } - - defaultLogSize, defaultAdInt, defaultPerfDur := 1024, 1000, 1000 - defaultLowL3, defaultMidL3, defaultHighL3, defaultLowMB, defaultMidMB, defaultHighMB := 20, 30, 50, 10, 30, 50 - cfg := Config{ - LogDriver: "stdio", - LogDir: constant.DefaultLogDir, - LogSize: defaultLogSize, - LogLevel: "info", - CgroupRoot: constant.DefaultCgroupRoot, - CacheCfg: CacheConfig{ - Enable: false, - DefaultLimitMode: "static", - DefaultResctrlDir: "/sys/fs/resctrl", - AdjustInterval: defaultAdInt, - PerfDuration: defaultPerfDur, - L3Percent: MultiLvlPercent{ - Low: defaultLowL3, - Mid: defaultMidL3, - High: defaultHighL3, - }, - MemBandPercent: MultiLvlPercent{ - Low: defaultLowMB, - Mid: defaultMidMB, - High: defaultHighMB, - }, - }, - BlkioCfg: BlkioConfig{ - Enable: false, - }, - MemCfg: MemoryConfig{ - Enable: false, - Strategy: constant.DefaultMemStrategy, - CheckInterval: constant.DefaultMemCheckInterval, - }, - } - - defer func() { - CgroupRoot = cfg.CgroupRoot - }() - - if !util.PathExist(path) { - return &cfg, nil - } - - b, err := util.ReadSmallFile(filepath.Clean(path)) - if err != nil { - return nil, err - } - - reader := bytes.NewReader(b) - if err := json.NewDecoder(reader).Decode(&cfg); err != nil { - return nil, err - } - - return &cfg, nil -} - -// String return string format. -func (cfg *Config) String() string { - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return "{}" - } - return string(data) -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go deleted file mode 100644 index 86c3df8..0000000 --- a/pkg/config/config_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-05-07 -// Description: config load test - -package config - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" -) - -var ( - cfgA = ` -{ - "autoCheck": true, - "logDriver": "file", - "logDir": "/tmp/rubik-test", - "logSize": 2048, - "logLevel": "debug", - "cgroupRoot": "/tmp/rubik-test/cgroup" -}` -) - -// TestNewConfig is NewConfig function test -func TestNewConfig(t *testing.T) { - tmpConfigFile := filepath.Join(constant.TmpTestDir, "config.json") - os.Remove(tmpConfigFile) - - // coverage - NewConfig("") - - // test_rubik_load_config_file_0001 - defaultLogSize := 1024 - cfg, err := NewConfig(tmpConfigFile) - assert.NoError(t, err) - assert.Equal(t, cfg.AutoCheck, false) - assert.Equal(t, cfg.LogDriver, "stdio") - assert.Equal(t, cfg.LogDir, constant.DefaultLogDir) - assert.Equal(t, cfg.LogSize, defaultLogSize) - assert.Equal(t, cfg.LogLevel, "info") - assert.Equal(t, cfg.CgroupRoot, constant.DefaultCgroupRoot) - - // test_rubik_load_config_file_0003 - err = os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - - logSize := 2048 - err = ioutil.WriteFile(tmpConfigFile, []byte(cfgA), constant.DefaultFileMode) - assert.NoError(t, err) - cfg, err = NewConfig(tmpConfigFile) - assert.NoError(t, err) - assert.Equal(t, cfg.AutoCheck, true) - assert.Equal(t, cfg.LogDriver, "file") - assert.Equal(t, cfg.LogDir, "/tmp/rubik-test") - assert.Equal(t, cfg.LogSize, logSize) - assert.Equal(t, cfg.LogLevel, "debug") - assert.Equal(t, cfg.CgroupRoot, "/tmp/rubik-test/cgroup") - - // test_rubik_load_config_file_0002 - err = ioutil.WriteFile(tmpConfigFile, []byte("abc"), constant.DefaultFileMode) - assert.NoError(t, err) - _, err = NewConfig(tmpConfigFile) - assert.Contains(t, err.Error(), "invalid character") - - size := 20000000 - big := make([]byte, size, size) - err = ioutil.WriteFile(tmpConfigFile, big, constant.DefaultFileMode) - assert.NoError(t, err) - _, err = NewConfig(tmpConfigFile) - assert.Contains(t, err.Error(), "too big") - - err = os.Remove(tmpConfigFile) - assert.NoError(t, err) -} - -// TestConfig_String is config string function test -func TestConfig_String(t *testing.T) { - tmpConfigFile := filepath.Join(constant.TmpTestDir, "config.json") - os.Remove(tmpConfigFile) - - cfg, err := NewConfig(tmpConfigFile) - assert.NoError(t, err) - - cfgString := cfg.String() - assert.Equal(t, cfgString, `{ - "logDriver": "stdio", - "logDir": "/var/log/rubik", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup", - "cacheConfig": { - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - }, - "blkioConfig": {}, - "memoryConfig": { - "strategy": "none", - "checkInterval": 5 - } -}`) -} diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go deleted file mode 100644 index a1d9476..0000000 --- a/pkg/constant/constant.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: This file contains default constants used in the project - -// Package constant is for constant definition -package constant - -import ( - "errors" - "os" - "time" -) - -const ( - // RubikSock is path for rubik socket file - RubikSock = "/run/rubik/rubik.sock" - // ConfigFile is rubik config file - ConfigFile = "/var/lib/rubik/config.json" - // DefaultLogDir is default log dir - DefaultLogDir = "/var/log/rubik" - // LockFile is rubik lock file - LockFile = "/run/rubik/rubik.lock" - // ReadTimeout is timeout for http read - ReadTimeout = 60 * time.Second - // WriteTimeout is timeout for http write - WriteTimeout = 60 * time.Second - // DefaultSucceedCode is succeed code - DefaultSucceedCode = 0 - // DefaultCgroupRoot is mount point - DefaultCgroupRoot = "/sys/fs/cgroup" - // CPUCgroupFileName is name of cgroup file used for cpu qos level setting - CPUCgroupFileName = "cpu.qos_level" - // MemoryCgroupFileName is name of cgroup file used for memory qos level setting - MemoryCgroupFileName = "memory.qos_level" - // DefaultFileMode is file mode for cgroup files - DefaultFileMode os.FileMode = 0600 - // DefaultDirMode is dir default mode - DefaultDirMode os.FileMode = 0700 - // DefaultUmask is default umask - DefaultUmask = 0077 - // MaxCgroupPathLen is max cgroup path length for pod - MaxCgroupPathLen = 4096 - // MaxPodIDLen is max pod id length - MaxPodIDLen = 256 - // MaxPodsPerRequest is max pods number per http request - MaxPodsPerRequest = 100 - // TmpTestDir is tmp directory for test - TmpTestDir = "/tmp/rubik-test" - // TaskChanCapacity is capacity for task chan - TaskChanCapacity = 1024 - // WorkerNum is number of workers - WorkerNum = 1 - // KubepodsCgroup is kubepods root cgroup - KubepodsCgroup = "kubepods" - // PodCgroupNamePrefix is pod cgroup name prefix - PodCgroupNamePrefix = "pod" - // NodeNameEnvKey is node name environment variable key - NodeNameEnvKey = "RUBIK_NODE_NAME" - // PriorityAnnotationKey is annotation key to mark offline pod - PriorityAnnotationKey = "volcano.sh/preemptable" - // CacheLimitAnnotationKey is annotation key to set L3/Mb resctrl group - CacheLimitAnnotationKey = "volcano.sh/cache-limit" - // QuotaBurstAnnotationKey is annotation key to set cpu.cfs_burst_ns - QuotaBurstAnnotationKey = "volcano.sh/quota-burst-time" - // BlkioKey is annotation key to set blkio limit - BlkioKey = "volcano.sh/blkio-limit" - // DefaultMemCheckInterval indicates the default memory check interval 5s. - DefaultMemCheckInterval = 5 - // DefaultMaxMemCheckInterval indicates the default max memory check interval 30s. - DefaultMaxMemCheckInterval = 30 - // DefaultMemStrategy indicates the default memory strategy. - DefaultMemStrategy = "none" -) - -// LevelType is type definition of qos level -type LevelType int32 - -const ( - // MinLevel is min level for qos level - MinLevel LevelType = -1 - // MaxLevel is max level for qos level - MaxLevel LevelType = 0 -) - -// Int is type casting for type LevelType -func (l LevelType) Int() int { - return int(l) -} - -const ( - // ErrCodeFailed for normal failed - ErrCodeFailed = 1 -) - -// error define ref from src/internal/oserror/errors.go -var ( - // ErrFileTooBig file too big - ErrFileTooBig = errors.New("file too big") -) - -const ( - // InvalidBurst for invalid quota burst - InvalidBurst = -1 -) diff --git a/pkg/iocost/iocost.go b/pkg/iocost/iocost.go deleted file mode 100644 index 99d6da1..0000000 --- a/pkg/iocost/iocost.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: HuangYuqing -// Create: 2022-10-26 -// Description: iocost setting for pods. - -// Package iocost is for iocost. -package iocost - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "unicode" - - "github.com/pkg/errors" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -const ( - iocostModelFile = "blkio.cost.model" - iocostWeightFile = "blkio.cost.weight" - iocostQosFile = "blkio.cost.qos" - wbBlkioinoFile = "memory.wb_blkio_ino" - blkSubName = "blkio" - memSubName = "memory" - offlineWeight = 10 - onlineWeight = 1000 - paramMaxLen = 512 - devNoMax = 256 - scale = 10 - sysDevBlock = "/sys/dev/block" -) - -var ( - hwSupport = false - iocostEnable = false -) - -// HwSupport tell if the os support iocost. -func HwSupport() bool { - return hwSupport -} - -func init() { - qosFile := filepath.Join(constant.DefaultCgroupRoot, blkSubName, iocostQosFile) - if util.PathExist(qosFile) { - hwSupport = true - } -} - -// SetIOcostEnable set iocost disable or enable -func SetIOcostEnable(status bool) { - iocostEnable = status -} - -// ConfigIOcost for config iocost in cgroup v1. -func ConfigIOcost(iocostConfigArray []config.IOcostConfig) error { - if !iocostEnable { - return errors.Errorf("iocost feature is disable") - } - - if err := clearIOcost(); err != nil { - log.Infof(err.Error()) - } - - for _, iocostConfig := range iocostConfigArray { - if !iocostConfig.Enable { - // notice: dev's iocost is disable by clearIOcost - continue - } - - devno, err := getBlkDeviceNo(iocostConfig.Dev) - if err != nil { - log.Errorf(err.Error()) - continue - } - - if iocostConfig.Model == "linear" { - if err := configLinearModel(iocostConfig.Param, devno); err != nil { - log.Errorf(err.Error()) - continue - } - } else { - log.Errorf("curent rubik not support non-linear model") - continue - } - - if err := configQos(true, devno); err != nil { - log.Errorf(err.Error()) - continue - } - } - return nil -} - -// SetPodWeight set pod weight -func SetPodWeight(pod *typedef.PodInfo) error { - if !iocostEnable { - return errors.Errorf("iocost feature is disable") - } - weightFile := filepath.Join(pod.CgroupRoot, - blkSubName, pod.CgroupPath, iocostWeightFile) - if err := configWeight(pod.Offline, weightFile); err != nil { - return err - } - if err := bindMemcgBlkio(pod.Containers); err != nil { - return err - } - return nil -} - -// ShutDown for clear iocost if feature is enable. -func ShutDown() error { - if !iocostEnable { - return errors.Errorf("iocost feature is disable") - } - if err := clearIOcost(); err != nil { - return err - } - return nil -} - -func getBlkDeviceNo(devName string) (string, error) { - devPath := filepath.Join("/dev", devName) - fi, err := os.Stat(devPath) - if err != nil { - return "", errors.Errorf("stat %s failed with error: %v", devName, err) - } - - if fi.Mode()&os.ModeDevice == 0 { - return "", errors.Errorf("%s is not a device", devName) - } - - st, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return "", errors.Errorf("failed to get Sys(), %v has type %v", devName, st) - } - - devno := st.Rdev - major, minor := devno/devNoMax, devno%devNoMax - return fmt.Sprintf("%v:%v", major, minor), nil -} - -func configWeight(offline bool, file string) error { - var weight uint64 = offlineWeight - if !offline { - weight = onlineWeight - } - return writeIOcost(file, strconv.FormatUint(weight, scale)) -} - -func configQos(enable bool, devno string) error { - t := 0 - if enable { - t = 1 - } - qosStr := fmt.Sprintf("%v enable=%v ctrl=user min=100.00 max=100.00", devno, t) - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostQosFile) - return writeIOcost(filePath, qosStr) -} - -func configLinearModel(linearModelParam config.Param, devno string) error { - if linearModelParam.Rbps <= 0 || linearModelParam.Rseqiops <= 0 || linearModelParam.Rrandiops <= 0 || - linearModelParam.Wbps <= 0 || linearModelParam.Wseqiops <= 0 || linearModelParam.Wrandiops <= 0 { - return errors.Errorf("invalid iocost.params, the value must not 0") - } - paramStr := fmt.Sprintf("%v rbps=%v rseqiops=%v rrandiops=%v wbps=%v wseqiops=%v wrandiops=%v", - devno, - linearModelParam.Rbps, linearModelParam.Rseqiops, linearModelParam.Rrandiops, - linearModelParam.Wbps, linearModelParam.Wseqiops, linearModelParam.Wrandiops) - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostModelFile) - return writeIOcost(filePath, paramStr) -} - -func bindMemcgBlkio(containers map[string]*typedef.ContainerInfo) error { - for _, container := range containers { - memPath := container.CgroupPath(memSubName) - blkPath := container.CgroupPath(blkSubName) - ino, err := getDirInode(blkPath) - if err != nil { - log.Errorf("get director:%v, inode err:%v", blkPath, err.Error()) - continue - } - wbBlkFile := filepath.Join(memPath, wbBlkioinoFile) - if err := writeIOcost(wbBlkFile, strconv.FormatUint(ino, scale)); err != nil { - log.Errorf("write file %v err:%v", wbBlkFile, err.Error()) - continue - } - } - return nil -} - -func getDirInode(file string) (uint64, error) { - fi, err := os.Stat(file) - if err != nil { - return 0, err - } - st, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return 0, errors.Errorf("failed to get Sys(), %v has type %v", file, st) - } - return st.Ino, nil -} - -func clearIOcost() error { - qosFilePath := filepath.Join(config.CgroupRoot, blkSubName, iocostQosFile) - qosParamByte, err := ioutil.ReadFile(qosFilePath) - if err != nil { - return errors.Errorf("read file:%v failed, err:%v", qosFilePath, err.Error()) - } - - if len(qosParamByte) == 0 { - return errors.Errorf("read file:%v is empty", qosFilePath) - } - - qosParams := strings.Split(string(qosParamByte), "\n") - for _, param := range qosParams { - paramList := strings.FieldsFunc(param, unicode.IsSpace) - if len(paramList) != 0 { - if err := configQos(false, paramList[0]); err != nil { - return errors.Errorf("write file:%v failed, err:%v", qosFilePath, err.Error()) - } - } - } - return nil -} - -func writeIOcost(file, param string) error { - if len(param) > paramMaxLen { - return errors.Errorf("param size exceeds %v", paramMaxLen) - } - if !util.PathExist(file) { - return errors.Errorf("path %v not exist, maybe iocost is unsupport", file) - } - err := ioutil.WriteFile(file, []byte(param), constant.DefaultFileMode) - return err -} diff --git a/pkg/iocost/iocost_test.go b/pkg/iocost/iocost_test.go deleted file mode 100644 index 910710f..0000000 --- a/pkg/iocost/iocost_test.go +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: hanchao -// Create: 2022-10-28 -// Description: iocost test - -// Package iocost is for iocost. -package iocost - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - "unicode" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/try" - "isula.org/rubik/pkg/typedef" -) - -const paramsLen = 2 - -func TestIOcostFeatureSwitch(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - SetIOcostEnable(false) - err := ConfigIOcost(nil) - assert.Equal(t, err.Error(), "iocost feature is disable") - err = SetPodWeight(nil) - assert.Equal(t, err.Error(), "iocost feature is disable") - err = ShutDown() - assert.Equal(t, err.Error(), "iocost feature is disable") -} - -// TestIocostConfig is testing for IocostConfig interface. -func TestIocostConfig(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - SetIOcostEnable(true) - devs, err := getAllBlockDevice() - assert.NoError(t, err) - var devname, devno string - for k, v := range devs { - devname = k - devno = v - break - } - - tests := []struct { - name string - config config.IOcostConfig - qosCheck bool - modelCheck bool - qosParam string - modelParam string - }{ - { - name: "Test iocost enable", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - }, - qosCheck: true, - modelCheck: true, - qosParam: devno + " enable=1", - modelParam: devno + " ctrl=user model=linear " + - "rbps=600 rseqiops=600 rrandiops=600 " + - "wbps=600 wseqiops=600 wrandiops=600", - }, - { - name: "Test iocost disable", - config: config.IOcostConfig{ - Dev: devname, - Enable: false, - Model: "linear", - Param: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - }, - qosCheck: true, - modelCheck: false, - qosParam: devno + " enable=0", - }, - { - name: "Test iocost enable", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 500, Rseqiops: 500, Rrandiops: 500, - Wbps: 500, Wseqiops: 500, Wrandiops: 500, - }, - }, - qosCheck: true, - modelCheck: true, - qosParam: devno + " enable=1", - modelParam: devno + " ctrl=user model=linear " + - "rbps=500 rseqiops=500 rrandiops=500 " + - "wbps=500 wseqiops=500 wrandiops=500", - }, - { - name: "Test iocost no dev error", - config: config.IOcostConfig{ - Dev: "xxx", - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - }, - qosCheck: true, - modelCheck: false, - qosParam: devno + " enable=0", - }, - { - name: "Test iocost enable", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 500, Rseqiops: 500, Rrandiops: 500, - Wbps: 500, Wseqiops: 500, Wrandiops: 500, - }, - }, - qosCheck: true, - modelCheck: true, - qosParam: devno + " enable=1", - modelParam: devno + " ctrl=user model=linear " + - "rbps=500 rseqiops=500 rrandiops=500 " + - "wbps=500 wseqiops=500 wrandiops=500", - }, - { - name: "Test iocost non-linear error", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linearx", - Param: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - }, - qosCheck: true, - modelCheck: false, - qosParam: devno + " enable=0", - }, - { - name: "Test iocost enable", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 500, Rseqiops: 500, Rrandiops: 500, - Wbps: 500, Wseqiops: 500, Wrandiops: 500, - }, - }, - qosCheck: true, - modelCheck: true, - qosParam: devno + " enable=1", - modelParam: devno + " ctrl=user model=linear " + - "rbps=500 rseqiops=500 rrandiops=500 " + - "wbps=500 wseqiops=500 wrandiops=500", - }, - { - name: "Test iocost param error", - config: config.IOcostConfig{ - Dev: devname, - Enable: true, - Model: "linear", - Param: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 0, - }, - }, - qosCheck: true, - modelCheck: false, - qosParam: devno + " enable=0", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - params := []config.IOcostConfig{ - tt.config, - } - err := ConfigIOcost(params) - assert.NoError(t, err) - if tt.qosCheck { - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostQosFile) - qosParamByte, err := ioutil.ReadFile(filePath) - assert.NoError(t, err) - qosParams := strings.Split(string(qosParamByte), "\n") - for _, qosParam := range qosParams { - paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) - if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { - assert.Equal(t, tt.qosParam, qosParam[:len(tt.qosParam)]) - break - } - } - } - if tt.modelCheck { - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostModelFile) - modelParamByte, err := ioutil.ReadFile(filePath) - assert.NoError(t, err) - modelParams := strings.Split(string(modelParamByte), "\n") - for _, modelParam := range modelParams { - paramList := strings.FieldsFunc(modelParam, unicode.IsSpace) - if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { - assert.Equal(t, tt.modelParam, modelParam[:len(tt.modelParam)]) - break - } - } - } - }) - } -} - -// TestSetPodWeight is testing for SetPodWeight interface. -func TestSetPodWeight(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - // deploy enviroment - const testCgroupPath = "/rubik-test" - rubikBlkioTestPath := filepath.Join(config.CgroupRoot, blkSubName, testCgroupPath) - rubikMemTestPath := filepath.Join(config.CgroupRoot, memSubName, testCgroupPath) - try.MkdirAll(rubikBlkioTestPath, constant.DefaultDirMode) - try.MkdirAll(rubikMemTestPath, constant.DefaultDirMode) - defer try.RemoveAll(rubikBlkioTestPath) - defer try.RemoveAll(rubikMemTestPath) - SetIOcostEnable(true) - - tests := []struct { - name string - pod *typedef.PodInfo - wantErr bool - want string - }{ - { - name: "Test online qos level", - pod: &typedef.PodInfo{ - CgroupRoot: config.CgroupRoot, - CgroupPath: testCgroupPath, - Offline: false, - }, - wantErr: false, - want: "default 1000\n", - }, - { - name: "Test offline qos level", - pod: &typedef.PodInfo{ - CgroupRoot: config.CgroupRoot, - CgroupPath: testCgroupPath, - Offline: true, - }, - wantErr: false, - want: "default 10\n", - }, - { - name: "Test error cgroup path", - pod: &typedef.PodInfo{ - CgroupRoot: config.CgroupRoot, - CgroupPath: "var/log/rubik/rubik-test", - Offline: true, - }, - wantErr: true, - want: "default 10\n", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := SetPodWeight(tt.pod) - if tt.wantErr { - assert.Equal(t, err != nil, true) - return - } - assert.NoError(t, err) - weightFile := filepath.Join(rubikBlkioTestPath, "blkio.cost.weight") - weightOnline, err := ioutil.ReadFile(weightFile) - assert.NoError(t, err) - assert.Equal(t, string(weightOnline), tt.want) - }) - } -} - -// TestBindMemcgBlkio is testing for bindMemcgBlkio -func TestBindMemcgBlkio(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - // deploy enviroment - const testCgroupPath = "rubik-test" - rubikBlkioTestPath := filepath.Join(config.CgroupRoot, blkSubName, testCgroupPath) - rubikMemTestPath := filepath.Join(config.CgroupRoot, memSubName, testCgroupPath) - try.MkdirAll(rubikBlkioTestPath, constant.DefaultDirMode) - try.MkdirAll(rubikMemTestPath, constant.DefaultDirMode) - defer try.RemoveAll(rubikBlkioTestPath) - defer try.RemoveAll(rubikMemTestPath) - SetIOcostEnable(true) - - containers := make(map[string]*typedef.ContainerInfo, 5) - for i := 0; i < 5; i++ { - dirName := "container" + strconv.Itoa(i) - blkContainer := filepath.Join(rubikBlkioTestPath, dirName) - memContainer := filepath.Join(rubikMemTestPath, dirName) - try.MkdirAll(blkContainer, constant.DefaultDirMode) - try.MkdirAll(memContainer, constant.DefaultDirMode) - containers[dirName] = &typedef.ContainerInfo{ - Name: dirName, - CgroupRoot: config.CgroupRoot, - CgroupAddr: filepath.Join(testCgroupPath, dirName), - } - } - err := bindMemcgBlkio(containers) - assert.NoError(t, err) - - for key, container := range containers { - memcgPath := container.CgroupPath(memSubName) - blkcgPath := container.CgroupPath(blkSubName) - ino, err := getDirInode(blkcgPath) - assert.NoError(t, err) - wbBlkioInfo := filepath.Join(memcgPath, wbBlkioinoFile) - blkioInoStr, err := ioutil.ReadFile(wbBlkioInfo) - assert.NoError(t, err) - result := fmt.Sprintf("wb_blkio_path:/%v/%v\nwb_blkio_ino:%v\n", testCgroupPath, key, ino) - assert.Equal(t, result, string(blkioInoStr)) - } -} - -// TestClearIOcost is testing for ClearIOcost interface. -func TestClearIOcost(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - devs, err := getAllBlockDevice() - assert.NoError(t, err) - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostQosFile) - for _, devno := range devs { - qosStr := fmt.Sprintf("%v enable=1", devno) - err := writeIOcost(filePath, qosStr) - assert.NoError(t, err) - } - err = ShutDown() - assert.NoError(t, err) - qosParamByte, err := ioutil.ReadFile(filePath) - assert.NoError(t, err) - qosParams := strings.Split(string(qosParamByte), "\n") - for _, qosParam := range qosParams { - paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) - if len(paramList) >= paramsLen { - assert.Equal(t, paramList[1], "enable=0") - } - } -} - -// TestGetBlkDevice is testing for get block device interface. -func TestGetBlkDevice(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - devs, err := getAllBlockDevice() - assert.NoError(t, err) - for index, dev := range devs { - devno, err := getBlkDeviceNo(index) - assert.NoError(t, err) - assert.Equal(t, dev, devno) - } - _, err = getBlkDeviceNo("") - assert.Equal(t, err != nil, true) - _, err = getBlkDeviceNo("xxx") - assert.Equal(t, err != nil, true) -} - -// TestConfigQos is testing for ConfigQos interface. -func TestConfigQos(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - SetIOcostEnable(true) - devs, err1 := getAllBlockDevice() - assert.NoError(t, err1) - var devno string - for _, v := range devs { - devno = v - break - } - - tests := []struct { - name string - enable bool - want string - }{ - { - name: "Test qos disable", - enable: false, - want: devno + " enable=0", - }, - { - name: "Test qos enable", - enable: true, - want: devno + " enable=1", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := configQos(tt.enable, devno) - assert.NoError(t, err) - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostQosFile) - qosParamByte, err := ioutil.ReadFile(filePath) - assert.NoError(t, err) - qosParams := strings.Split(string(qosParamByte), "\n") - for _, qosParam := range qosParams { - paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) - if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { - assert.Equal(t, tt.want, qosParam[:len(tt.want)]) - break - } - } - }) - } -} - -// TestConfigLinearModel is testing for ConfigLinearModel interface. -func TestConfigLinearModel(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on support iocost machine", t.Name()) - } - - SetIOcostEnable(true) - devs, err := getAllBlockDevice() - assert.NoError(t, err) - var devno string - for _, v := range devs { - devno = v - break - } - - tests := []struct { - name string - linearModel config.Param - wantErr bool - modelParam string - }{ - { - name: "Test linear model", - linearModel: config.Param{ - Rbps: 500, Rseqiops: 500, Rrandiops: 500, - Wbps: 500, Wseqiops: 500, Wrandiops: 500, - }, - wantErr: false, - modelParam: devno + " ctrl=user model=linear " + - "rbps=500 rseqiops=500 rrandiops=500 " + - "wbps=500 wseqiops=500 wrandiops=500", - }, - { - name: "Test linear model", - linearModel: config.Param{ - Rbps: 600, Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - wantErr: false, - modelParam: devno + " ctrl=user model=linear " + - "rbps=600 rseqiops=600 rrandiops=600 " + - "wbps=600 wseqiops=600 wrandiops=600", - }, - { - name: "Test missing parameter", - linearModel: config.Param{ - Rseqiops: 600, Rrandiops: 600, - Wbps: 600, Wseqiops: 600, Wrandiops: 600, - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := configLinearModel(tt.linearModel, devno) - if tt.wantErr { - assert.Equal(t, err != nil, true) - return - } - assert.NoError(t, err) - filePath := filepath.Join(config.CgroupRoot, blkSubName, iocostModelFile) - modelParamByte, err := ioutil.ReadFile(filePath) - assert.NoError(t, err) - - modelParams := strings.Split(string(modelParamByte), "\n") - for _, modelParam := range modelParams { - paramList := strings.FieldsFunc(modelParam, unicode.IsSpace) - if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { - assert.Equal(t, tt.modelParam, modelParam[:len(tt.modelParam)]) - break - } - } - }) - } -} - -func TestPartFunction(t *testing.T) { - const testCgroup = "/var/rubikcgroup/" - qosParam := strings.Repeat("a", paramMaxLen+1) - err := writeIOcost(testCgroup, qosParam) - assert.Equal(t, err.Error(), "param size exceeds "+strconv.Itoa(paramMaxLen)) - - _, err = getDirInode(testCgroup) - assert.Equal(t, true, err != nil) -} - -func getAllBlockDevice() (map[string]string, error) { - files, err := ioutil.ReadDir(sysDevBlock) - if err != nil { - log.Infof("read dir %v failed, err:%v", sysDevBlock, err.Error()) - return nil, err - } - devs := make(map[string]string) - for _, f := range files { - if f.Mode()&os.ModeSymlink != 0 { - fullName := filepath.Join(sysDevBlock, f.Name()) - realPath, err := os.Readlink(fullName) - if err != nil { - continue - } - vs := strings.Split(realPath, "/") - const penultimate = 2 - if len(vs) > penultimate && strings.Compare(vs[len(vs)-penultimate], "block") == 0 { - const excludeBlock = "dm-" - dmLen := len(excludeBlock) - if len(vs[len(vs)-1]) > dmLen && strings.Compare(vs[len(vs)-1][:dmLen], excludeBlock) == 0 { - continue - } - devs[vs[len(vs)-1]] = f.Name() - } - } - } - return devs, nil -} diff --git a/pkg/memory/dynlevel.go b/pkg/memory/dynlevel.go deleted file mode 100644 index 3c78350..0000000 --- a/pkg/memory/dynlevel.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Yang Feiyu -// Create: 2022-6-7 -// Description: memory setting for pods - -package memory - -import ( - "io/ioutil" - "path/filepath" - "time" - - "k8s.io/apimachinery/pkg/util/wait" - - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - -type memoryInfo struct { - total int64 - free int64 - available int64 -} - -type dynLevel struct { - m *MemoryManager - memInfo memoryInfo - st status -} - -func newDynLevel(m *MemoryManager) (f *dynLevel) { - return &dynLevel{ - st: newStatus(), - m: m, - } -} - -func (f *dynLevel) Run() { - go wait.Until(f.timerProc, time.Duration(f.m.checkInterval)*time.Second, f.m.stop) -} - -// UpdateConfig is used to update memory config -func (f *dynLevel) UpdateConfig(pod *typedef.PodInfo) { - // there is no config need to update -} - -func (f *dynLevel) timerProc() { - f.updateStatus() - log.Logf("memory manager updates status with memory free: %v, memory total: %v", f.memInfo.free, f.memInfo.total) - f.reclaim() - log.Logf("memory manager reclaims done and pressure level is %s", &f.st) -} - -func (f *dynLevel) updateStatus() { - memInfo, err := getMemoryInfo() - if err != nil { - log.Errorf("getMemoryInfo failed with error: %v, it should not happen", err) - return - } - f.memInfo = memInfo - f.st.transitionStatus(float64(memInfo.free) / float64(memInfo.total)) -} - -func (f *dynLevel) limitOfflineContainers(ft fileType) { - containers := f.m.cpm.ListOfflineContainers() - for _, c := range containers { - if err := f.limitContainer(c, ft); err != nil { - log.Errorf("limit memory for container: %v failed, filetype: %v, err: %v", c.ID, ft, err) - } - } -} - -func (f *dynLevel) limitContainer(c *typedef.ContainerInfo, ft fileType) error { - path := c.CgroupPath("memory") - limit, err := readMemoryFile(filepath.Join(path, memoryUsageFile)) - if err != nil { - return err - } - - for i := 0; i < maxRetry; i++ { - limit += int64(float64(f.memInfo.free) * extraFreePercentage) - if err = writeMemoryLimit(path, typedef.FormatInt64(limit), ft); err == nil { - break - } - log.Errorf("failed to write memory limit from path: %v, will retry now, retry num: %v", path, i) - } - - return err -} - -// dropCaches will echo 3 > /proc/sys/vm/drop_caches -func (f *dynLevel) dropCaches() { - var err error - for i := 0; i < maxRetry; i++ { - if err = ioutil.WriteFile(dropCachesFilePath, []byte("3"), constant.DefaultFileMode); err == nil { - log.Logf("drop caches success") - return - } - log.Errorf("drop caches failed, error: %v, will retry later, retry num: %v", err, i) - } -} - -func (f *dynLevel) forceEmptyOfflineContainers() { - containers := f.m.cpm.ListOfflineContainers() - for _, c := range containers { - if err := writeForceEmpty(c.CgroupPath("memory")); err != nil { - log.Errorf("force empty for container: %v failed, err: %v", c.ID, err) - } - - } -} - -func (f *dynLevel) reclaimInPressure() { - switch f.st.pressureLevel { - case low: - // do soft limit - f.limitOfflineContainers(msoftLimit) - case mid: - f.forceEmptyOfflineContainers() - case high: - // do hard limit - f.limitOfflineContainers(mlimit) - case critical: - // drop caches and do hard limit - f.dropCaches() - f.limitOfflineContainers(mlimit) - } -} - -func (f *dynLevel) reclaimInRelieve() { - f.st.relieveCnt++ - containers := f.m.cpm.ListOfflineContainers() - for _, c := range containers { - recoverContainerMemoryLimit(c, f.st.relieveCnt == relieveMaxCnt) - } -} - -func (f *dynLevel) reclaim() { - if f.st.isNormal() { - return - } - - if f.st.isRelieve() { - f.reclaimInRelieve() - return - } - - f.reclaimInPressure() -} - -func writeForceEmpty(cgroupPath string) error { - var err error - for i := 0; i < maxRetry; i++ { - if err = writeMemoryFile(cgroupPath, memoryForceEmptyFile, "0"); err == nil { - log.Logf("force cgroup memory %v empty success", cgroupPath) - return nil - } - log.Errorf("force clean memory failed for %s: %v, will retry later, retry num: %v", cgroupPath, err, i) - } - - return err -} - -func recoverContainerMemoryLimit(c *typedef.ContainerInfo, reachMax bool) { - // ratio 0.1 means, newLimit = oldLimit * 1.1 - const ratio = 0.1 - var memLimit int64 - path := c.CgroupPath("memory") - if reachMax { - memLimit = maxSysMemLimit - if err := writeMemoryLimit(path, typedef.FormatInt64(memLimit), mlimit); err != nil { - log.Errorf("failed to write memory limit from path:%v container:%v", path, c.ID) - } - - if err := writeMemoryLimit(path, typedef.FormatInt64(memLimit), msoftLimit); err != nil { - log.Errorf("failed to write memory soft limit from path:%v container:%v", path, c.ID) - } - return - } - - memLimit, err := readMemoryFile(filepath.Join(path, memoryLimitFile)) - if err != nil { - log.Errorf("failed to read from path:%v container:%v", path, c.ID) - return - } - - memLimit = int64(float64(memLimit) * (1 + ratio)) - if memLimit < 0 { - // it means the limit value has reached max, just return - return - } - - if err := writeMemoryLimit(path, typedef.FormatInt64(memLimit), mlimit); err != nil { - log.Errorf("failed to write memory limit from path:%v container:%v", path, c.ID) - } -} diff --git a/pkg/memory/fssr.go b/pkg/memory/fssr.go deleted file mode 100644 index b024d34..0000000 --- a/pkg/memory/fssr.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: hanchao -// Create: 2022-9-2 -// Description: -// 1. When Rubik starts, all offline memory.high is configured to 80% of total memory by default. -// 2. When memory pressure increases: Available memory freeMemory < reservedMemory(totalMemory * 5%). -// newly memory.high=memory.high-totalMemory * 10%. -// 3. When memory is rich over a period of time: freeMemory > 3 * reservedMemory, In this case, 1% of -// the totalMemory is reserved for offline applications. High=memory.high+totalMemory * 1% until memory -// free is between reservedMemory and 3 * reservedMemory. - -// Package memory provide memory reclaim strategy for offline tasks. -package memory - -import ( - "time" - - "k8s.io/apimachinery/pkg/util/wait" - - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - -type fssrStatus int - -const ( - reservePercentage = 0.05 - waterlinePercentage = 0.8 - relievePercentage = 0.02 - reclaimPercentage = 0.1 - prerelieveInterval = "30m" - reserveRatio = 3 - highAsyncRatio = 90 -) - -const ( - fssrNormal fssrStatus = iota - fssrReclaim - fssrPreRelieve - fssrRelieve -) - -type fssr struct { - mmgr *MemoryManager - preRelieveStartDate time.Time - st fssrStatus - total int64 - limit int64 - reservedMemory int64 - highAsyncRatio int64 -} - -func newFssr(m *MemoryManager) (f *fssr) { - f = new(fssr) - f.init(m) - return f -} - -func (f *fssr) init(m *MemoryManager) { - memInfo, err := getMemoryInfo() - if err != nil { - log.Infof("initialization of fssr failed") - return - } - - f.mmgr = m - f.total = memInfo.total - f.reservedMemory = int64(reservePercentage * float64(f.total)) - f.limit = int64(waterlinePercentage * float64(f.total)) - f.st = fssrNormal - f.highAsyncRatio = highAsyncRatio - f.initOfflineContainerLimit() - - log.Infof("total: %v, reserved Memory: %v, limit memory: %v", f.total, f.reservedMemory, f.limit) -} - -func (f *fssr) Run() { - go wait.Until(f.timerProc, time.Duration(f.mmgr.checkInterval)*time.Second, f.mmgr.stop) -} - -// UpdateConfig is used to update memory config -func (f *fssr) UpdateConfig(pod *typedef.PodInfo) { - for _, c := range pod.Containers { - f.initContainerMemoryLimit(c) - } -} - -func (f *fssr) timerProc() { - f.updateStatus() - if f.needAdjust() { - newLimit := f.calculateNewLimit() - f.adjustOfflineContainerMemory(newLimit) - } -} - -func (f *fssr) initOfflineContainerLimit() { - if f.mmgr.cpm == nil { - log.Infof("init offline container limit failed, cpm is nil") - return - } - - containers := f.mmgr.cpm.ListOfflineContainers() - for _, c := range containers { - f.initContainerMemoryLimit(c) - } -} - -func (f *fssr) needAdjust() bool { - if f.st == fssrReclaim || f.st == fssrRelieve { - return true - } - return false -} - -func (f *fssr) updateStatus() { - curMemInfo, err := getMemoryInfo() - if err != nil { - log.Errorf("get memory info failed, err:%v", err) - return - } - oldStatus := f.st - - // Use free instead of Available - if curMemInfo.free < f.reservedMemory { - f.st = fssrReclaim - } else if curMemInfo.free > reserveRatio*f.reservedMemory { - switch f.st { - case fssrNormal: - f.st = fssrPreRelieve - f.preRelieveStartDate = time.Now() - case fssrPreRelieve, fssrRelieve: - t, _ := time.ParseDuration(prerelieveInterval) - if f.preRelieveStartDate.Add(t).Before(time.Now()) { - f.st = fssrRelieve - } - case fssrReclaim: - f.st = fssrNormal - default: - log.Errorf("status incorrect, this should not happen") - } - } - - log.Infof("update change status from %v to %v, cur available %v, cur free %v", - oldStatus, f.st, curMemInfo.available, curMemInfo.free) -} - -func (f *fssr) calculateNewLimit() int64 { - newLimit := f.limit - if f.st == fssrReclaim { - newLimit = f.limit - int64(reclaimPercentage*float64(f.total)) - if newLimit < 0 || newLimit <= f.reservedMemory { - newLimit = f.reservedMemory - log.Infof("reclaim offline containers current limit %v is too small, set as reserved memory %v", newLimit, f.reservedMemory) - } - } else if f.st == fssrRelieve { - newLimit = f.limit + int64(relievePercentage*float64(f.total)) - if newLimit > int64(waterlinePercentage*float64(f.total)) { - newLimit = int64(waterlinePercentage * float64(f.total)) - log.Infof("relieve offline containers limit soft memory exceeds waterline, set limit as waterline %v", waterlinePercentage*float64(f.total)) - } - } - return newLimit -} - -func (f *fssr) initContainerMemoryLimit(c *typedef.ContainerInfo) { - path := c.CgroupPath("memory") - if err := writeMemoryLimit(path, typedef.FormatInt64(f.limit), mhigh); err != nil { - log.Errorf("failed to initialize the limit soft memory of offline container %v: %v", c.ID, err) - } else { - log.Infof("initialize the limit soft memory of the offline container %v to %v successfully", c.ID, f.limit) - } - - if err := writeMemoryLimit(path, typedef.FormatInt64(f.highAsyncRatio), mhighAsyncRatio); err != nil { - log.Errorf("failed to initialize the async high ration of offline container %v: %v", c.ID, err) - } else { - log.Infof("initialize the async high ration of the offline container %v:%v success", c.ID, f.highAsyncRatio) - } -} - -func (f *fssr) adjustOfflineContainerMemory(limit int64) { - if f.mmgr.cpm == nil { - log.Infof("reclaim offline containers failed, cpm is nil") - return - } - - containers := f.mmgr.cpm.ListOfflineContainers() - for _, c := range containers { - path := c.CgroupPath("memory") - if err := writeMemoryLimit(path, typedef.FormatInt64(limit), mhigh); err != nil { - log.Errorf("relieve offline containers limit soft memory %v failed, err is %v", c.ID, err) - } else { - f.limit = limit - } - } -} diff --git a/pkg/memory/memory.go b/pkg/memory/memory.go deleted file mode 100644 index f25bd34..0000000 --- a/pkg/memory/memory.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Song Yanting -// Create: 2022-06-10 -// Description: memory setting for pods - -package memory - -import ( - "bufio" - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - securejoin "github.com/cyphar/filepath-securejoin" - "github.com/pkg/errors" - - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - -const ( - mlimit fileType = iota - msoftLimit - mhigh - mhighAsyncRatio -) - -const ( - dropCachesFilePath = "/proc/sys/vm/drop_caches" - memoryLimitFile = "memory.limit_in_bytes" - memorySoftLimitFile = "memory.soft_limit_in_bytes" - memoryHighFile = "memory.high" - memoryHighAsyncRatioFile = "memory.high_async_ratio" - memoryUsageFile = "memory.usage_in_bytes" - memoryForceEmptyFile = "memory.force_empty" - // maxSysMemLimit 9223372036854771712 is the default cgroup memory limit value - maxSysMemLimit = 9223372036854771712 - maxRetry = 3 - relieveMaxCnt = 5 - extraFreePercentage = 0.02 -) - -type fileType int - -type memDriver interface { - Run() - UpdateConfig(pod *typedef.PodInfo) -} - -// MemoryManager manages memory reclaim works. -type MemoryManager struct { - cpm *checkpoint.Manager - md memDriver - checkInterval int - stop chan struct{} -} - -// NewMemoryManager creates a new memory manager -func NewMemoryManager(cpm *checkpoint.Manager, memConfig config.MemoryConfig) (*MemoryManager, error) { - interval := memConfig.CheckInterval - if err := validateInterval(interval); err != nil { - return nil, err - } - log.Logf("new memory manager with interval:%d", interval) - mm := MemoryManager{ - cpm: cpm, - checkInterval: interval, - stop: config.ShutdownChan, - } - switch memConfig.Strategy { - case "fssr": - mm.md = newFssr(&mm) - case "dynlevel": - mm.md = newDynLevel(&mm) - case "none": - log.Infof("strategy is set to none") - return nil, nil - default: - return nil, errors.Errorf("unsupported memStrategy, expect dynlevel|fssr|none") - } - return &mm, nil -} - -func validateInterval(interval int) error { - if interval > 0 && interval <= constant.DefaultMaxMemCheckInterval { - return nil - } - return errors.Errorf("check interval should between 0 and %v", constant.DefaultMemCheckInterval) -} - -// Run wait every interval and execute run -func (m *MemoryManager) Run() { - m.md.Run() -} - -// UpdateConfig is used to update memory config -func (m *MemoryManager) UpdateConfig(pod *typedef.PodInfo) { - m.md.UpdateConfig(pod) -} - -func writeMemoryLimit(cgroupPath string, value string, ft fileType) error { - var filename string - switch ft { - case mlimit: - filename = memoryLimitFile - case msoftLimit: - filename = memorySoftLimitFile - case mhigh: - filename = memoryHighFile - case mhighAsyncRatio: - filename = memoryHighAsyncRatioFile - default: - return errors.Errorf("unsupported file type %v", ft) - } - - if err := writeMemoryFile(cgroupPath, filename, value); err != nil { - return errors.Errorf("set memory file:%s/%s=%s failed, err:%v", cgroupPath, filename, value, err) - } - - return nil -} - -func writeMemoryFile(cgroupPath, filename, value string) error { - cgFilePath, err := securejoin.SecureJoin(cgroupPath, filename) - if err != nil { - return errors.Errorf("join path failed for %s and %s: %v", cgroupPath, filename, err) - } - - return ioutil.WriteFile(cgFilePath, []byte(value), constant.DefaultFileMode) -} - -func readMemoryFile(path string) (int64, error) { - const ( - base, width = 10, 64 - ) - content, err := ioutil.ReadFile(filepath.Clean(path)) - if err != nil { - return 0, err - } - - memBytes := strings.Split(string(content), "\n")[0] - return strconv.ParseInt(memBytes, base, width) -} - -// getMemoryInfo returns memory info -func getMemoryInfo() (memoryInfo, error) { - var m memoryInfo - var total, free, available int64 - const memInfoFile = "/proc/meminfo" - - f, err := os.Open(memInfoFile) - if err != nil { - return m, err - } - - defer f.Close() - - // MemTotal: 15896176 kB - // MemFree: 3811032 kB - scan := bufio.NewScanner(f) - for scan.Scan() { - if bytes.HasPrefix(scan.Bytes(), []byte("MemTotal:")) { - if _, err := fmt.Sscanf(scan.Text(), "MemTotal:%d", &total); err != nil { - return m, err - } - } - - if bytes.HasPrefix(scan.Bytes(), []byte("MemFree:")) { - if _, err := fmt.Sscanf(scan.Text(), "MemFree:%d", &free); err != nil { - return m, err - } - } - - if bytes.HasPrefix(scan.Bytes(), []byte("MemAvailable:")) { - if _, err := fmt.Sscanf(scan.Text(), "MemAvailable:%d", &available); err != nil { - return m, err - } - } - } - - if total == 0 || free == 0 || available == 0 { - return m, errors.Errorf("Memory value should be larger than 0, MemTotal:%d, MemFree:%d, MemAvailable:%d", total, free, available) - } - - m.free = free * 1024 - m.total = total * 1024 - m.available = available * 1024 - - return m, nil -} diff --git a/pkg/memory/status.go b/pkg/memory/status.go deleted file mode 100644 index c3be203..0000000 --- a/pkg/memory/status.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Yang Feiyu -// Create: 2022-6-7 -// Description: memory status functions - -package memory - -import log "isula.org/rubik/pkg/tinylog" - -const ( - // lowPressure means free / total < 30% - lowPressure = 0.3 - midPressure = 0.15 - highPressure = 0.1 - criticalPressure = 0.05 -) - -type levelInt int - -const ( - normal levelInt = iota - relieve - low - mid - high - critical -) - -type status struct { - pressureLevel levelInt - relieveCnt int -} - -func newStatus() status { - return status{ - pressureLevel: normal, - } -} - -func (s *status) set(pressureLevel levelInt) { - s.pressureLevel = pressureLevel - s.relieveCnt = 0 -} - -func (s *status) isNormal() bool { - return s.pressureLevel == normal -} - -func (s *status) isRelieve() bool { - return s.pressureLevel == relieve -} - -func (s *status) transitionStatus(freePercentage float64) { - if freePercentage > lowPressure { - switch s.pressureLevel { - case normal: - case low, mid, high, critical: - log.Logf("change status from pressure to relieve") - s.set(relieve) - case relieve: - if s.relieveCnt == relieveMaxCnt { - s.set(normal) - log.Logf("change status from relieve to normal") - } - } - return - } - s.pressureLevel = getLevelInPressure(freePercentage) -} - -func (s *status) String() string { - switch s.pressureLevel { - case normal: - return "normal" - case relieve: - return "relieve" - case low: - return "low" - case mid: - return "mid" - case high: - return "high" - case critical: - return "critical" - default: - return "unknown" - } -} - -func getLevelInPressure(freePercentage float64) levelInt { - var pressureLevel levelInt - if freePercentage <= criticalPressure { - pressureLevel = critical - } else if freePercentage <= highPressure { - pressureLevel = high - } else if freePercentage <= midPressure { - pressureLevel = mid - } else { - pressureLevel = low - } - return pressureLevel -} diff --git a/pkg/memory/status_test.go b/pkg/memory/status_test.go deleted file mode 100644 index 080307b..0000000 --- a/pkg/memory/status_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Yang Feiyu -// Create: 2022-6-7 -// Description: tests for memory status functions - -package memory - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSetStatus(t *testing.T) { - s := newStatus() - s.pressureLevel = relieve - s.set(normal) - assert.Equal(t, s.pressureLevel, normal) - assert.Equal(t, s.relieveCnt, 0) -} - -func TestIsNormal(t *testing.T) { - s := newStatus() - s.pressureLevel = relieve - assert.Equal(t, s.isNormal(), false) - s.set(normal) - assert.Equal(t, s.isNormal(), true) - assert.Equal(t, s.relieveCnt, 0) -} - -func TestIsRelieve(t *testing.T) { - s := newStatus() - s.pressureLevel = relieve - assert.Equal(t, s.isRelieve(), true) -} - -func TestGetLevelInPressure(t *testing.T) { - tests := []struct { - freePercentage float64 - level levelInt - }{ - { - freePercentage: 0.04, - level: critical, - }, - { - freePercentage: 0.09, - level: high, - }, - { - freePercentage: 0.13, - level: mid, - }, - { - freePercentage: 0.25, - level: low, - }, - } - - for _, tt := range tests { - tmp := getLevelInPressure(tt.freePercentage) - assert.Equal(t, tmp, tt.level) - } -} - -func TestTransitionStatus(t *testing.T) { - s := newStatus() - s.transitionStatus(0.04) - assert.Equal(t, s.pressureLevel, critical) - - s.transitionStatus(0.6) - assert.Equal(t, s.pressureLevel, relieve) - s.relieveCnt = relieveMaxCnt - - s.transitionStatus(0.6) - assert.Equal(t, s.pressureLevel, normal) -} diff --git a/pkg/perf/perf.go b/pkg/perf/perf.go deleted file mode 100644 index a79aa3c..0000000 --- a/pkg/perf/perf.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: JingRui -// Create: 2022-01-26 -// Description: cgroup perf stats - -// Package perf provide perf functions -package perf - -import ( - "encoding/binary" - "path/filepath" - "runtime" - "time" - "unsafe" - - "github.com/pkg/errors" - "golang.org/x/sys/unix" - - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" -) - -var ( - hwSupport = false -) - -// HwSupport tell if the os support perf hw pmu events. -func HwSupport() bool { - return hwSupport -} - -// PerfStat is perf stat info -type PerfStat struct { - Instructions uint64 - CpuCycles uint64 - CacheMisses uint64 - CacheReferences uint64 - LLCAccess uint64 - LLCMiss uint64 -} - -type cgEvent struct { - cgfd int - cpu int - fds map[string]int - leader int -} - -type eventConfig struct { - config uint64 - eType uint32 - eventName string -} - -func getEventConfig() []eventConfig { - return []eventConfig{ - { - eType: unix.PERF_TYPE_HARDWARE, - config: unix.PERF_COUNT_HW_INSTRUCTIONS, - eventName: "instructions", - }, - { - eType: unix.PERF_TYPE_HARDWARE, - config: unix.PERF_COUNT_HW_CPU_CYCLES, - eventName: "cycles", - }, - { - eType: unix.PERF_TYPE_HARDWARE, - config: unix.PERF_COUNT_HW_CACHE_REFERENCES, - eventName: "cachereferences", - }, - { - eType: unix.PERF_TYPE_HARDWARE, - config: unix.PERF_COUNT_HW_CACHE_MISSES, - eventName: "cachemiss", - }, - { - eType: unix.PERF_TYPE_HW_CACHE, - config: unix.PERF_COUNT_HW_CACHE_LL | unix.PERF_COUNT_HW_CACHE_OP_READ<<8 | unix.PERF_COUNT_HW_CACHE_RESULT_MISS<<16, - eventName: "llcmiss", - }, - { - eType: unix.PERF_TYPE_HW_CACHE, - config: unix.PERF_COUNT_HW_CACHE_LL | unix.PERF_COUNT_HW_CACHE_OP_READ<<8 | unix.PERF_COUNT_HW_CACHE_RESULT_ACCESS<<16, - eventName: "llcaccess", - }, - } -} - -func newEvent(cgfd, cpu int) (*cgEvent, error) { - e := cgEvent{ - cgfd: cgfd, - cpu: cpu, - fds: make(map[string]int), - leader: -1, - } - - for _, ec := range getEventConfig() { - if err := e.openHardware(ec); err != nil { - e.destroy() - return nil, err - } - } - - return &e, nil -} - -func (e *cgEvent) openHardware(ec eventConfig) error { - attr := unix.PerfEventAttr{ - Type: ec.eType, - Config: ec.config, - } - - fd, err := unix.PerfEventOpen(&attr, e.cgfd, e.cpu, e.leader, unix.PERF_FLAG_PID_CGROUP|unix.PERF_FLAG_FD_CLOEXEC) - if err != nil { - return errors.Errorf("perf open for event:%s cpu:%d failed: %v", ec.eventName, e.cpu, err) - } - - if e.leader == -1 { - e.leader = fd - } - e.fds[ec.eventName] = fd - return nil -} - -func (e *cgEvent) start() error { - if err := unix.IoctlSetInt(e.leader, unix.PERF_EVENT_IOC_RESET, 0); err != nil { - return err - } - if err := unix.IoctlSetInt(e.leader, unix.PERF_EVENT_IOC_ENABLE, 0); err != nil { - return err - } - return nil -} - -func (e *cgEvent) stop() error { - if err := unix.IoctlSetInt(e.leader, unix.PERF_EVENT_IOC_DISABLE, 0); err != nil { - return err - } - return nil -} - -func (e *cgEvent) read(eventName string) uint64 { - var val uint64 - - p := make([]byte, 64) - num, err := unix.Read(e.fds[eventName], p) - if err != nil { - log.Errorf("read perf data failed %v", err) - return 0 - } - - if num != int(unsafe.Sizeof(val)) { - log.Errorf("invalid perf data length %d", num) - return 0 - } - - return binary.LittleEndian.Uint64(p) -} - -func (e *cgEvent) destroy() { - for _, fd := range e.fds { - unix.Close(fd) - } -} - -// perf is perf manager -type perf struct { - Events []*cgEvent - Cgfd int -} - -// newPerf create the perf manager -func newPerf(cgpath string) (*perf, error) { - cgfd, err := unix.Open(cgpath, unix.O_RDONLY, 0) - if err != nil { - return nil, err - } - - p := perf{ - Cgfd: cgfd, - Events: make([]*cgEvent, 0), - } - - for cpu := 0; cpu < runtime.NumCPU(); cpu++ { - e, err := newEvent(cgfd, cpu) - if err != nil { - continue - } - p.Events = append(p.Events, e) - } - - if len(p.Events) == 0 { - return nil, errors.New("new perf event for all cpus failed") - } - - if len(p.Events) != runtime.NumCPU() { - log.Errorf("new perf event for part of cpus failed") - } - - return &p, nil -} - -// Start start perf -func (p *perf) Start() error { - for _, e := range p.Events { - if err := e.start(); err != nil { - return err - } - } - return nil -} - -// Stop stop perf -func (p *perf) Stop() error { - for _, e := range p.Events { - if err := e.stop(); err != nil { - return err - } - } - return nil -} - -// Read read perf result -func (p *perf) Read() PerfStat { - var stat PerfStat - for _, e := range p.Events { - stat.Instructions += e.read("instructions") - stat.CpuCycles += e.read("cycles") - stat.CacheReferences += e.read("cachereferences") - stat.CacheMisses += e.read("cachemiss") - stat.LLCMiss += e.read("llcmiss") - stat.LLCAccess += e.read("llcaccess") - } - return stat -} - -// Destroy free resources -func (p *perf) Destroy() { - for _, e := range p.Events { - e.destroy() - } - unix.Close(p.Cgfd) -} - -// CgroupStat report perf stat for cgroup -func CgroupStat(cgpath string, dur time.Duration) (*PerfStat, error) { - p, err := newPerf(cgpath) - defer func() { - if p != nil { - p.Destroy() - } - }() - - if err != nil { - return nil, errors.Errorf("perf init failed: %v", err) - } - - if err := p.Start(); err != nil { - return nil, errors.Errorf("perf start failed: %v", err) - } - time.Sleep(dur) - if err := p.Stop(); err != nil { - return nil, errors.Errorf("perf stop failed: %v", err) - } - - stat := p.Read() - return &stat, nil -} - -func init() { - _, err := CgroupStat(filepath.Join(config.CgroupRoot, "perf_event", constant.KubepodsCgroup), time.Millisecond) - if err == nil { - hwSupport = true - } -} diff --git a/pkg/perf/perf_test.go b/pkg/perf/perf_test.go deleted file mode 100644 index 25b1932..0000000 --- a/pkg/perf/perf_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2022-02-09 -// Description: cgroup perf stats testcase - -// Package perf provide perf functions -package perf - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" -) - -// TestCgroupStat testcase -func TestCgroupStat(t *testing.T) { - if !HwSupport() { - t.Skipf("%s only run on physical machine", t.Name()) - } - testCGRoot := filepath.Join(config.CgroupRoot, "perf_event", t.Name()) - assert.NoError(t, os.MkdirAll(testCGRoot, constant.DefaultDirMode)) - ioutil.WriteFile(filepath.Join(testCGRoot, "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - defer func() { - ioutil.WriteFile(filepath.Join(config.CgroupRoot, "perf_event", "tasks"), []byte(fmt.Sprint(os.Getpid())), constant.DefaultFileMode) - assert.NoError(t, os.Remove(testCGRoot)) - }() - type args struct { - cgpath string - dur time.Duration - } - tests := []struct { - name string - args args - want *PerfStat - wantErr bool - }{ - { - name: "TC-normal", - args: args{cgpath: testCGRoot, dur: time.Second}, - }, - { - name: "TC-empty cgroup path", - args: args{cgpath: "", dur: time.Second}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := CgroupStat(tt.args.cgpath, tt.args.dur) - if (err != nil) != tt.wantErr { - t.Errorf("CgroupStat() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} diff --git a/pkg/qos/qos.go b/pkg/qos/qos.go deleted file mode 100644 index 35aaed5..0000000 --- a/pkg/qos/qos.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: QoS setting for pods - -package qos - -import ( - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - securejoin "github.com/cyphar/filepath-securejoin" - "github.com/pkg/errors" - - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -// SupportCgroupTypes are supported cgroup types for qos setting -var SupportCgroupTypes = []string{"cpu", "memory"} - -// SetQosLevel set pod qos_level -func SetQosLevel(pod *typedef.PodInfo) error { - if err := setQos(pod); err != nil { - return errors.Errorf("set qos for pod %s(%s) error: %v", pod.Name, pod.UID, err) - } - if err := validateQos(pod); err != nil { - return errors.Errorf("validate qos for pod %s(%s) error: %v", pod.Name, pod.UID, err) - } - - log.Logf("Set pod %s(UID=%s, offline=%v) qos level OK", pod.Name, pod.UID, pod.Offline) - return nil -} - -func UpdateQosLevel(pod *typedef.PodInfo) error { - if err := validateQos(pod); err != nil { - log.Logf("Checking pod %s(%s) value failed: %v, reset it", err, pod.Name, pod.UID) - if err := setQos(pod); err != nil { - return errors.Errorf("set qos for pod %s(%s) error: %v", pod.Name, pod.UID, err) - } - } - - return nil -} - -// setQos is used for setting pod's qos level following it's cgroup path -func setQos(pod *typedef.PodInfo) error { - if len(pod.UID) > constant.MaxPodIDLen { - return errors.Errorf("Pod id too long") - } - - // default qos_level is online, no need to set online pod qos_level - if !pod.Offline { - log.Logf("Set level=%v for pod %s(%s)", constant.MaxLevel, pod.Name, pod.UID) - return nil - } - log.Logf("Set level=%v for pod %s(%s)", constant.MinLevel, pod.Name, pod.UID) - - cgroupMap, err := initCgroupPath(pod.CgroupRoot, pod.CgroupPath) - if err != nil { - return err - } - - for kind, cgPath := range cgroupMap { - switch kind { - case "cpu": - if err := setQosLevel(cgPath, constant.CPUCgroupFileName, int(constant.MinLevel)); err != nil { - return err - } - case "memory": - if err := setQosLevel(cgPath, constant.MemoryCgroupFileName, int(constant.MinLevel)); err != nil { - return err - } - } - } - - return nil -} - -func setQosLevel(root, file string, target int) error { - if !util.IsDirectory(root) { - return errors.Errorf("Invalid cgroup path %q", root) - } - if old, err := getQosLevel(root, file); err == nil && target > old { - return errors.Errorf("Not support change qos level from low to high") - } - // walk through all sub paths - if err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error { - if f != nil && f.IsDir() { - cgFilePath, err := securejoin.SecureJoin(path, file) - if err != nil { - return errors.Errorf("Join path failed for %s and %s: %v", path, file, err) - } - if err = ioutil.WriteFile(cgFilePath, []byte(strconv.Itoa(target)), - constant.DefaultFileMode); err != nil { - return errors.Errorf("Setting qos level failed for %s=%d: %v", cgFilePath, target, err) - } - } - - return nil - }); err != nil { - return err - } - - return nil -} - -// validateQos is used for checking pod's qos level if equal to the value it should be set up to -func validateQos(pod *typedef.PodInfo) error { - var ( - cpuInfo, memInfo int - err error - qosLevel int - ) - - if !pod.Offline { - qosLevel = int(constant.MaxLevel) - } else { - qosLevel = int(constant.MinLevel) - } - - cgroupMap, err := initCgroupPath(pod.CgroupRoot, pod.CgroupPath) - if err != nil { - return err - } - for kind, cgPath := range cgroupMap { - switch kind { - case "cpu": - if cpuInfo, err = getQosLevel(cgPath, constant.CPUCgroupFileName); err != nil { - return errors.Errorf("read %s failed: %v", constant.CPUCgroupFileName, err) - } - case "memory": - if memInfo, err = getQosLevel(cgPath, constant.MemoryCgroupFileName); err != nil { - return errors.Errorf("read %s failed: %v", constant.MemoryCgroupFileName, err) - } - } - } - - if (cpuInfo != qosLevel) || (memInfo != qosLevel) { - return errors.Errorf("check level failed") - } - - return nil -} - -func getQosLevel(root, file string) (int, error) { - var ( - qosLevel int - rootQos []byte - err error - ) - - rootQos, err = util.ReadSmallFile(filepath.Join(root, file)) // nolint - if err != nil { - return constant.ErrCodeFailed, errors.Errorf("get root qos level failed: %v", err) - } - // walk through all sub paths - if err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error { - if f != nil && f.IsDir() { - cgFilePath, err := securejoin.SecureJoin(path, file) - if err != nil { - return errors.Errorf("join path failed: %v", err) - } - data, err := util.ReadSmallFile(filepath.Clean(cgFilePath)) - if err != nil { - return errors.Errorf("get qos level failed: %v", err) - } - if strings.Compare(string(data), string(rootQos)) != 0 { - return errors.Errorf("qos differs") - } - } - return nil - }); err != nil { - return constant.ErrCodeFailed, err - } - qosLevel, err = strconv.Atoi(strings.TrimSpace(string(rootQos))) - if err != nil { - return constant.ErrCodeFailed, err - } - - return qosLevel, nil -} - -// initCgroupPath return pod's cgroup full path -func initCgroupPath(cgroupRoot, cgroupPath string) (map[string]string, error) { - if cgroupRoot == "" { - cgroupRoot = constant.DefaultCgroupRoot - } - cgroupMap := make(map[string]string, len(SupportCgroupTypes)) - for _, kind := range SupportCgroupTypes { - if err := checkCgroupPath(cgroupPath); err != nil { - return nil, err - } - fullPath := filepath.Join(cgroupRoot, kind, cgroupPath) - if len(fullPath) > constant.MaxCgroupPathLen { - return nil, errors.Errorf("length of cgroup path exceeds max limit %d", constant.MaxCgroupPathLen) - } - cgroupMap[kind] = fullPath - } - - return cgroupMap, nil -} - -func checkCgroupPath(path string) error { - pathPrefix, blacklist := "kubepods", []string{"kubepods", "kubepods/besteffort", "kubepods/burstable"} - cPath := filepath.Clean(path) - - if !strings.HasPrefix(cPath, pathPrefix) { - return errors.Errorf("invalid cgroup path %v, should start with %v", path, pathPrefix) - } - - for _, invalidPath := range blacklist { - if cPath == invalidPath { - return errors.Errorf("invalid cgroup path %v, without podID", path) - } - } - - return nil -} diff --git a/pkg/qos/qos_test.go b/pkg/qos/qos_test.go deleted file mode 100644 index c90a649..0000000 --- a/pkg/qos/qos_test.go +++ /dev/null @@ -1,392 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: QoS testing - -package qos - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/typedef" -) - -type getQosTestArgs struct { - root string - file string -} - -type getQosTestCase struct { - name string - args getQosTestArgs - want int - wantErr bool -} - -const ( - qosFileWithValueNegativeOne string = "qos_level_with_negative_one" - qosFileWithValueZero string = "qos_level_with_value_zero" - qosFileWithValueInvalid string = "qos_level_with_value_invalid" -) - -func newGetTestCases(qosDir string) []getQosTestCase { - return []getQosTestCase{ - { - name: "TC1-get qos diff with value -1", - args: getQosTestArgs{root: qosDir, file: qosFileWithValueNegativeOne}, - want: 1, - wantErr: true, - }, - { - name: "TC2-get qos ok with value 0", - args: getQosTestArgs{root: qosDir, file: qosFileWithValueZero}, - want: 0, - wantErr: false, - }, - { - name: "TC3-get qos failed with invalid value", - args: getQosTestArgs{root: qosDir, file: qosFileWithValueInvalid}, - want: 1, - wantErr: true, - }, - { - name: "TC4-get qos failed with invalid file", - args: getQosTestArgs{root: qosDir, file: "file/not/exist"}, - want: 1, - wantErr: true, - }, - { - name: "TC5-get qos failed with not exist file", - args: getQosTestArgs{root: "/path/not/exist/file", file: "file_not_exist"}, - want: 1, - wantErr: true, - }, - } -} - -// test_rubik_check_cgroup_qoslevel_with_podinfo_0001 -func Test_getQos(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - qosDir, err := ioutil.TempDir(constant.TmpTestDir, "qos") - assert.NoError(t, err) - - os.MkdirAll(filepath.Join(qosDir, "diff"), constant.DefaultDirMode) - err = ioutil.WriteFile(filepath.Join(qosDir, qosFileWithValueNegativeOne), []byte("-1"), constant.DefaultFileMode) - assert.NoError(t, err) - err = ioutil.WriteFile(filepath.Join(qosDir, "diff", qosFileWithValueNegativeOne), []byte("0"), - constant.DefaultFileMode) - assert.NoError(t, err) - for _, dir := range []string{"", "diff"} { - err = ioutil.WriteFile(filepath.Join(qosDir, dir, qosFileWithValueZero), []byte("0"), - constant.DefaultFileMode) - assert.NoError(t, err) - err = ioutil.WriteFile(filepath.Join(qosDir, dir, qosFileWithValueInvalid), []byte("a"), - constant.DefaultFileMode) - assert.NoError(t, err) - } - - tests := newGetTestCases(qosDir) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getQosLevel(tt.args.root, tt.args.file) - if (err != nil) != tt.wantErr { - t.Errorf("getQosLevel() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("getQosLevel() got = %v, want %v", got, tt.want) - } - }) - } -} - -type setQoSTestArgs struct { - root string - file string - qosLevel int -} - -type setQosTestCase struct { - name string - args setQoSTestArgs - wantErr bool -} - -func newSetTestCases(qosDir string, qosFilePath *os.File) []setQosTestCase { - return []setQosTestCase{ - { - name: "TC1-set qos ok with value -1", - args: setQoSTestArgs{ - root: qosDir, - file: "cpu", - qosLevel: -1, - }, - wantErr: false, - }, - { - name: "TC1.1-set qos not ok with previous value is -1", - args: setQoSTestArgs{ - root: qosDir, - file: "cpu", - qosLevel: 0, - }, - wantErr: true, - }, - { - name: "TC2-set qos not ok with empty cgroup path", - args: setQoSTestArgs{ - root: "", - file: "cpu", - qosLevel: 0, - }, - wantErr: true, - }, - { - name: "TC3-set qos not ok with invalid cgroup path", - args: setQoSTestArgs{ - root: qosFilePath.Name(), - file: "cpu", - qosLevel: 0, - }, - wantErr: true, - }, - } -} - -// Test_setQos is setQosLevel function test -func Test_setQos(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - qosDir, err := ioutil.TempDir(constant.TmpTestDir, "qos") - assert.NoError(t, err) - qosFilePath, err := ioutil.TempFile(qosDir, "qos_file") - assert.NoError(t, err) - - tests := newSetTestCases(qosDir, qosFilePath) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := setQosLevel(tt.args.root, tt.args.file, tt.args.qosLevel); (err != nil) != tt.wantErr { - t.Errorf("setQosLevel() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - err = qosFilePath.Close() - assert.NoError(t, err) -} - -type fields struct { - CgroupRoot string - CgroupPath string -} - -type podInfoTestcase struct { - name string - fields fields - want map[string]string - wantErr bool -} - -func newPodInfoTestcases(cgRoot string) []podInfoTestcase { - return []podInfoTestcase{ - { - name: "TC1-get cgroup path ok with pre-define cgroupRoot", - fields: fields{ - CgroupRoot: cgRoot, - CgroupPath: "kubepods/podaaa", - }, - want: map[string]string{"cpu": filepath.Join(cgRoot, "cpu", "kubepods/podaaa"), - "memory": filepath.Join(cgRoot, "memory", "kubepods/podaaa")}, - }, - { - name: "TC2-get cgroup path ok with non define cgroupRoot", - fields: fields{ - CgroupPath: "kubepods/podbbb", - }, - want: map[string]string{"cpu": filepath.Join(constant.DefaultCgroupRoot, "cpu", - "kubepods/podbbb"), "memory": filepath.Join(constant.DefaultCgroupRoot, "memory", "kubepods/podbbb")}, - }, - { - name: "TC3-get invalid cgroup path", - fields: fields{CgroupPath: "invalid/cgroup/prefix/podbbb"}, - wantErr: true, - }, - { - name: "TC4-cgroup path too long", - fields: fields{ - CgroupPath: "kubepods/cgroup/prefix/podbbb" + strings.Repeat("/long", constant.MaxCgroupPathLen), - }, - wantErr: true, - }, - { - name: "TC5-cgroup invalid cgroup path kubepods", - fields: fields{CgroupPath: "kubepods"}, - wantErr: true, - }, - { - name: "TC6-cgroup invalid cgroup path kubepods/besteffort", - fields: fields{CgroupRoot: "", CgroupPath: "kubepods/besteffort/../besteffort"}, - wantErr: true, - }, - { - name: "TC7-cgroup invalid cgroup path kubepods/burstable", - fields: fields{CgroupRoot: "", CgroupPath: "kubepods/burstable//"}, - wantErr: true, - }, - } -} - -// test_rubik_check_podinfo_0002 -func TestPodInfo_CgroupFullPath(t *testing.T) { - cgRoot := filepath.Join(constant.TmpTestDir, t.Name()) - - tests := newPodInfoTestcases(cgRoot) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pod := &typedef.PodInfo{ - CgroupRoot: tt.fields.CgroupRoot, - CgroupPath: tt.fields.CgroupPath, - } - cgroupMap, err := initCgroupPath(pod.CgroupRoot, pod.CgroupPath) - fmt.Println(err) - if (err != nil) != tt.wantErr { - t.Errorf("initCgroupPath() = %v, want %v", err, tt.wantErr) - } else if !reflect.DeepEqual(cgroupMap, tt.want) { - t.Errorf("initCgroupPath() = %v, want %v", cgroupMap, tt.want) - } - }) - } -} - -type setQosFields struct { - CgroupPath string - Offline bool - PodID string -} - -type setQosTestCase2 struct { - name string - fields setQosFields - wantSetErr bool - wantValidateErr bool -} - -func newSetQosTestCase2() []setQosTestCase2 { - repeatName := 70 - return []setQosTestCase2{ - { - name: "TC1-setup qos ok", - fields: setQosFields{ - CgroupPath: "kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b2", - Offline: true, - PodID: "poda5cb0d50-1234-1234-1234-e0ae4b7884b2", - }, - }, - { - name: "TC3-setup too long podID", - fields: setQosFields{ - CgroupPath: "kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b3", - Offline: false, - PodID: "poda5cb0d50" + strings.Repeat("-1234", repeatName), - }, - wantValidateErr: true, - wantSetErr: true, - }, - { - name: "TC4-setup invalid cgroupPath", - fields: setQosFields{ - CgroupPath: "besteffort/poda5cb0d50-1234-1234-e0ae4b7884b2", - Offline: true, - PodID: "poda5cb0d50-1234-1234-e0ae4b7884b2", - }, - wantValidateErr: true, - wantSetErr: true, - }, - { - name: "TC5-not exist qos file", - fields: setQosFields{ - CgroupPath: "kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b3", - Offline: true, - PodID: "poda5cb0d50-1234-1234-1234-e0ae4b7884b2", - }, - wantValidateErr: true, - wantSetErr: true, - }, - } -} - -// test_rubik_check_podinfo_0001 -// test_rubik_check_cgroup_qoslevel_with_podinfo_0001 -// test_rubik_check_cgroup_qoslevel_with_podinfo_0002 -func TestPodInfo_SetQos(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, t.Name()) - assert.NoError(t, err) - podCPUCgroup := filepath.Join(cgRoot, "/cpu/kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b2") - podMemoryCgroup := filepath.Join(cgRoot, "/memory/kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b2") - tests := newSetQosTestCase2() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pod := &typedef.PodInfo{ - UID: tt.fields.PodID, - CgroupPath: tt.fields.CgroupPath, - CgroupRoot: cgRoot, - Offline: tt.fields.Offline, - } - if err != nil { - if !tt.wantSetErr { - t.Errorf("new PodInfo for %s failed: %v", tt.fields.PodID, err) - } - return - } - os.MkdirAll(podCPUCgroup, constant.DefaultFileMode) - os.MkdirAll(podMemoryCgroup, constant.DefaultFileMode) - if err := setQos(pod); (err != nil) != tt.wantSetErr { - t.Errorf("setQos() error = %v, wantErr %v", err, tt.wantSetErr) - } - if err := validateQos(pod); (err != nil) != tt.wantValidateErr { - t.Errorf("validateQos() error = %v, wantErr %v", err, tt.wantValidateErr) - } - err = os.RemoveAll(podCPUCgroup) - assert.NoError(t, err) - err = os.RemoveAll(podMemoryCgroup) - assert.NoError(t, err) - }) - } - - // test cgroup qoslevel differ with pod qoslevel - pod := &typedef.PodInfo{ - UID: "poda5cb0d50-1234-1234-1234-e0ae4b7884b2", - CgroupPath: "kubepods/besteffort/poda5cb0d50-1234-1234-1234-e0ae4b7884b2", - CgroupRoot: cgRoot, - Offline: false, - } - os.MkdirAll(podCPUCgroup, constant.DefaultFileMode) - os.MkdirAll(podMemoryCgroup, constant.DefaultFileMode) - err = setQos(pod) - assert.NoError(t, err) - pod.Offline = true - err = validateQos(pod) - assert.Equal(t, true, err != nil) -} diff --git a/pkg/quota/quota_burst.go b/pkg/quota/quota_burst.go deleted file mode 100644 index 641e514..0000000 --- a/pkg/quota/quota_burst.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Yanting Song -// Create: 2022-07-19 -// Description: quota burst setting for pods - -// Package quota is for quota settings -package quota - -import ( - "io/ioutil" - "math/big" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - - -// SetPodsQuotaBurst sync pod's burst quota when autoconfig is set -func SetPodsQuotaBurst(podInfos map[string]*typedef.PodInfo) { - for _, pi := range podInfos { - setPodQuotaBurst(pi) - } -} - -// UpdatePodQuotaBurst update pod's burst quota -func UpdatePodQuotaBurst(opi, npi *typedef.PodInfo) { - // cpm.GetPod returns nil if pod.UID not exist - if opi == nil || npi == nil { - log.Errorf("quota-burst got invalid nil podInfo") - return - } - if opi.QuotaBurst == npi.QuotaBurst { - return - } - setPodQuotaBurst(npi) -} - -// SetPodQuotaBurst set each container's cpu.cfs_burst_ns -func SetPodQuotaBurst(podInfo *typedef.PodInfo) { - // cpm.GetPod returns nil if pod.UID not exist - if podInfo == nil { - log.Errorf("quota-burst got invalid nil podInfo") - return - } - setPodQuotaBurst(podInfo) -} - -func setPodQuotaBurst(podInfo *typedef.PodInfo) { - if podInfo.QuotaBurst == constant.InvalidBurst { - return - } - burst := big.NewInt(podInfo.QuotaBurst).String() - for _, c := range podInfo.Containers { - err := setCtrQuotaBurst([]byte(burst), c) - if err != nil { - log.Errorf("set container quota burst failed: %v", err) - } - } -} - -func setCtrQuotaBurst(burst []byte, c *typedef.ContainerInfo) error { - const ( - fname = "cpu.cfs_burst_us" - subsys = "cpu" - ) - cgpath := c.CgroupPath(subsys) - fpath := filepath.Join(cgpath, fname) - - if _, err := os.Stat(fpath); err != nil && os.IsNotExist(err) { - return errors.Errorf("quota-burst path=%v missing", fpath) - } - - if err := ioutil.WriteFile(fpath, burst, constant.DefaultFileMode); err != nil { - return errors.Errorf("quota-burst path=%v setting failed: %v", fpath, err) - } - log.Infof("quota-burst path=%v setting success", cgpath) - return nil -} diff --git a/pkg/quota/quota_burst_test.go b/pkg/quota/quota_burst_test.go deleted file mode 100644 index 7169efb..0000000 --- a/pkg/quota/quota_burst_test.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Yanting Song -// Create: 2022-07-19 -// Description: This file is used for test quota burst - -package quota - -import ( - "io/ioutil" - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/typedef" -) - -const ( - cfsBurstUs = "cpu.cfs_burst_us" - cpuSubsys = "cpu" -) - -var cis = []*typedef.ContainerInfo{ - { - Name: "FooCon", - ID: "testCon1", - PodID: "testPod1", - CgroupRoot: constant.TmpTestDir, - CgroupAddr: "kubepods/testPod1/testCon1", - }, - { - Name: "BarCon", - ID: "testCon2", - PodID: "testPod2", - CgroupRoot: constant.TmpTestDir, - CgroupAddr: "kubepods/testPod2/testCon2", - }, - { - Name: "BiuCon", - ID: "testCon3", - PodID: "testPod3", - CgroupRoot: constant.TmpTestDir, - CgroupAddr: "kubepods/testPod3/testCon3", - }, - { - Name: "NotExist", - ID: "testCon4", - PodID: "testPod4", - CgroupRoot: constant.TmpTestDir, - CgroupAddr: "kubepods/testPod4/testCon4", - }, -} - -var pis = []*typedef.PodInfo{ - // valid QuotaBurstQuota value - { - Name: "FooPod", - UID: cis[0].PodID, - Containers: map[string]*typedef.ContainerInfo{ - cis[0].Name: cis[0], - }, - QuotaBurst: 0, - }, - // invalid QuotaBurstQuota value - { - Name: "BarPod", - UID: cis[1].PodID, - Containers: map[string]*typedef.ContainerInfo{ - cis[1].Name: cis[1], - }, - QuotaBurst: -1, - }, - // valid QuotaBurstQuota value - { - Name: "BiuPod", - UID: cis[2].PodID, - Containers: map[string]*typedef.ContainerInfo{ - cis[2].Name: cis[2], - }, - QuotaBurst: 1, - }, -} - -var notExistPod = &typedef.PodInfo{ - Name: "NotExistPod", - UID: cis[3].PodID, - Containers: map[string]*typedef.ContainerInfo{ - cis[3].Name: cis[3], - }, - QuotaBurst: 0, -} - -type updateQuotaBurstTestCase struct { - oldPodInfo *typedef.PodInfo - newPodInfo *typedef.PodInfo - name string - wantValue string -} - -type podQuotaBurstTestCase struct { - podInfo *typedef.PodInfo - name string - wantValue string -} - -func createCgroupPath(t *testing.T) error { - for _, ci := range pis { - for _, ctr := range ci.Containers { - ctrAddr := filepath.Join(constant.TmpTestDir, cpuSubsys, ctr.CgroupAddr) - if err := os.MkdirAll(ctrAddr, constant.DefaultDirMode); err != nil { - return err - } - if _, err := os.Create(filepath.Join(ctrAddr, cfsBurstUs)); err != nil { - return err - } - } - } - return nil -} - -// TestSetPodsQuotaBurst tests auto check pod's quota burst -func TestSetPodsQuotaBurst(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - - pods := make(map[string]*typedef.PodInfo, len(pis)) - for _, pi := range pis { - pods[pi.Name] = pi - } - if err := createCgroupPath(t); err != nil { - t.Errorf("createCgroupPath got %v ", err) - } - - SetPodsQuotaBurst(pods) - - for i, pi := range pis { - ctrAddr := filepath.Join(constant.TmpTestDir, cpuSubsys, cis[i].CgroupAddr) - var quotaBurst []byte - if quotaBurst, err = ioutil.ReadFile(filepath.Join(ctrAddr, cfsBurstUs)); err != nil { - t.Errorf("readFile got %v ", err) - } - var expected string - if pi.QuotaBurst == constant.InvalidBurst { - expected = "" - } else { - expected = strconv.Itoa(int(pi.QuotaBurst)) - } - assert.Equal(t, expected, string(quotaBurst)) - } -} - -// TestUpdateCtrQuotaBurst tests update pod's quota burst -func TestUpdateCtrQuotaBurst(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - - if err := createCgroupPath(t); err != nil { - t.Errorf("createCgroupPath got %v ", err) - } - for _, tt := range []updateQuotaBurstTestCase{ - { - name: "TC1-update quota burst with old podinfo is nil", - oldPodInfo: nil, - newPodInfo: pis[0], - wantValue: "", - }, - { - name: "TC2-update quota burst without change", - oldPodInfo: pis[0], - newPodInfo: pis[0], - wantValue: "", - }, - { - name: "TC3-update quota burst", - oldPodInfo: pis[1], - newPodInfo: pis[0], - wantValue: "0", - }, - } { - t.Run(tt.name, func(t *testing.T) { - UpdatePodQuotaBurst(tt.oldPodInfo, tt.newPodInfo) - for _, ctr := range tt.newPodInfo.Containers { - ctrAddr := filepath.Join(constant.TmpTestDir, cpuSubsys, ctr.CgroupAddr) - var quotaBurst []byte - if quotaBurst, err = ioutil.ReadFile(filepath.Join(ctrAddr, cfsBurstUs)); err != nil { - t.Errorf("readFile got %v ", err) - } - assert.Equal(t, tt.wantValue, string(quotaBurst)) - } - }) - } -} - -// TestSetPodQuotaBurst tests set quota burst of pod -func TestSetPodQuotaBurst(t *testing.T) { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - defer os.RemoveAll(constant.TmpTestDir) - if err := createCgroupPath(t); err != nil { - t.Errorf("createCgroupPath got %v ", err) - } - err = setCtrQuotaBurst([]byte("0"), notExistPod.Containers["NotExist"]) - assert.Contains(t, err.Error(), "missing") - - for i, tt := range []podQuotaBurstTestCase{ - { - name: "TC1-set pod burst with valid quota value 0", - podInfo: pis[0], - wantValue: "0", - }, - { - name: "TC2-set pod burst with invalid quota value", - podInfo: pis[1], - wantValue: "", - }, - { - name: "TC3-set pod burst with nil podinfo", - podInfo: nil, - wantValue: "", - }, - } { - t.Run(tt.name, func(t *testing.T) { - SetPodQuotaBurst(tt.podInfo) - ctrAddr := filepath.Join(constant.TmpTestDir, cpuSubsys, cis[i].CgroupAddr) - var quotaBurst []byte - if quotaBurst, err = ioutil.ReadFile(filepath.Join(ctrAddr, cfsBurstUs)); err != nil { - t.Errorf("readFile got %v ", err) - } - assert.Equal(t, tt.wantValue, string(quotaBurst)) - }) - } -} diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go deleted file mode 100644 index 14e9cf8..0000000 --- a/pkg/rubik/rubik.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-05-20 -// Description: This file is used for rubik struct - -// Package rubik is for rubik struct -package rubik - -import ( - "context" - "fmt" - "os" - "os/signal" - "sync/atomic" - "syscall" - - "github.com/coreos/go-systemd/daemon" - "github.com/pkg/errors" - "golang.org/x/sys/unix" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - - "isula.org/rubik/pkg/autoconfig" - "isula.org/rubik/pkg/blkio" - "isula.org/rubik/pkg/cachelimit" - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/iocost" - "isula.org/rubik/pkg/memory" - "isula.org/rubik/pkg/perf" - "isula.org/rubik/pkg/qos" - "isula.org/rubik/pkg/quota" - "isula.org/rubik/pkg/sync" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/util" -) - -// Rubik defines rubik struct -type Rubik struct { - config *config.Config - kubeClient *kubernetes.Clientset - cpm *checkpoint.Manager - mm *memory.MemoryManager - nodeName string -} - -// NewRubik creates a new rubik object -func NewRubik(cfgPath string) (*Rubik, error) { - cfg, err := config.NewConfig(cfgPath) - if err != nil { - return nil, errors.Errorf("load config failed: %v", err) - } - - if err = log.InitConfig(cfg.LogDriver, cfg.LogDir, cfg.LogLevel, int64(cfg.LogSize)); err != nil { - return nil, errors.Errorf("init log config failed: %v", err) - } - - r := &Rubik{ - config: cfg, - } - - if err := r.initComponents(); err != nil { - return nil, err - } - - return r, nil -} - -func (r *Rubik) initComponents() error { - if err := r.initKubeClient(); err != nil { - return err - } - - if err := r.initCheckpoint(); err != nil { - return err - } - - if err := r.initEventHandler(); err != nil { - return err - } - - if err := r.initNodeConfig(); err != nil { - return err - } - - if r.config.MemCfg.Enable { - if err := r.initMemoryManager(); err != nil { - return err - } - } - - return nil -} - -// Monitor monitors shutdown signal -func (r *Rubik) Monitor() { - <-config.ShutdownChan - os.Exit(1) -} - -// Sync sync pods qos level -func (r *Rubik) Sync() error { - if !r.config.AutoCheck { - return nil - } - return sync.Sync(r.cpm.ListOfflinePods()) -} - -// CacheLimit init cache limit module -func (r *Rubik) CacheLimit() error { - if r.config.CacheCfg.Enable { - if r.cpm == nil { - return fmt.Errorf("checkpoint is not initialized before cachelimit") - } - return cachelimit.Init(r.cpm, &r.config.CacheCfg) - } - return nil -} - -// QuotaBurst sync all pod's cpu burst quota -func (r *Rubik) QuotaBurst() { - if r.config.AutoCheck { - quota.SetPodsQuotaBurst(r.cpm.ListAllPods()) - } -} - -// initKubeClient initialize kubeClient if autoconfig is enabled -func (r *Rubik) initKubeClient() error { - conf, err := rest.InClusterConfig() - if err != nil { - return err - } - - kubeClient, err := kubernetes.NewForConfig(conf) - if err != nil { - return err - } - - r.kubeClient = kubeClient - log.Infof("the kube-client is initialized successfully") - return nil -} - -// initEventHandler initialize the event handler and set the rubik callback function corresponding to the pod event. -func (r *Rubik) initEventHandler() error { - if r.kubeClient == nil { - return fmt.Errorf("kube-client is not initialized") - } - - autoconfig.Backend = r - if err := autoconfig.Init(r.kubeClient, r.nodeName); err != nil { - return err - } - - log.Infof("the event-handler is initialized successfully") - return nil -} - -func (r *Rubik) initMemoryManager() error { - mm, err := memory.NewMemoryManager(r.cpm, r.config.MemCfg) - if err != nil { - return err - } - - r.mm = mm - log.Infof("init memory manager ok") - return nil -} - -func (r *Rubik) initCheckpoint() error { - if r.kubeClient == nil { - return fmt.Errorf("kube-client is not initialized") - } - - cpm := checkpoint.NewManager(r.config.CgroupRoot) - node := os.Getenv(constant.NodeNameEnvKey) - if node == "" { - return fmt.Errorf("missing %s", constant.NodeNameEnvKey) - } - - r.nodeName = node - const specNodeNameFormat = "spec.nodeName=%s" - pods, err := r.kubeClient.CoreV1().Pods("").List(context.Background(), - metav1.ListOptions{FieldSelector: fmt.Sprintf(specNodeNameFormat, node)}) - if err != nil { - return err - } - cpm.SyncFromCluster(pods.Items) - - r.cpm = cpm - log.Infof("the checkpoint is initialized successfully") - return nil -} - -func (r *Rubik) initNodeConfig() error { - for _, nc := range r.config.NodeConfig { - if nc.NodeName == r.nodeName && nc.IOcostEnable { - iocost.SetIOcostEnable(true) - return iocost.ConfigIOcost(nc.IOcostConfig) - } - } - return nil -} - -// AddEvent handle add event from informer -func (r *Rubik) AddEvent(pod *corev1.Pod) { - // Rubik does not process add event of pods that are not in the running state. - if pod.Status.Phase != corev1.PodRunning { - return - } - r.cpm.AddPod(pod) - - pi := r.cpm.GetPod(pod.UID) - if err := qos.SetQosLevel(pi); err != nil { - log.Errorf("AddEvent handle error: %v", err) - } - quota.SetPodQuotaBurst(pi) - - if r.config.BlkioCfg.Enable { - blkio.SetBlkio(pod) - } - if r.config.MemCfg.Enable { - r.mm.UpdateConfig(pi) - } - iocost.SetPodWeight(pi) -} - -// UpdateEvent handle update event from informer -func (r *Rubik) UpdateEvent(oldPod *corev1.Pod, newPod *corev1.Pod) { - // Rubik does not process updates of pods that are not in the running state - // if the container is not running, delete it. - if newPod.Status.Phase != corev1.PodRunning { - r.cpm.DelPod(newPod.UID) - return - } - - // after the Rubik is started, the pod adding events are transferred through the update handler of Kubernetes. - if !r.cpm.PodExist(newPod.UID) { - r.cpm.AddPod(newPod) - - if r.config.BlkioCfg.Enable { - blkio.SetBlkio(newPod) - } - - pi := r.cpm.GetPod(newPod.UID) - quota.SetPodQuotaBurst(pi) - - if r.config.MemCfg.Enable { - r.mm.UpdateConfig(pi) - } - iocost.SetPodWeight(pi) - } else { - opi := r.cpm.GetPod(oldPod.UID) - r.cpm.UpdatePod(newPod) - if r.config.BlkioCfg.Enable { - blkio.WriteBlkio(oldPod, newPod) - } - npi := r.cpm.GetPod(newPod.UID) - quota.UpdatePodQuotaBurst(opi, npi) - iocost.SetPodWeight(npi) - } - - cpmPod := r.cpm.GetPod(newPod.UID) - if err := qos.UpdateQosLevel(cpmPod); err != nil { - log.Errorf("UpdateEvent handle error: %v", err) - } -} - -// DeleteEvent handle update event from informer -func (r *Rubik) DeleteEvent(pod *corev1.Pod) { - r.cpm.DelPod(pod.UID) -} - -func run(fcfg string) int { - rubik, err := NewRubik(fcfg) - if err != nil { - fmt.Printf("new rubik failed: %v\n", err) - return constant.ErrCodeFailed - } - - if rubik.mm != nil { - rubik.mm.Run() - } - - log.Infof("perf hw support = %v", perf.HwSupport()) - if err = rubik.CacheLimit(); err != nil { - log.Errorf("cache limit init error: %v", err) - return constant.ErrCodeFailed - } - - rubik.QuotaBurst() - if err = rubik.Sync(); err != nil { - log.Errorf("sync qos level failed: %v", err) - } - - log.Logf("Start rubik with cfg\n%v", rubik.config) - go signalHandler() - - // Notify systemd that rubik is ready SdNotify() only tries to - // notify if the NOTIFY_SOCKET environment is set, so it's - // safe to call when systemd isn't present. - // Ignore the return values here because they're not valid for - // platforms that don't use systemd. For platforms that use - // systemd, rubik doesn't log if the notification failed. - _, _ = daemon.SdNotify(false, daemon.SdNotifyReady) - - rubik.Monitor() - - return 0 -} - -// Run start rubik server -func Run(fcfg string) int { - unix.Umask(constant.DefaultUmask) - if len(os.Args) > 1 { - fmt.Println("args not allowed") - return constant.ErrCodeFailed - } - - lock, err := util.CreateLockFile(constant.LockFile) - if err != nil { - fmt.Printf("set rubik lock failed: %v, check if there is another rubik running\n", err) - return constant.ErrCodeFailed - } - - ret := run(fcfg) - - util.RemoveLockFile(lock, constant.LockFile) - return ret -} - -func signalHandler() { - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) - - var forceCount int32 = 3 - for sig := range signalChan { - if sig == syscall.SIGTERM || sig == syscall.SIGINT { - if atomic.AddInt32(&config.ShutdownFlag, 1) == 1 { - log.Infof("Signal %v received and starting exit...", sig) - close(config.ShutdownChan) - } - - if atomic.LoadInt32(&config.ShutdownFlag) >= forceCount { - log.Infof("3 interrupts signal received, forcing rubik shutdown") - os.Exit(1) - } - } - } -} diff --git a/pkg/rubik/rubik_test.go b/pkg/rubik/rubik_test.go deleted file mode 100644 index 1a23b36..0000000 --- a/pkg/rubik/rubik_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-05-20 -// Description: This file is used for rubik package test - -package rubik - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - "isula.org/rubik/pkg/checkpoint" - "isula.org/rubik/pkg/config" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/typedef" - "isula.org/rubik/pkg/util" -) - -func TestNewRubik(t *testing.T) { - os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - defer os.RemoveAll(constant.TmpTestDir) - tmpConfigFile := filepath.Join(constant.TmpTestDir, "config.json") - os.Remove(tmpConfigFile) - - os.MkdirAll(tmpConfigFile, constant.DefaultDirMode) - _, err := NewRubik(tmpConfigFile) - assert.Equal(t, true, err != nil) - - err = os.RemoveAll(tmpConfigFile) - assert.NoError(t, err) - _, err = os.Create(tmpConfigFile) - assert.NoError(t, err) - - err = reCreateConfigFile(tmpConfigFile, `{ - "autoCheck": true, - "logDriver": "file", - "logDir": "/tmp/rubik-test", - "logSize": 2048, - "logLevel": "debug", - "cgroupRoot": "/tmp/rubik-test/cgroup" -}`) - assert.Equal(t, true, strings.Contains(err.Error(), "must be defined")) - - err = reCreateConfigFile(tmpConfigFile, `{ - "logLevel": "debugabc" -}`) - assert.Equal(t, true, err != nil) -} - -func reCreateConfigFile(path, content string) error { - err := os.Remove(path) - if err != nil { - return err - } - fd, err := os.Create(path) - if err != nil { - return err - } - defer fd.Close() - _, err = fd.WriteString(content) - if err != nil { - return err - } - _, err = NewRubik(path) - if err != nil { - return err - } - - return nil -} - -var cfgA = ` -{ - "autoCheck": true, - "logDriver": "file", - "logDir": "/tmp/rubik-test", - "logSize": 2048, - "logLevel": "debug", - "cgroupRoot": "/tmp/rubik-test/cgroup" -}` - -// TestRunAbnormality test run server abnormality -func TestRunAbnormality(t *testing.T) { - old := os.Args - defer func() { - os.Args = old - }() - configFile := "config.json" - fcfg := filepath.Join(constant.TmpTestDir, configFile) - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - if err != nil { - assert.NoError(t, err) - } - err = ioutil.WriteFile(fcfg, []byte(cfgA), constant.DefaultFileMode) - if err != nil { - assert.NoError(t, err) - } - // case: argument not valid - os.Args = []string{"invalid", "failed"} - ret := Run(fcfg) - assert.Equal(t, constant.ErrCodeFailed, ret) - os.Args = []string{"rubik"} - // case: file is locked - lock, err := util.CreateLockFile(constant.LockFile) - ret = Run(fcfg) // set rubik lock failed: ... - assert.Equal(t, constant.ErrCodeFailed, ret) - util.RemoveLockFile(lock, constant.LockFile) - // case: invalid config.json - err = ioutil.WriteFile(fcfg, []byte("invalid"), constant.DefaultFileMode) - if err != nil { - assert.NoError(t, err) - } - ret = Run(fcfg) - assert.Equal(t, constant.ErrCodeFailed, ret) -} - -// TestRun test run server -func TestRun(t *testing.T) { - if os.Getenv("BE_TESTRUN") == "1" { - // case: config.json missing, use default config.json - ret := Run("/dev/should/not/exist") - fmt.Println(ret) - return - } - cmd := exec.Command(os.Args[0], "-test.run=TestRun") - cmd.Env = append(os.Environ(), "BE_TESTRUN=1") - err := cmd.Start() - assert.NoError(t, err) - sleepTime := 3 - time.Sleep(time.Duration(sleepTime) * time.Second) - err = cmd.Process.Signal(syscall.SIGINT) - assert.NoError(t, err) -} - -// TestCacheLimit is CacheLimit function test -func TestCacheLimit(t *testing.T) { - rubik := &Rubik{ - config: &config.Config{ - CacheCfg: config.CacheConfig{ - Enable: true, - DefaultLimitMode: "invalid", - DefaultResctrlDir: constant.TmpTestDir + "invalid", - }, - }, - cpm: &checkpoint.Manager{ - Checkpoint: &checkpoint.Checkpoint{ - Pods: make(map[string]*typedef.PodInfo), - }, - }, - } - - err := rubik.CacheLimit() - assert.Equal(t, true, err != nil) - rubik.config.CacheCfg.Enable = false - err = rubik.CacheLimit() - assert.NoError(t, err) -} - -// TestSmallRun test run function -func TestSmallRun(t *testing.T) { - tmpConfigFile := filepath.Join(constant.TmpTestDir, "config.json") - os.Remove(tmpConfigFile) - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - assert.NoError(t, err) - fd, err := os.Create(tmpConfigFile) - assert.NoError(t, err) - fd.WriteString(`{ - "cacheConfig": { - "enable": true, - "defaultLimitMode": "invalid" - } -}`) - assert.NoError(t, err) - err = fd.Close() - assert.NoError(t, err) - - res := run(tmpConfigFile) - assert.Equal(t, constant.ErrCodeFailed, res) -} - -// TestInitKubeClient test initKubeClient -func TestInitKubeClient(t *testing.T) { - r := &Rubik{} - err := r.initKubeClient() - assert.Equal(t, true, strings.Contains(err.Error(), "must be defined")) -} - -// TestInitEventHandler test initEventHandler -func TestInitEventHandler(t *testing.T) { - r := &Rubik{} - err := r.initEventHandler() - assert.Equal(t, true, strings.Contains(err.Error(), "kube-client is not initialized")) - - r.kubeClient = &kubernetes.Clientset{} - err = r.initEventHandler() - assert.NoError(t, err) -} - -// TestInitCheckpoint test initCheckpoint -func TestInitCheckpoint(t *testing.T) { - r := &Rubik{config: &config.Config{CgroupRoot: ""}} - err := r.initCheckpoint() - assert.Equal(t, true, strings.Contains(err.Error(), "kube-client is not initialized")) - - r.kubeClient = &kubernetes.Clientset{} - err = r.initCheckpoint() - assert.Equal(t, true, strings.Contains(err.Error(), "missing")) -} - -// TestAddUpdateDelEvent test Event -func TestAddUpdateDelEvent(t *testing.T) { - cfg, err := config.NewConfig("") - assert.NoError(t, err) - r := &Rubik{config: cfg} - - cpm := checkpoint.NewManager("") - r.cpm = cpm - oldPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: "aaa", - Name: "podaaa", - }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - }, - } - r.AddEvent(oldPod) - oldPod.Status.Phase = corev1.PodRunning - r.AddEvent(oldPod) - assert.Equal(t, "podaaa", r.cpm.Checkpoint.Pods["aaa"].Name) - - newPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: "aaa", - Name: "podbbb", - }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - }, - } - r.UpdateEvent(oldPod, newPod) - _, ok := r.cpm.Checkpoint.Pods["aaa"] - assert.Equal(t, false, ok) - - r.AddEvent(oldPod) - newPod.Status.Phase = corev1.PodRunning - r.UpdateEvent(oldPod, newPod) - assert.Equal(t, "podbbb", r.cpm.Checkpoint.Pods["aaa"].Name) - - r.DeleteEvent(newPod) - _, ok = r.cpm.Checkpoint.Pods["aaa"] - assert.Equal(t, false, ok) - r.UpdateEvent(oldPod, newPod) - assert.Equal(t, "podbbb", r.cpm.Checkpoint.Pods["aaa"].Name) -} diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go deleted file mode 100644 index aa063ea..0000000 --- a/pkg/sync/sync.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-04-20 -// Description: qos setting sync - -package sync - -import ( - "isula.org/rubik/pkg/cachelimit" - "isula.org/rubik/pkg/qos" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - -// Sync qos setting -func Sync(pods map[string]*typedef.PodInfo) error { - for _, pod := range pods { - if err := qos.SetQosLevel(pod); err != nil { - log.Errorf("sync set pod %v qoslevel error: %v", pod.UID, err) - } - if cachelimit.ClEnabled() { - syncCache(pod) - } - } - - return nil -} - -func syncCache(pi *typedef.PodInfo) { - err := cachelimit.SyncLevel(pi) - if err != nil { - log.Errorf("sync pod %v level error: %v", pi.UID, err) - return - } - if err = cachelimit.SetCacheLimit(pi); err != nil { - log.Errorf("sync pod %v cache limit error: %v", pi.UID, err) - } -} diff --git a/pkg/tinylog/tinylog.go b/pkg/tinylog/tinylog.go deleted file mode 100644 index 6c5ca98..0000000 --- a/pkg/tinylog/tinylog.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Haomin Tsai -// Create: 2021-09-28 -// Description: This file is used for rubik log - -// Package tinylog is for rubik log -package tinylog - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" - - "isula.org/rubik/pkg/constant" -) - -// CtxKey used for UUID -type CtxKey string - -const ( - // UUID is log uuid - UUID = "uuid" - - logStdio = 0 - logFile = 1 - logDriverStdio = "stdio" - logDriverFile = "file" - - logDebug = 0 - logInfo = 1 - logError = 2 - logStack = 20 - logStackFrom = 2 - logLevelInfo = "info" - logLevelStack = "stack" - - logFileNum = 10 - logSizeMin int64 = 10 // 10MB - logSizeMax int64 = 1024 * 1024 // 1TB - unitMB int64 = 1024 * 1024 -) - -var ( - logDriver = logStdio - logFname = filepath.Join(constant.DefaultLogDir, "rubik.log") - logLevel = 0 - logSize int64 = 1024 - logFileMaxSize int64 - logFileSize int64 - - lock = sync.Mutex{} -) - -func makeLogDir(logDir string) error { - if !filepath.IsAbs(logDir) { - return fmt.Errorf("log-dir %v must be an absolute path", logDir) - } - - if err := os.MkdirAll(logDir, constant.DefaultDirMode); err != nil { - return fmt.Errorf("create log directory %v failed", logDir) - } - - return nil -} - -// InitConfig init log config -func InitConfig(driver, logdir, level string, size int64) error { - if driver == "" { - driver = logDriverStdio - } - if driver != logDriverStdio && driver != logDriverFile { - return fmt.Errorf("invalid log driver %s", driver) - } - logDriver = logStdio - if driver == logDriverFile { - logDriver = logFile - } - - if level == "" { - level = logLevelInfo - } - levelstr, err := logLevelFromString(level) - if err != nil { - return err - } - logLevel = levelstr - - if size < logSizeMin || size > logSizeMax { - return fmt.Errorf("invalid log size %d", size) - } - logSize = size - logFileMaxSize = logSize / logFileNum - - if driver == "file" { - if err := makeLogDir(logdir); err != nil { - return err - } - logFname = filepath.Join(logdir, "rubik.log") - if f, err := os.Stat(logFname); err == nil { - atomic.StoreInt64(&logFileSize, f.Size()) - } - } - - return nil -} - -// DropError drop unused error -func DropError(args ...interface{}) { - argn := len(args) - if argn == 0 { - return - } - arg := args[argn-1] - if arg != nil { - fmt.Printf("drop error: %v\n", arg) - } -} - -func logLevelToString(level int) string { - switch level { - case logDebug: - return "debug" - case logInfo: - return "info" - case logError: - return "error" - case logStack: - return logLevelStack - default: - return "" - } -} - -func logLevelFromString(level string) (int, error) { - switch level { - case "debug": - return logDebug, nil - case "info", "": - return logInfo, nil - case "error": - return logError, nil - default: - return logInfo, fmt.Errorf("invalid log level %s", level) - } -} - -func logRename() { - for i := logFileNum - 1; i > 1; i-- { - old := logFname + fmt.Sprintf(".%d", i-1) - new := logFname + fmt.Sprintf(".%d", i) - if _, err := os.Stat(old); err == nil { - DropError(os.Rename(old, new)) - } - } - DropError(os.Rename(logFname, logFname+".1")) -} - -func logRotate(line int64) string { - if atomic.AddInt64(&logFileSize, line) > logFileMaxSize*unitMB { - logRename() - atomic.StoreInt64(&logFileSize, line) - } - - return logFname -} - -func writeLine(line string) { - if logDriver == logStdio { - fmt.Printf("%s", line) - return - } - - lock.Lock() - defer lock.Unlock() - - f, err := os.OpenFile(logRotate(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) - if err != nil { - return - } - - DropError(f.WriteString(line)) - DropError(f.Close()) -} - -func logf(level string, format string, args ...interface{}) { - tag := fmt.Sprintf("%s [rubik] level=%s ", time.Now().Format("2006-01-02 15:04:05.000"), level) - raw := fmt.Sprintf(format, args...) + "\n" - - depth := 1 - if level == logLevelStack { - depth = logStack - } - - for i := logStackFrom; i < logStackFrom+depth; i++ { - line := tag + raw - pc, file, linum, ok := runtime.Caller(i) - if ok { - fs := strings.Split(runtime.FuncForPC(pc).Name(), "/") - fs = strings.Split("."+fs[len(fs)-1], ".") - fn := fs[len(fs)-1] - line = tag + fmt.Sprintf("%s:%d:%s() ", file, linum, fn) + raw - } else if level == logLevelStack { - break - } - writeLine(line) - } -} - -// Logf log info level -func Logf(format string, args ...interface{}) { - if logInfo >= logLevel { - logf(logLevelToString(logInfo), format, args...) - } -} - -// Infof log info level -func Infof(format string, args ...interface{}) { - if logInfo >= logLevel { - logf(logLevelToString(logInfo), format, args...) - } -} - -// Debugf log debug level -func Debugf(format string, args ...interface{}) { - if logDebug >= logLevel { - logf(logLevelToString(logDebug), format, args...) - } -} - -// Errorf log error level -func Errorf(format string, args ...interface{}) { - if logError >= logLevel { - logf(logLevelToString(logError), format, args...) - } -} - -// Stackf log stack dump -func Stackf(format string, args ...interface{}) { - logf("stack", format, args...) -} - -// Entry is log entry -type Entry struct { - Ctx context.Context -} - -// WithCtx create entry with ctx -func WithCtx(ctx context.Context) *Entry { - return &Entry{ - Ctx: ctx, - } -} - -func (e *Entry) level(l int) string { - uuid, ok := e.Ctx.Value(CtxKey(UUID)).(string) - if ok { - return logLevelToString(l) + " UUID=" + uuid - } - return logLevelToString(l) -} - -// Logf write logs -func (e *Entry) Logf(f string, args ...interface{}) { - if logInfo < logLevel { - return - } - logf(e.level(logInfo), f, args...) -} - -// Infof write logs -func (e *Entry) Infof(f string, args ...interface{}) { - if logInfo < logLevel { - return - } - logf(e.level(logInfo), f, args...) -} - -// Debugf write verbose logs -func (e *Entry) Debugf(f string, args ...interface{}) { - if logDebug < logLevel { - return - } - logf(e.level(logDebug), f, args...) -} - -// Errorf write error logs -func (e *Entry) Errorf(f string, args ...interface{}) { - if logError < logLevel { - return - } - logf(e.level(logError), f, args...) -} diff --git a/pkg/tinylog/tinylog_test.go b/pkg/tinylog/tinylog_test.go deleted file mode 100644 index f36b040..0000000 --- a/pkg/tinylog/tinylog_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-05-24 -// Description: This file is used for testing tinylog - -package tinylog - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/try" -) - -// test_rubik_set_logdriver_0001 -func TestInitConfigLogDriver(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - // case: rubik.log already exist. - try.WriteFile(logFilePath, []byte(""), constant.DefaultFileMode) - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - - err = os.RemoveAll(logDir) - assert.NoError(t, err) - - // logDriver is file - err = InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logFile, logDriver) - logString := "Test InitConfig with logDriver file" - Logf(logString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, true, strings.Contains(string(b), logString)) - - // logDriver is stdio - os.Remove(logFilePath) - err = InitConfig("stdio", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logStdio, logDriver) - logString = "Test InitConfig with logDriver stdio" - Logf(logString) - b, err = ioutil.ReadFile(logFilePath) - assert.Equal(t, true, err != nil) - - // logDriver invalid - err = InitConfig("std", logDir, "", logSize) - assert.Equal(t, true, err != nil) - - // logDriver is null - err = InitConfig("", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logStdio, logDriver) -} - -// test_rubik_set_logdir_0001 -func TestInitConfigLogDir(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - // LogDir valid - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - logString := "Test InitConfig with logDir valid" - Logf(logString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, true, strings.Contains(string(b), logString)) - - // logDir invalid - err = InitConfig("file", "invalid/log", "", logSize) - assert.Equal(t, true, err != nil) -} - -type logTC struct { - name, logLevel string - wantErr, debug, info, error bool -} - -func createLogTC() []logTC { - return []logTC{ - { - name: "TC1-logLevel debug", - logLevel: "debug", - wantErr: false, - debug: true, - info: true, - error: true, - }, - { - name: "TC2-logLevel info", - logLevel: "info", - wantErr: false, - debug: false, - info: true, - error: true, - }, - { - name: "TC3-logLevel error", - logLevel: "error", - wantErr: false, - debug: false, - info: false, - error: true, - }, - { - name: "TC4-logLevel null", - logLevel: "", - wantErr: false, - debug: false, - info: true, - error: true, - }, - { - name: "TC5-logLevel invalid", - logLevel: "inf", - wantErr: true, - }, - } -} - -// test_rubik_set_loglevel_0001 -func TestInitConfigLogLevel(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - debugLogSting, infoLogSting, errorLogSting, logLogString := "Test InitConfig debug log", - "Test InitConfig info log", "Test InitConfig error log", "Test InitConfig log log" - for _, tt := range createLogTC() { - t.Run(tt.name, func(t *testing.T) { - err := InitConfig("file", logDir, tt.logLevel, logSize) - if (err != nil) != tt.wantErr { - t.Errorf("InitConfig() = %v, want %v", err, tt.wantErr) - } else if tt.wantErr == false { - Debugf(debugLogSting) - Infof(infoLogSting) - Errorf(errorLogSting) - Logf(logLogString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) - assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) - os.Remove(logFilePath) - - ctx := context.WithValue(context.Background(), CtxKey(UUID), "abc123") - WithCtx(ctx).Debugf(debugLogSting) - WithCtx(ctx).Infof(infoLogSting) - WithCtx(ctx).Errorf(errorLogSting) - WithCtx(ctx).Logf(logLogString) - b, err = ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) - assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) - assert.Equal(t, true, strings.Contains(string(b), "abc123")) - err = os.RemoveAll(logDir) - assert.NoError(t, err) - } - }) - } -} - -// test_rubik_set_logsize_0001 -func TestInitConfigLogSize(t *testing.T) { - logDir := try.GenTestDir().String() - // LogSize invalid - err := InitConfig("file", logDir, "", logSizeMin-1) - assert.Equal(t, true, err != nil) - err = InitConfig("file", logDir, "", logSizeMax+1) - assert.Equal(t, true, err != nil) - - // logSize valid - testSize, printLine, repeat := 100, 50000, 100 - err = InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - for i := 0; i < printLine; i++ { - Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) - } - err = InitConfig("file", logDir, "", int64(testSize)) - assert.NoError(t, err) - for i := 0; i < printLine; i++ { - Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) - } - var size int64 - err = filepath.Walk(logDir, func(_ string, f os.FileInfo, _ error) error { - size += f.Size() - return nil - }) - assert.NoError(t, err) - assert.Equal(t, true, size < int64(testSize)*unitMB) - err = os.RemoveAll(constant.TmpTestDir) - assert.NoError(t, err) -} - -// TestLogStack is Stackf function test -func TestLogStack(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - Stackf("test stack log") - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - fmt.Println(string(b)) - assert.Equal(t, true, strings.Contains(string(b), t.Name())) - line := strings.Split(string(b), "\n") - maxLineNum := 5 - assert.Equal(t, true, len(line) < maxLineNum) -} - -// TestDropError is DropError function test -func TestDropError(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - DropError() - dropError := "test drop error" - DropError(dropError) - DropError(nil) - _, err = ioutil.ReadFile(logFilePath) - assert.Equal(t, true, err != nil) -} - -// TestLogOthers is log other tests -func TestLogOthers(t *testing.T) { - logDir := filepath.Join(try.GenTestDir().String(), "regular-file") - try.WriteFile(logDir, []byte{}, constant.DefaultFileMode) - - err := makeLogDir(logDir) - assert.Equal(t, true, err != nil) - - level1 := 3 - s := logLevelToString(level1) - assert.Equal(t, "", s) - level2 := 20 - s = logLevelToString(level2) - assert.Equal(t, "stack", s) - - logDriver = 1 - logFname = filepath.Join(constant.TmpTestDir, "log-not-exist") - os.MkdirAll(logFname, constant.DefaultDirMode) - writeLine("abc") - - s = WithCtx(context.Background()).level(1) - assert.Equal(t, "info", s) - - logLevel = logError + 1 - WithCtx(context.Background()).Errorf("abc") -} diff --git a/pkg/try/try.go b/pkg/try/try.go deleted file mode 100644 index a0c4a33..0000000 --- a/pkg/try/try.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: jingrui -// Create: 2022-04-17 -// Description: try provide some helper functions for unit-test. -// -// Package try provide some helper function for unit-test, if you want -// to use try outside unit-test, please add notes. -// -// 1. Try some function quiet|log|die on error. because some test -// function does not care the error returned, but the code checker -// always generate unuseful warnings. This method can suppress the -// noisy warnings. -// -// 2. Provide testdir helper to generate tmpdir for unitest. -// -package try - -import ( - "fmt" - "io/ioutil" - "os" - - securejoin "github.com/cyphar/filepath-securejoin" - "github.com/google/uuid" - - "isula.org/rubik/pkg/constant" -) - -// Ret provide some action for error. -type Ret struct { - val interface{} - err error -} - -func newRet(err error) Ret { - return Ret{ - val: nil, - err: err, - } -} - -// OrDie try func or die with fail. -func (r Ret) OrDie() { - if r.err == nil { - return - } - fmt.Printf("try failed, die with error %v", r.err) - os.Exit(-1) -} - -// ErrMessage get ret error string format -func (r Ret) ErrMessage() string { - if r.err == nil { - return "" - } - return fmt.Sprintf("%v", r.err) -} - -// String get ret val and convert to string -func (r Ret) String() string { - val, ok := r.val.(string) - if ok { - return val - } - return "" -} - -// SecureJoin wrap error to Ret. -func SecureJoin(root, unsafe string) Ret { - name, err := securejoin.SecureJoin(root, unsafe) - ret := newRet(err) - if err == nil { - ret.val = name - } - return ret -} - -// MkdirAll wrap error to Ret. -func MkdirAll(path string, perm os.FileMode) Ret { - if err := os.MkdirAll(path, perm); err != nil { - return newRet(err) - } - return newRet(nil) -} - -// RemoveAll wrap error to Ret. -func RemoveAll(path string) Ret { - if err := os.RemoveAll(path); err != nil { - return newRet(err) - } - return newRet(nil) -} - -// WriteFile wrap error to Ret. -func WriteFile(filename string, data []byte, perm os.FileMode) Ret { - ret := newRet(nil) - ret.val = filename - if err := ioutil.WriteFile(filename, data, perm); err != nil { - ret.err = err - } - return ret -} - -const ( - testdir = "/tmp/rubik-test" -) - -// GenTestDir gen testdir -func GenTestDir() Ret { - name := fmt.Sprintf("%s/%s", testdir, uuid.New().String()) - ret := MkdirAll(name, constant.DefaultDirMode) - ret.val = name - return ret -} - -// DelTestDir del testdir, this function only need call once. -func DelTestDir() Ret { - return RemoveAll(testdir) -} diff --git a/pkg/try/try_test.go b/pkg/try/try_test.go deleted file mode 100644 index 6d17d02..0000000 --- a/pkg/try/try_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: jingrui -// Create: 2022-04-17 -// Description: try provide some helper functions for unit-test. -// -// Package try provide some helper function for unit-test, if you want -// to use try outside unit-test, please add notes. - -package try - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" -) - -// Test_OrDie test try some-func or die. -func Test_OrDie(t *testing.T) { - ret := GenTestDir() - ret.OrDie() - dname := ret.String() - WriteFile(SecureJoin(dname, "die.txt").String(), []byte("ok"), constant.DefaultFileMode).OrDie() - RemoveAll(dname).OrDie() -} - -// Test_ErrMessage test try some-func or check the error. -func Test_ErrMessage(t *testing.T) { - ret := GenTestDir() - assert.Equal(t, ret.ErrMessage(), "") - dname := ret.String() - WriteFile(SecureJoin(dname, "log.txt").String(), []byte("ok"), constant.DefaultFileMode).ErrMessage() - assert.Equal(t, RemoveAll(dname).ErrMessage(), "") -} diff --git a/pkg/typedef/convert.go b/pkg/typedef/convert.go deleted file mode 100644 index 77af888..0000000 --- a/pkg/typedef/convert.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-03-25 -// Description: This package stores basic type conversion functions. - -package typedef - -import ( - "crypto/rand" - "math/big" - "strconv" -) - -// FormatInt64 convert the int 64 type to a string -func FormatInt64(n int64) string { - const base = 10 - return strconv.FormatInt(n, base) -} - -// ParseInt64 convert the string type to Int64 -func ParseInt64(str string) (int64, error) { - const ( - base = 10 - bitSize = 64 - ) - return strconv.ParseInt(str, base, bitSize) -} - -// ParseFloat64 convert the string type to Float64 -func ParseFloat64(str string) (float64, error) { - const bitSize = 64 - return strconv.ParseFloat(str, bitSize) -} - -// FormatFloat64 convert the Float64 type to string -func FormatFloat64(f float64) string { - const ( - precision = -1 - bitSize = 64 - format = 'f' - ) - return strconv.FormatFloat(f, format, precision, bitSize) -} - -// RandInt provide safe rand int in range [0, max) -func RandInt(max int) int { - n, err := rand.Int(rand.Reader, big.NewInt(int64(max))) - if err != nil { - return 0 - } - return int(n.Int64()) -} diff --git a/pkg/typedef/convert_test.go b/pkg/typedef/convert_test.go deleted file mode 100644 index 35cc50c..0000000 --- a/pkg/typedef/convert_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-07-10 -// Description: This package stores basic type conversion functions. - -// Package typedef provides a common type conversion method. -package typedef - -import ( - "math" - "testing" -) - -// TestFormatInt64 is testcase for FormatInt64 -func TestFormatInt64(t *testing.T) { - type args struct { - n int64 - } - validNum := 100 - tests := []struct { - name string - args args - want string - }{ - { - name: "TC-convert the int 64 to string", - args: args{n: int64(validNum)}, - want: "100", - }, - { - name: "TC-convert the big int", - args: args{math.MaxInt64}, - want: "9223372036854775807", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := FormatInt64(tt.args.n); got != tt.want { - t.Errorf("FormatInt64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestParseInt64 is testcase for ParseInt64 -func TestParseInt64(t *testing.T) { - type args struct { - str string - } - validNum := 100 - tests := []struct { - name string - args args - want int64 - wantErr bool - }{ - { - name: "TC-convert the int 64 to string", - args: args{str: "100"}, - want: int64(validNum), - }, - { - name: "TC-convert the big int", - args: args{str: "9223372036854775807"}, - want: math.MaxInt64, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseInt64(tt.args.str) - if (err != nil) != tt.wantErr { - t.Errorf("ParseInt64() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("ParseInt64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestFormatFloat64 is testcase for FormatFloat64 -func TestFormatFloat64(t *testing.T) { - type args struct { - f float64 - } - validNum := 100.0 - tests := []struct { - name string - args args - want string - }{ - { - name: "TC-convert the float64 to string", - args: args{f: validNum}, - want: "100", - }, - { - name: "TC-convert the big float", - args: args{math.MaxFloat64}, - want: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := FormatFloat64(tt.args.f); got != tt.want { - t.Errorf("FormatFloat64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestParseFloat64 is testcase for ParseFloat64 -func TestParseFloat64(t *testing.T) { - type args struct { - str string - } - validNum := 100.0 - tests := []struct { - name string - args args - want float64 - wantErr bool - }{ - { - name: "TC-convert the string to float64", - args: args{str: "100"}, - want: validNum, - }, - { - name: "TC-convert the big float", - args: args{str: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - want: math.MaxFloat64, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseFloat64(tt.args.str) - if (err != nil) != tt.wantErr { - t.Errorf("ParseFloat64() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("ParseFloat64() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/typedef/types.go b/pkg/typedef/types.go deleted file mode 100644 index a9a1e28..0000000 --- a/pkg/typedef/types.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jing Rui -// Create: 2021-04-27 -// Description: This file contains default constants used in the project - -// Package typedef is general used types. -package typedef - -import ( - "path/filepath" - - corev1 "k8s.io/api/core/v1" -) - -// ContainerInfo represent container -type ContainerInfo struct { - // Basic Information - Name string `json:"name"` - ID string `json:"id"` - PodID string `json:"podID"` - CgroupRoot string `json:"cgroupRoot"` - CgroupAddr string `json:"cgroupAddr"` -} - -// NewContainerInfo create container info -func NewContainerInfo(container corev1.Container, podID, conID, cgroupRoot, podCgroupPath string) *ContainerInfo { - c := ContainerInfo{ - Name: container.Name, - ID: conID, - PodID: podID, - CgroupRoot: cgroupRoot, - CgroupAddr: filepath.Join(podCgroupPath, conID), - } - return &c -} - -// CgroupPath return full cgroup path -func (ci *ContainerInfo) CgroupPath(subsys string) string { - if ci == nil || ci.Name == "" { - return "" - } - return filepath.Join(ci.CgroupRoot, subsys, ci.CgroupAddr) -} - -// Clone return deepcopy object. -func (ci *ContainerInfo) Clone() *ContainerInfo { - copy := *ci - return © -} - -// PodInfo represent pod -type PodInfo struct { - // Basic Information - Containers map[string]*ContainerInfo `json:"containers,omitempty"` - Name string `json:"name"` - UID string `json:"uid"` - CgroupPath string `json:"cgroupPath"` - Namespace string `json:"namespace"` - CgroupRoot string `json:"cgroupRoot"` - - // Service Information - Offline bool `json:"offline"` - CacheLimitLevel string `json:"cacheLimitLevel,omitempty"` - - // value of quota burst - QuotaBurst int64 `json:"quotaBurst"` -} - -// Clone return deepcopy object -func (pi *PodInfo) Clone() *PodInfo { - if pi == nil { - return nil - } - copy := *pi - // deepcopy reference object - copy.Containers = make(map[string]*ContainerInfo, len(pi.Containers)) - for _, c := range pi.Containers { - copy.Containers[c.Name] = c.Clone() - } - return © -} - -// AddContainerInfo store container info to checkpoint -func (pi *PodInfo) AddContainerInfo(containerInfo *ContainerInfo) { - // key should not be empty - if containerInfo.Name == "" { - return - } - pi.Containers[containerInfo.Name] = containerInfo -} diff --git a/pkg/typedef/types_test.go b/pkg/typedef/types_test.go deleted file mode 100644 index fe368b7..0000000 --- a/pkg/typedef/types_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jing Rui -// Create: 2022-07-10 -// Description: This file contains default constants used in the project - -// Package typedef is general used types. -package typedef - -import ( - "io/ioutil" - "log" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/constant" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -func init() { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - if err != nil { - log.Fatalf("Failed to create tmp test dir for testing!") - } -} - -func genContainer() corev1.Container { - c := corev1.Container{} - c.Name = "testContainer" - c.Resources.Requests = make(corev1.ResourceList) - c.Resources.Limits = make(corev1.ResourceList) - c.Resources.Requests["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - c.Resources.Limits["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - c.Resources.Limits["memory"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - - return c -} - -// TestNewContainerInfo is testcase for NewContainerInfo -func TestNewContainerInfo(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - - c := genContainer() - type args struct { - container corev1.Container - podID string - conID string - cgroupRoot string - podCgroupPath string - } - tests := []struct { - want *ContainerInfo - name string - args args - }{ - { - name: "TC", - args: args{container: c, podID: "podID", cgroupRoot: cgRoot, conID: "cID", podCgroupPath: podCGPath}, - want: &ContainerInfo{ - Name: "testContainer", - ID: "cID", - PodID: "podID", - CgroupRoot: cgRoot, - CgroupAddr: filepath.Join(podCGPath, "cID"), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := NewContainerInfo(tt.args.container, tt.args.podID, tt.args.conID, tt.args.cgroupRoot, tt.args.podCgroupPath); !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewContainerInfo() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestContainerInfo_CgroupPath is testcase for ContainerInfo.CgroupPath -func TestContainerInfo_CgroupPath(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(podCGPath) - - emptyCi := &ContainerInfo{} - assert.Equal(t, "", emptyCi.CgroupPath("cpu")) - - ci := emptyCi.Clone() - - ci.Name = "testContainer" - ci.ID = "cID" - ci.PodID = "podID" - ci.CgroupRoot = cgRoot - ci.CgroupAddr = filepath.Join(podCGPath, "cID") - assert.Equal(t, ci.CgroupPath("cpu"), - filepath.Join(cgRoot, "cpu", filepath.Join(podCGPath, "cID"))) -} - -// TestPodInfo_Clone is testcase for PodInfo.Clone -func TestPodInfo_Clone(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(podCGPath) - emptyPI := &PodInfo{} - pi := emptyPI.Clone() - pi.Containers = make(map[string]*ContainerInfo) - pi.Name = "testPod" - pi.UID = "abcd" - pi.CgroupPath = cgRoot - - containerWithOutName := genContainer() - containerWithOutName.Name = "" - - emptyNameCI := NewContainerInfo(containerWithOutName, "testPod", "cID", cgRoot, podCGPath) - pi.AddContainerInfo(emptyNameCI) - assert.Equal(t, len(pi.Containers), 0) - - ci := NewContainerInfo(genContainer(), "testPod", "cID", cgRoot, podCGPath) - pi.AddContainerInfo(ci) - newPi := pi.Clone() - assert.Equal(t, len(newPi.Containers), 1) -} diff --git a/pkg/util/file.go b/pkg/util/file.go deleted file mode 100644 index 9d9728b..0000000 --- a/pkg/util/file.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: filepath related common functions - -package util - -import ( - "io/ioutil" - "os" - "path/filepath" - "syscall" - - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" -) - -const ( - fileMaxSize = 10 * 1024 * 1024 // 10MB -) - -// CreateFile create full path including dir and file. -func CreateFile(path string) error { - if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { - return err - } - - f, err := os.Create(path) - if err != nil { - return err - } - - return f.Close() -} - -// IsDirectory returns true if the file exists and it is a dir -func IsDirectory(path string) bool { - fi, err := os.Lstat(path) - if err != nil { - return false - } - - return fi.IsDir() -} - -// ReadSmallFile read small file less than 10MB -func ReadSmallFile(path string) ([]byte, error) { - st, err := os.Lstat(path) - if err != nil { - return nil, err - } - if st.Size() > fileMaxSize { - return nil, constant.ErrFileTooBig - } - return ioutil.ReadFile(path) // nolint: gosec -} - -// PathExist returns true if the path exists -func PathExist(path string) bool { - if _, err := os.Lstat(path); err != nil { - return false - } - - return true -} - -// CreateLockFile creates a lock file -func CreateLockFile(p string) (*os.File, error) { - path := filepath.Clean(p) - if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { - return nil, err - } - - lock, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DefaultFileMode) - if err != nil { - return nil, err - } - - if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - log.DropError(lock.Close()) - return nil, err - } - - return lock, nil -} - -// RemoveLockFile removes lock file - this function used cleanup resource, -// errors will ignored to make sure more source is cleaned. -func RemoveLockFile(lock *os.File, path string) { - log.DropError(syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)) - log.DropError(lock.Close()) - log.DropError(os.Remove(path)) -} diff --git a/pkg/util/file_test.go b/pkg/util/file_test.go deleted file mode 100644 index 9f7f3dc..0000000 --- a/pkg/util/file_test.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-17 -// Description: filepath related common functions testing - -package util - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/constant" -) - -// TestIsDirectory is IsDirectory function test -func TestIsDirectory(t *testing.T) { - directory, err := ioutil.TempDir(constant.TmpTestDir, t.Name()) - assert.NoError(t, err) - defer os.RemoveAll(directory) - - filePath, err := ioutil.TempFile(directory, t.Name()) - assert.NoError(t, err) - - type args struct { - path string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "TC1-directory is exist", - args: args{path: directory}, - want: true, - }, - { - name: "TC2-directory is not exist", - args: args{path: "/directory/is/not/exist"}, - want: false, - }, - { - name: "TC3-test file path", - args: args{path: filePath.Name()}, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsDirectory(tt.args.path); got != tt.want { - t.Errorf("IsDirectory() = %v, want %v", got, tt.want) - } - }) - } - err = filePath.Close() - assert.NoError(t, err) -} - -// TestPathIsExist is PathExist function test -func TestPathIsExist(t *testing.T) { - filePath, err := ioutil.TempDir(constant.TmpTestDir, "file_exist") - assert.NoError(t, err) - defer os.RemoveAll(filePath) - - type args struct { - path string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "TC1-path is exist", - args: args{path: filePath}, - want: true, - }, - { - name: "TC2-path is not exist", - args: args{path: "/path/is/not/exist"}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := PathExist(tt.args.path); got != tt.want { - t.Errorf("PathExist() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestReadSmallFile is test for read file -func TestReadSmallFile(t *testing.T) { - filePath, err := ioutil.TempDir(constant.TmpTestDir, "read_file") - assert.NoError(t, err) - defer os.RemoveAll(filePath) - - // case1: ok - err = ioutil.WriteFile(filepath.Join(filePath, "ok"), []byte{}, constant.DefaultFileMode) - assert.NoError(t, err) - _, err = ReadSmallFile(filepath.Join(filePath, "ok")) - assert.NoError(t, err) - - // case2: too big - size := 20000000 - big := make([]byte, size, size) - err = ioutil.WriteFile(filepath.Join(filePath, "big"), big, constant.DefaultFileMode) - assert.NoError(t, err) - _, err = ReadSmallFile(filepath.Join(filePath, "big")) - assert.Error(t, err) - - // case3: file not exist - _, err = ReadSmallFile(filepath.Join(filePath, "missing")) - assert.Error(t, err) -} - -// TestCreateLockFile is CreateLockFile function test -func TestCreateLockFile(t *testing.T) { - lockFile := filepath.Join(constant.TmpTestDir, "rubik.lock") - err := os.RemoveAll(lockFile) - assert.NoError(t, err) - - lock, err := CreateLockFile(lockFile) - assert.NoError(t, err) - RemoveLockFile(lock, lockFile) -} - -// TestLockFail is CreateLockFile fail test -func TestLockFail(t *testing.T) { - lockFile := filepath.Join(constant.TmpTestDir, "rubik.lock") - err := os.RemoveAll(constant.TmpTestDir) - assert.NoError(t, err) - os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - - _, err = os.Create(filepath.Join(constant.TmpTestDir, "rubik-lock")) - assert.NoError(t, err) - _, err = CreateLockFile(filepath.Join(constant.TmpTestDir, "rubik-lock", "rubik.lock")) - assert.Equal(t, true, err != nil) - err = os.RemoveAll(filepath.Join(constant.TmpTestDir, "rubik-lock")) - assert.NoError(t, err) - - err = os.MkdirAll(lockFile, constant.DefaultDirMode) - assert.NoError(t, err) - _, err = CreateLockFile(lockFile) - assert.Equal(t, true, err != nil) - err = os.RemoveAll(lockFile) - assert.NoError(t, err) - - _, err = CreateLockFile(lockFile) - assert.NoError(t, err) - _, err = CreateLockFile(lockFile) - assert.Equal(t, true, err != nil) - err = os.RemoveAll(lockFile) - assert.NoError(t, err) -} diff --git a/pkg/util/pod.go b/pkg/util/pod.go deleted file mode 100644 index 1669f16..0000000 --- a/pkg/util/pod.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2022-05-25 -// Description: Pod related common functions - -package util - -import ( - "path/filepath" - "strings" - - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/constant" - log "isula.org/rubik/pkg/tinylog" - "isula.org/rubik/pkg/typedef" -) - -const configHashAnnotationKey = "kubernetes.io/config.hash" - -// IsOffline judges whether pod is offline pod -func IsOffline(pod *corev1.Pod) bool { - return pod.Annotations[constant.PriorityAnnotationKey] == "true" -} - -func GetPodCacheLimit(pod *corev1.Pod) string { - return pod.Annotations[constant.CacheLimitAnnotationKey] -} - -// GetQuotaBurst checks CPU quota burst annotation value. -func GetQuotaBurst(pod *corev1.Pod) int64 { - quota := pod.Annotations[constant.QuotaBurstAnnotationKey] - if quota == "" { - return constant.InvalidBurst - } - - quotaBurst, err := typedef.ParseInt64(quota) - if err != nil { - log.Errorf("pod %s burst quota annotation value %v is invalid, expect integer", pod.Name, quotaBurst) - return constant.InvalidBurst - } - if quotaBurst < 0 { - log.Errorf("pod %s burst quota annotation value %v is invalid, expect positive", pod.Name, quotaBurst) - return constant.InvalidBurst - } - return quotaBurst -} - -// GetPodCgroupPath returns cgroup path of pod -func GetPodCgroupPath(pod *corev1.Pod) string { - var cgroupPath string - id := string(pod.UID) - if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { - id = configHash - } - - switch pod.Status.QOSClass { - case corev1.PodQOSGuaranteed: - cgroupPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBurstable: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), - constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBestEffort: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), - constant.PodCgroupNamePrefix+id) - } - - return cgroupPath -} diff --git a/pkg/util/pod_test.go b/pkg/util/pod_test.go deleted file mode 100644 index e8f5fb2..0000000 --- a/pkg/util/pod_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jingxiao Lu -// Create: 2022-05-25 -// Description: tests for pod.go - -package util - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/constant" -) - -const ( - trueStr = "true" -) - -func TestIsOffline(t *testing.T) { - var pod = &corev1.Pod{} - pod.Annotations = make(map[string]string) - pod.Annotations[constant.PriorityAnnotationKey] = trueStr - if !IsOffline(pod) { - t.Fatalf("%s failed for Annotations is %s", t.Name(), trueStr) - } - - delete(pod.Annotations, constant.PriorityAnnotationKey) - if IsOffline(pod) { - t.Fatalf("%s failed for Annotations no such key", t.Name()) - } -} - -// TestGetQuotaBurst is testcase for GetQuotaBurst -func TestGetQuotaBurst(t *testing.T) { - pod := &corev1.Pod{} - pod.Annotations = make(map[string]string) - maxInt64PlusOne := "9223372036854775808" - tests := []struct { - name string - quotaBurst string - want int64 - }{ - { - name: "TC1-valid quota burst", - quotaBurst: "1", - want: 1, - }, - { - name: "TC2-empty quota burst", - quotaBurst: "", - want: -1, - }, - { - name: "TC3-zero quota burst", - quotaBurst: "0", - want: 0, - }, - { - name: "TC4-negative quota burst", - quotaBurst: "-100", - want: -1, - }, - { - name: "TC5-float quota burst", - quotaBurst: "100.34", - want: -1, - }, - { - name: "TC6-nonnumerical quota burst", - quotaBurst: "nonnumerical", - want: -1, - }, - { - name: "TC7-exceed max int64", - quotaBurst: maxInt64PlusOne, - want: -1, - }, - } - for _, tt := range tests { - pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.quotaBurst - assert.Equal(t, GetQuotaBurst(pod), tt.want) - } -} - -func TestGetPodCgroupPath(t *testing.T) { - var pod = &corev1.Pod{} - pod.UID = "AAA" - var guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+string(pod.UID)) - var burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+string(pod.UID)) - var besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+string(pod.UID)) - pod.Annotations = make(map[string]string) - - // no pod.Annotations[configHashAnnotationKey] - pod.Status.QOSClass = corev1.PodQOSGuaranteed - if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { - t.Fatalf("%s failed for PodQOSGuaranteed without configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBurstable - if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { - t.Fatalf("%s failed for PodQOSBurstable without configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBestEffort - if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { - t.Fatalf("%s failed for PodQOSBestEffort without configHash", t.Name()) - } - pod.Status.QOSClass = "" - if !assert.Equal(t, GetPodCgroupPath(pod), "") { - t.Fatalf("%s failed for not setting QOSClass without configHash", t.Name()) - } - - // has pod.Annotations[configHashAnnotationKey] - pod.Annotations[configHashAnnotationKey] = "BBB" - var id = pod.Annotations[configHashAnnotationKey] - guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+id) - besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+id) - pod.Status.QOSClass = corev1.PodQOSGuaranteed - if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { - t.Fatalf("%s failed for PodQOSGuaranteed with configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBurstable - if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { - t.Fatalf("%s failed for PodQOSBurstable with configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBestEffort - if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { - t.Fatalf("%s failed for PodQOSBestEffort with configHash", t.Name()) - } - pod.Status.QOSClass = "" - if !assert.Equal(t, GetPodCgroupPath(pod), "") { - t.Fatalf("%s failed for not setting QOSClass with configHash", t.Name()) - } -} diff --git a/pkg/version/version.go b/pkg/version/version.go deleted file mode 100644 index e285a9b..0000000 --- a/pkg/version/version.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Xiang Li -// Create: 2021-04-25 -// Description: version releated - -// Package version is for version check -package version - -import ( - "fmt" - "os" - "runtime" -) - -var ( - // Version represents rubik version - Version string - // Release represents rubik release number - Release string - // GitCommit represents git commit number - GitCommit string - // BuildTime represents build time - BuildTime string -) - -func init() { - var showVersion bool - if len(os.Args) == 2 && os.Args[1] == "-v" { - showVersion = true - } - - if showVersion { - fmt.Println("Version: ", Version) - fmt.Println("Release: ", Release) - fmt.Println("Go Version: ", runtime.Version()) - fmt.Println("Git Commit: ", GitCommit) - fmt.Println("Built: ", BuildTime) - fmt.Println("OS/Arch: ", runtime.GOOS+"/"+runtime.GOARCH) - os.Exit(0) - } -} diff --git a/rubik.go b/rubik.go deleted file mode 100644 index fd8bedb..0000000 --- a/rubik.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jingrui -// Create: 2021-04-17 -// Description: This file is main program of rubik - -package main - -import ( - "os" - - _ "isula.org/rubik/pkg/version" - "isula.org/rubik/pkg/constant" - "isula.org/rubik/pkg/rubik" -) - -func main() { - os.Exit(rubik.Run(constant.ConfigFile)) -} diff --git a/tests/data/fuzz-test/README.md b/tests/data/fuzz-test/README.md deleted file mode 100644 index 84a17c6..0000000 --- a/tests/data/fuzz-test/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## How to Construct the go fuzz test - -> Note -> Before you start, make sure you have `go-fuzz-build` and `go-fuzz` binaries, you can get them at [go-fuzz](https://github.com/dvyukov/go-fuzz) - -1. cd `rubik/tests/data/fuzz-test` and make folder the form like `fuzz-test-xxx` -2. put the materials used by fuzz in the folder you created, they looks like the following form: -```bash -$ tree fuzz-test-newconfig - fuzz-test-newconfig # test case root dir - |-- corpus # dir to store mutation corpus - | |-- case1 # mutation corpus1 - | |-- case2 # mutation corpus2 - | |-- case3 # mutation corpus3 - |-- Fuzz # fuzz go file - |-- path # record relative path to put the Fuzz file in the package -``` -3. when the above meterials are ready, go to `rubik/tests/src` -4. the **ONLY Three Things** you need to do is: - 1. copy `TEMPLATE` file to the name you want(*must start with `fuzz_test`*), for example `fuzz_test_xxx.sh` - 2. change the variable `test_name` in the script you just copy same as the name you just gave(keep same with the folder you create in the first step) - 3. uncomment the last line `main "$1"` -5. To run single go fuzz shell script by doing `$ bash fuzz_test_xxx.sh`, it will stop fuzzing after 1 minute. - If you want to change the default run time, you could do like `$ bash fuzz_test_xxx.sh 2h` to keep running 2 hours -6. To run **all** go fuzz shell scripts by first go to `rubik/tests`, then run `$ bash test.sh 2h`. - It will run all go fuzz testcases and will stop fuzzing after `2h * number of go fuzz testcases` diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/Fuzz b/tests/data/fuzz-test/fuzz-test-newconfig/Fuzz deleted file mode 100644 index 74b89a6..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/Fuzz +++ /dev/null @@ -1,31 +0,0 @@ -package config - -import ( - "io/ioutil" - "os" - "path/filepath" - - "isula.org/rubik/pkg/constant" -) - -func Fuzz(data []byte) int { - if err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode); err != nil { - return -1 - } - defer os.RemoveAll(constant.TmpTestDir) - tmpDir, err := ioutil.TempDir(constant.TmpTestDir, "fuzz") - if err != nil { - return -1 - } - configFile := filepath.Join(tmpDir, "fuzz_config.json") - if err := ioutil.WriteFile(configFile, data, constant.DefaultFileMode); err != nil { - return -1 - } - _, err = NewConfig(configFile) - if err != nil { - return -1 - } - - return 1 -} - diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case1 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case1 deleted file mode 100644 index 469f609..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case1 +++ /dev/null @@ -1,24 +0,0 @@ -{ - "autoCheck": false, - "logDriver": "stdio", - "logDir": "/var/log/rubik", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup", - "cacheConfig": { - "enable": false, - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - } -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case2 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case2 deleted file mode 100644 index 2b570bd..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case2 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "logDriver": "file", - "logDir": "/var/lib/rubik/logs", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup" -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case3 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case3 deleted file mode 100644 index 7107b30..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case3 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "logDriver": "stdio", - "logDir": "", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup" -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case4 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case4 deleted file mode 100644 index d05d9b4..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case4 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "logDriver": "stdio", - "logDir": "/var/lib/rubik/logs", - "logSize": 9999999999, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup" -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case5 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case5 deleted file mode 100644 index f6dc7bc..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case5 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "logDriver": "stdio", - "logDir": "/var/lib/rubik/logs", - "logSize": 1024, - "logLevel": "!@#!@$", - "cgroupRoot": "/sys/fs/cgroup" -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case6 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case6 deleted file mode 100644 index b87bd49..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case6 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "logDriver": "stdio", - "logDir": "/var/lib/rubik/logs", - "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/path/not/exist" -} diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case7 b/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case7 deleted file mode 100644 index 2d58391..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/corpus/case7 +++ /dev/null @@ -1,18 +0,0 @@ -{ - "cacheConfig": { - "enable": false, - "defaultLimitMode": "dynamic", - "adjustInterval": 1000000, - "perfDuration": 999999, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - } -} \ No newline at end of file diff --git a/tests/data/fuzz-test/fuzz-test-newconfig/path b/tests/data/fuzz-test/fuzz-test-newconfig/path deleted file mode 100644 index b9ce5cd..0000000 --- a/tests/data/fuzz-test/fuzz-test-newconfig/path +++ /dev/null @@ -1 +0,0 @@ -pkg/config diff --git a/tests/data/fuzz-test/fuzz-test-pinghandler/Fuzz b/tests/data/fuzz-test/fuzz-test-pinghandler/Fuzz deleted file mode 100644 index 94ded66..0000000 --- a/tests/data/fuzz-test/fuzz-test-pinghandler/Fuzz +++ /dev/null @@ -1,23 +0,0 @@ -package httpserver - -import ( - "bufio" - "bytes" - "net/http" - "net/http/httptest" -) - -func Fuzz(data []byte) int { - r, err := http.NewRequest("GET", "/ping", bufio.NewReader(bytes.NewReader(data))) - if err != nil { - return -1 - } - w := httptest.NewRecorder() - handler := setupHandler() - handler.ServeHTTP(w, r) - if status := w.Code; status != http.StatusOK { - return -1 - } - - return 1 -} diff --git a/tests/data/fuzz-test/fuzz-test-pinghandler/corpus/case1 b/tests/data/fuzz-test/fuzz-test-pinghandler/corpus/case1 deleted file mode 100644 index 0f080be..0000000 --- a/tests/data/fuzz-test/fuzz-test-pinghandler/corpus/case1 +++ /dev/null @@ -1 +0,0 @@ -{"Pods": {"pod70f2828b-3f9c-42e2-97da-01c6072af4a6": {"CgroupPath": "kubepods/besteffort/pod70f2828b-3f9c-42e2-97da-01c6072af4a6", "QoSLevel": 0 }}} \ No newline at end of file diff --git a/tests/data/fuzz-test/fuzz-test-pinghandler/path b/tests/data/fuzz-test/fuzz-test-pinghandler/path deleted file mode 100644 index 9046824..0000000 --- a/tests/data/fuzz-test/fuzz-test-pinghandler/path +++ /dev/null @@ -1 +0,0 @@ -pkg/httpserver diff --git a/tests/data/fuzz-test/fuzz-test-roothandler/Fuzz b/tests/data/fuzz-test/fuzz-test-roothandler/Fuzz deleted file mode 100644 index 36910b5..0000000 --- a/tests/data/fuzz-test/fuzz-test-roothandler/Fuzz +++ /dev/null @@ -1,23 +0,0 @@ -package httpserver - -import ( - "bufio" - "bytes" - "net/http" - "net/http/httptest" -) - -func Fuzz(data []byte) int { - r, err := http.NewRequest("GET", "/", bufio.NewReader(bytes.NewReader(data))) - if err != nil { - return -1 - } - w := httptest.NewRecorder() - handler := setupHandler() - handler.ServeHTTP(w, r) - if status := w.Code; status != http.StatusOK { - return -1 - } - - return 1 -} diff --git a/tests/data/fuzz-test/fuzz-test-roothandler/corpus/case1 b/tests/data/fuzz-test/fuzz-test-roothandler/corpus/case1 deleted file mode 100644 index 91ab136..0000000 --- a/tests/data/fuzz-test/fuzz-test-roothandler/corpus/case1 +++ /dev/null @@ -1 +0,0 @@ -{"Pods": {"pod70f2828b-3f9c-42e2-97da-01c6072af4a6": {"CgroupPath": "kubepods/besteffort/pod70f2828b-3f9c-42e2-97da-01c6072af4a6", "QoSLevel": 0, "CacheLimitLevel": "low"}}} diff --git a/tests/data/fuzz-test/fuzz-test-roothandler/path b/tests/data/fuzz-test/fuzz-test-roothandler/path deleted file mode 100644 index 9046824..0000000 --- a/tests/data/fuzz-test/fuzz-test-roothandler/path +++ /dev/null @@ -1 +0,0 @@ -pkg/httpserver diff --git a/tests/data/fuzz-test/fuzz-test-versionhandler/Fuzz b/tests/data/fuzz-test/fuzz-test-versionhandler/Fuzz deleted file mode 100644 index 11ed700..0000000 --- a/tests/data/fuzz-test/fuzz-test-versionhandler/Fuzz +++ /dev/null @@ -1,23 +0,0 @@ -package httpserver - -import ( - "bufio" - "bytes" - "net/http" - "net/http/httptest" -) - -func Fuzz(data []byte) int { - r, err := http.NewRequest("GET", "/version", bufio.NewReader(bytes.NewReader(data))) - if err != nil { - return -1 - } - w := httptest.NewRecorder() - handler := setupHandler() - handler.ServeHTTP(w, r) - if status := w.Code; status != http.StatusOK { - return -1 - } - - return 1 -} diff --git a/tests/data/fuzz-test/fuzz-test-versionhandler/corpus/case1 b/tests/data/fuzz-test/fuzz-test-versionhandler/corpus/case1 deleted file mode 100644 index 0f080be..0000000 --- a/tests/data/fuzz-test/fuzz-test-versionhandler/corpus/case1 +++ /dev/null @@ -1 +0,0 @@ -{"Pods": {"pod70f2828b-3f9c-42e2-97da-01c6072af4a6": {"CgroupPath": "kubepods/besteffort/pod70f2828b-3f9c-42e2-97da-01c6072af4a6", "QoSLevel": 0 }}} \ No newline at end of file diff --git a/tests/data/fuzz-test/fuzz-test-versionhandler/path b/tests/data/fuzz-test/fuzz-test-versionhandler/path deleted file mode 100644 index 9046824..0000000 --- a/tests/data/fuzz-test/fuzz-test-versionhandler/path +++ /dev/null @@ -1 +0,0 @@ -pkg/httpserver diff --git a/tests/lib/commonlib.sh b/tests/lib/commonlib.sh deleted file mode 100755 index 1b6421f..0000000 --- a/tests/lib/commonlib.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-15 -# Description: common lib for integration test - -TOP_DIR=$(git rev-parse --show-toplevel) -RUBIK_TEST_ROOT="${TOP_DIR}"/.fortest -RUBIK_LIB=${RUBIK_TEST_ROOT}/rubik_lib -RUBIK_RUN=${RUBIK_TEST_ROOT}/rubik_run -RUBIK_LOG=${RUBIK_TEST_ROOT}/rubik_log -RUBIK_CONFIG="${RUBIK_LIB}"/config.json -RUBIK_NAME="rubik-agent" -QOS_HANDLER="http://localhost" -VERSION_HANDLER="http://localhost/version" -PING_HANDLER="http://localhost/ping" -PAUSE_IMAGE="k8s.gcr.io/pause:3.2" -OPENEULER_IMAGE="openeuler-22.03-lts:latest" -SKIP_FLAG=111 - -mkdir -p "${RUBIK_TEST_ROOT}" -mkdir -p "${RUBIK_LIB}" "${RUBIK_RUN}" "${RUBIK_LOG}" -exit_flag=0 - -## Description: build_rubik_img will build rubik image -# Usage: build_rubik_img -# Input: $1: baseimage default is scratch -# $2: images tag default is rubik_version -# Output: rubik image -# Example: build_rubik_img -function build_rubik_img() { - image_base=${1:-"scratch"} - image_tag=${2:-"fortest"} - rubik_img="rubik:$image_tag" - if [ "$image_base" != "scratch" ]; then - cp "${RUBIK_LIB}"/Dockerfile "${RUBIK_LIB}"/Dockerfilebak - sed -i "s#scratch#${image_base}#g" "${RUBIK_LIB}"/Dockerfile - fi - if [ ! -f "${TOP_DIR}"/rubik ]; then - make release - fi - cp "$TOP_DIR"/rubik "$RUBIK_LIB" - docker images | grep ^rubik | grep "${image_tag}" > /dev/null 2>&1 - if [ $? -ne 0 ]; then - docker build -f "${RUBIK_LIB}"/Dockerfile -t "${rubik_img}" "${RUBIK_LIB}" - [ "$image_base" != "scratch" ] && rm "${RUBIK_LIB}"/Dockerfile && mv "${RUBIK_LIB}"/Dockerfilebak "${RUBIK_LIB}"/Dockerfile - fi -} - -function generate_config_file() { - # get config from yaml config map - sed -n '/config.json:/{:a;n;/---/q;p;ba}' "$RUBIK_LIB"/rubik-daemonset.yaml > "${RUBIK_CONFIG}" - # disable autoConfig - sed -i 's/\"autoConfig\": true/\"autoConfig\": false/' "${RUBIK_CONFIG}" -} - -function prepare_rubik() { - runtime_check - if pgrep rubik > /dev/null 2>&1; then - echo "rubik is already running, please stop it first" - exit 1 - fi - cp "$TOP_DIR"/Dockerfile "$TOP_DIR"/hack/rubik-daemonset.yaml "${RUBIK_LIB}" - image_base=${1:-"scratch"} - image_tag=${2:-"fortest"} - rubik_img="rubik:$image_tag" - build_rubik_img "${image_base}" "${image_tag}" - generate_config_file -} - -function run_rubik() { - prepare_rubik - image_check - if [ ! -f "${RUBIK_CONFIG}" ]; then - rubik_pid=$(docker run -tid --name=${RUBIK_NAME} --pid=host --cap-add SYS_ADMIN \ - -v "${RUBIK_RUN}":/run/rubik -v "${RUBIK_LOG}":/var/log/rubik -v /sys/fs:/sys/fs "${rubik_img}") - else - rubik_pid=$(docker run -tid --name=${RUBIK_NAME} --pid=host --cap-add SYS_ADMIN \ - -v "${RUBIK_RUN}":/run/rubik -v "${RUBIK_LOG}":/var/log/rubik -v /sys/fs:/sys/fs \ - -v "${RUBIK_CONFIG}":/var/lib/rubik/config.json "${rubik_img}") - fi - return_code=$? - echo -n "$rubik_pid" - return $return_code -} - -function kill_rubik() { - docker inspect ${RUBIK_NAME} > /dev/null 2>&1 - if [ $? -eq 0 ]; then - docker logs ${RUBIK_NAME} - docker stop -t 0 ${RUBIK_NAME} - docker rm ${RUBIK_NAME} - fi -} - -function clean_all() { - rm -rf "${RUBIK_LIB}" "${RUBIK_RUN}" "${RUBIK_LOG}" - kill_rubik -} - -function runtime_check() { - if ! docker info > /dev/null 2>&1; then - echo "docker is not found, please install it via 'yum install docker'" - exit 1 - fi -} - -function image_check() { - openEuler_image="openeuler-22.03-lts" - pause_image="k8s.gcr.io/pause" - if ! docker images | grep $openEuler_image > /dev/null 2>&1; then - echo "openEuler image ${OPENEULER_IMAGE} is not found, please prepare it first before test begin" - exit 1 - fi - if ! docker images | grep ${pause_image} > /dev/null 2>&1; then - echo "pause image ${PAUSE_IMAGE} is not found, please prepare it first before test begin" - exit 1 - fi -} - -# Description: kernel_check will check wether the environment is met for testing -# Usage: kernel_check [...] -# Input: list of functional check requirements, default check all -# Output: 0(success) 1(fail) -# Example: kernel_check CPU、kernel_check CPU MEM CACHE、kernel_check ALL -function kernel_check() { - function cpu_check() { - ls /sys/fs/cgroup/cpu/cpu.qos_level > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "ls /sys/fs/cgroup/cpu/cpu.qos.level failed" - return 1 - fi - } - function mem_check() { - if [ ! -f /proc/sys/vm/memcg_qos_enable ]; then - echo "/proc/sys/vm/memcg_qos_enable is not an ordinary file" - return 1 - else - echo -n 1 > /proc/sys/vm/memcg_qos_enable - fi - ls /sys/fs/cgroup/memory/memory.qos_level > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "ls /sys/fs/cgroup/memory.qos_level failed" - return 1 - fi - } - function cache_check() { - ls /sys/fs/resctrl/schemata > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "ls /sys/fs/resctrl/schemata failed" - return 1 - fi - } - function check_all() { - cpu_check - mem_check - cache_check - } - for functional in $@; do - case $functional in - "CPU") - cpu_check - ;; - "MEM") - mem_check - ;; - "CACHE") - cache_check - ;; - "ALL" | *) - check_all - ;; - esac - done -} - -# Description: curl_cmd performs like curl command -# Usage: curl_cmd $http_handler $json_data $protocol -# Reminds: NOT recommend to use this method directly, use rubik_ping/rubik_version/rubik_qos instead in most occasions -# Input: -# $1: http_handler -# $2: json_data -# $3: protocol -# Output: curl command execute return message -# Return: success(0) or fail(not 0) -# Example: -# data=$(gen_pods_json json1 json2) -# QOS_HANDLER="http://localhost" -# PING_HANDLER="http://localhost/ping" -# VERSION_HANDLER="http://localhost/version" -# protocol="GET" -# -# curl_cmd $QOS_HANDLER $data $protocol -# curl_cmd $PING_HANDLER $protocol -# curl_cmd $VERSION_HANDLER $protocol -function curl_cmd() { - http_handler=$1 - data=$2 - protocol=$3 - result=$(curl -s -H "Accept: application/json" -H "Content-type: application/json" -X "$protocol" --data "$(echo -n "$data")" --unix-socket "${RUBIK_RUN}"/rubik.sock "$http_handler") - return_code=$? - echo "$result" - return $return_code -} - -function rubik_ping() { - curl_cmd "$PING_HANDLER" "" "GET" -} - -function rubik_version() { - curl_cmd "$VERSION_HANDLER" "" "GET" -} - -function rubik_qos() { - curl_cmd "$QOS_HANDLER" "$1" "POST" -} - -# Description: gen_single_pod_json will generate single pod qos info for one pod -# Usage: gen_single_pod_json $pod_id $cgroup_path $qos_level -# Input: $1: pod id, $2: cgroup path, $3: qos level -# Output: single pod qos info json data -# Example: json1=$(gen_single_pod_json "podaaaaaa" "this/is/cgroup/path" 1) -function gen_single_pod_json() { - pod_id=$1 - cgroup_path=$2 - qos_level=$3 - jq -n -c -r --arg pid "$pod_id" --arg cp "$cgroup_path" --arg qos "$qos_level" '{"Pods":{($pid): {"CgroupPath": $cp, "QoSLevel": ($qos|tonumber)}}}' -} - -function fn_check_result() { - if [ "$1" = "$2" ]; then - echo "PASS" - else - echo "FAIL" - ((exit_flag++)) - fi -} - -function fn_check_result_noeq() { - if [ "$1" != "$2" ]; then - echo "PASS" - else - echo "FAIL" - ((exit_flag++)) - fi -} - -function fn_check_string_contain() { - if echo "$2" | grep "$1"; then - echo "PASS" - else - echo "FAIL" - ((exit_flag++)) - fi -} - -function fn_check_string_not_contain() { - if ! echo "$2" | grep "$1"; then - echo "PASS" - else - echo "FAIL" - ((exit_flag++)) - fi -} - -# Description: long_char will generate long string by repeat char 'a' N times -# Usage: long_char $length -# Input: $1: length of string -# Output: repeate string with given length -# Example: long_char 10 -function long_char() { - length=$1 - head -c "$length" < /dev/zero | tr '\0' '\141' -} diff --git a/tests/lib/fuzz_commonlib.sh b/tests/lib/fuzz_commonlib.sh deleted file mode 100755 index eb7d813..0000000 --- a/tests/lib/fuzz_commonlib.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2020-08-27 -# Description: common functions for fuzz tests - -# Description: check the log and return the result -# if crash, return 1 -# if not, return 0 -# Usage: check_result /path/to/log -# $1: the full path of log -function check_result() { - local log=$1 - local time="[$(date '+%Y-%m-%d %H:%M:%S')]" - result=$(grep "crash" "$log" | tail -1 | awk '{print $10}') - result=${result%%,} - if [[ $result -eq 0 ]]; then - echo "PASS$time: No crash found" - return 0 - else - echo "FAIL$time: Crash found, See details in $log" - return 1 - fi -} - -# Description: sleep x s/m/h and kill the process -# Usage: kill_after 1h 10232 -# Input: $1: time to sleep -# $2: pid to kill -function kill_after() { - local time_out=$1 - local pid_to_kill=$2 - sleep "$time_out" - for j in $(seq 1 100); do - kill -9 "$pid_to_kill" > /dev/null 2>&1 - if pgrep -a "$pid_to_kill"; then - sleep 0.2 - else - break - fi - if [[ $j -eq 100 ]]; then - return 1 - fi - done -} - -# Description: compile Fuzz.go -# Usage: make_fuzz_zip $fuzz_file $fuzz_dir $test_dir -# Input: $1: path to Fuzz.go file -# $2: dir to put the Fuzz.go file -# $3: dir store the build output -# Return: success 0; failed 1 -# Warning: all input should be abs path :-) -function make_fuzz_zip() { - fuzz_file=$1 - fuzz_dir=$2 - data_dir=$3 - cp "$fuzz_file" "$fuzz_dir" - pushd "$fuzz_dir" > /dev/null 2>&1 || return 1 - mv Fuzz Fuzz.go - if ! go-fuzz-build "$fuzz_dir"; then - echo "go-fuzz-build failed" && return 1 - fi - mv "$fuzz_dir"/*.zip "$data_dir" - rm "$fuzz_dir/Fuzz.go" - popd > /dev/null 2>&1 || return 1 -} - -# Description: set enviroment for go fuzz test -# Usage: set_env "fuzz-test-abc" $top_dir -# Input: $1: test name -# $2: abs path for rubik project -# Note: 1. test_name must start with fuzz-test, for example fuzz-test-abc -# 2. go fuzz file must have name "Fuzz.go" -# 3. top_dir must be the abs path for the rubik project -# shellcheck disable=SC2034 -function set_env() { - test_name=$1 - top_dir=$2 - - test_root=$top_dir/tests/data/fuzz-test - test_dir=$test_root/$test_name - fuzz_file=$test_dir/Fuzz - fuzz_dir="$top_dir"/"$(cat "$test_dir"/path)" - fuzz_corpus="$test_dir/corpus" - fuzz_log="$test_dir/$test_name.log" - fuzz_crashers="$test_dir/crashers" - fuzz_suppressions="$test_dir/suppressions" - fuzz_zip="" -} - -function clean_env() { - rm -rf "$fuzz_zip" "$fuzz_crashers" "$fuzz_suppressions" - find /tmp -maxdepth 1 -iname "*fuzz*" -exec rm -rf {} \; -} diff --git a/tests/src/TEMPLATE b/tests/src/TEMPLATE deleted file mode 100755 index f450653..0000000 --- a/tests/src/TEMPLATE +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-12 -# Description: fuzz script template - -# top dir is path of where you put rubik project -top_dir=$(git rev-parse --show-toplevel) -# keep the name same as the folder you created before like "fuzz-test-xxx" -test_name="fuzz-test-root-handler" -# exit_flag is the flag to indicate if the test success(set 0) or failed(set 1) -exit_flag=0 -# get common functions used for test script -source "$top_dir"/tests/lib/fuzz_commonlib.sh - -# prepare the env before fuzz start -function pre_fun() { - # prepare env - set_env "${test_name}" "$top_dir" - # make fuzz zip file - make_fuzz_zip "$fuzz_file" "$fuzz_dir" "$test_dir" - fuzz_zip=$(ls "$test_dir"/*fuzz.zip) - if [[ -z "$fuzz_zip" ]]; then - echo "fuzz zip file not found" - exit 1 - fi -} - -# run fuzz -function test_fun() { - local time=$1 - if [[ -z "$time" ]]; then - time=1m - fi - go-fuzz -bin="$fuzz_zip" -workdir="$test_dir" &>> "$fuzz_log" & - pid=$! - if ! kill_after $time $pid > /dev/null 2>&1; then - echo "Can not kill process $pid" - fi - check_result "$fuzz_log" - res=$? - return $res -} - -function main() { - pre_fun - test_fun "$1" - res=$? - if [ $res -ne 0 ]; then - exit_flag=1 - else - clean_env - fi -} - -# uncomment following to make script working -main "$1" - -exit $exit_flag diff --git a/tests/src/fuzz_test_newconfig.sh b/tests/src/fuzz_test_newconfig.sh deleted file mode 100755 index 88fa3be..0000000 --- a/tests/src/fuzz_test_newconfig.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-12 -# Description: fuzz new config -# Number: test_rubik_fuzz_0004 - -top_dir=$(git rev-parse --show-toplevel) -test_name="fuzz-test-newconfig" -exit_flag=0 -source "$top_dir"/tests/lib/fuzz_commonlib.sh - -function pre_fun() { - set_env "${test_name}" "$top_dir" - make_fuzz_zip "$fuzz_file" "$fuzz_dir" "$test_dir" - fuzz_zip=$(ls "$test_dir"/*fuzz.zip) - if [[ -z "$fuzz_zip" ]]; then - echo "fuzz zip file not found" - exit 1 - fi -} - -function test_fun() { - local time=$1 - if [[ -z "$time" ]]; then - time=1m - fi - go-fuzz -bin="$fuzz_zip" -workdir="$test_dir" &>> "$fuzz_log" & - pid=$! - if ! kill_after $time $pid > /dev/null 2>&1; then - echo "Can not kill process $pid" - fi - check_result "$fuzz_log" - res=$? - return $res -} - -function main() { - pre_fun - test_fun "$1" - res=$? - if [ $res -ne 0 ]; then - exit_flag=1 - else - clean_env - fi -} - -main "$1" -exit $exit_flag diff --git a/tests/src/fuzz_test_pinghandler.sh b/tests/src/fuzz_test_pinghandler.sh deleted file mode 100755 index 170d191..0000000 --- a/tests/src/fuzz_test_pinghandler.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-12 -# Description: fuzz ping handler -# Number: test_rubik_fuzz_0002 - -top_dir=$(git rev-parse --show-toplevel) -test_name="fuzz-test-pinghandler" -exit_flag=0 -source "$top_dir"/tests/lib/fuzz_commonlib.sh - -function pre_fun() { - set_env "${test_name}" "$top_dir" - make_fuzz_zip "$fuzz_file" "$fuzz_dir" "$test_dir" - fuzz_zip=$(ls "$test_dir"/*fuzz.zip) - if [[ -z "$fuzz_zip" ]]; then - echo "fuzz zip file not found" - exit 1 - fi -} - -function test_fun() { - local time=$1 - if [[ -z "$time" ]]; then - time=1m - fi - go-fuzz -bin="$fuzz_zip" -workdir="$test_dir" &>> "$fuzz_log" & - pid=$! - if ! kill_after $time $pid > /dev/null 2>&1; then - echo "Can not kill process $pid" - fi - check_result "$fuzz_log" - res=$? - return $res -} - -function main() { - pre_fun - test_fun "$1" - res=$? - if [ $res -ne 0 ]; then - exit_flag=1 - else - clean_env - fi -} - -main "$1" -exit $exit_flag diff --git a/tests/src/fuzz_test_roothandler.sh b/tests/src/fuzz_test_roothandler.sh deleted file mode 100755 index 0d21c20..0000000 --- a/tests/src/fuzz_test_roothandler.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-12 -# Description: fuzz root handler -# Number: test_rubik_fuzz_0001 - -top_dir=$(git rev-parse --show-toplevel) -test_name="fuzz-test-roothandler" -exit_flag=0 -source "$top_dir"/tests/lib/fuzz_commonlib.sh - -function pre_fun() { - set_env "${test_name}" "$top_dir" - make_fuzz_zip "$fuzz_file" "$fuzz_dir" "$test_dir" - fuzz_zip=$(ls "$test_dir"/*fuzz.zip) - if [[ -z "$fuzz_zip" ]]; then - echo "fuzz zip file not found" - exit 1 - fi -} - -function test_fun() { - local time=$1 - if [[ -z "$time" ]]; then - time=1m - fi - go-fuzz -bin="$fuzz_zip" -workdir="$test_dir" &>> "$fuzz_log" & - pid=$! - if ! kill_after $time $pid > /dev/null 2>&1; then - echo "Can not kill process $pid" - fi - check_result "$fuzz_log" - res=$? - return $res -} - -function main() { - pre_fun - test_fun "$1" - res=$? - if [ $res -ne 0 ]; then - exit_flag=1 - else - clean_env - fi -} - -main "$1" -exit $exit_flag diff --git a/tests/src/fuzz_test_versionhandler.sh b/tests/src/fuzz_test_versionhandler.sh deleted file mode 100755 index 00a0756..0000000 --- a/tests/src/fuzz_test_versionhandler.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-12 -# Description: fuzz version handler -# Number: test_rubik_fuzz_0003 - -top_dir=$(git rev-parse --show-toplevel) -test_name="fuzz-test-versionhandler" -exit_flag=0 -source "$top_dir"/tests/lib/fuzz_commonlib.sh - -function pre_fun() { - set_env "${test_name}" "$top_dir" - make_fuzz_zip "$fuzz_file" "$fuzz_dir" "$test_dir" - fuzz_zip=$(ls "$test_dir"/*fuzz.zip) - if [[ -z "$fuzz_zip" ]]; then - echo "fuzz zip file not found" - exit 1 - fi -} - -function test_fun() { - local time=$1 - if [[ -z "$time" ]]; then - time=1m - fi - go-fuzz -bin="$fuzz_zip" -workdir="$test_dir" &>> "$fuzz_log" & - pid=$! - if ! kill_after $time $pid > /dev/null 2>&1; then - echo "Can not kill process $pid" - fi - check_result "$fuzz_log" - res=$? - return $res -} - -function main() { - pre_fun - test_fun "$1" - res=$? - if [ $res -ne 0 ]; then - exit_flag=1 - else - clean_env - fi -} - -main "$1" -exit $exit_flag diff --git a/tests/src/test_rubik_flags_0001.sh b/tests/src/test_rubik_flags_0001.sh deleted file mode 100755 index bbbf087..0000000 --- a/tests/src/test_rubik_flags_0001.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-30 -# Description: rubik flag check 0001 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -test_fun() { - # check rubik binary - if [ ! -f "${top_dir}"/rubik ]; then - pushd "${top_dir}" || exit 1 > /dev/null 2>&1 - make release - popd || exit 1 > /dev/null 2>&1 - fi - - # check rubik flag - if "${top_dir}"/rubik -v > /dev/null 2>&1; then - if ! "${top_dir}"/rubik --help > /dev/null 2>&1 && ! "${top_dir}"/rubik -h > /dev/null 2>&1; then - echo "PASS" - else - echo "FAILED" - fi - else - echo "FAILED" - fi -} - -test_fun - -exit "$exit_flag" diff --git a/tests/src/test_rubik_offline_0001.sh b/tests/src/test_rubik_offline_0001.sh deleted file mode 100755 index 232b02c..0000000 --- a/tests/src/test_rubik_offline_0001.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: 调整pod为离线业务后创建容器 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -function test_fun() { - cgroup_path="kubepods/${pod_id}" - qos_level_offline=-1 - # generate json data - data_offline=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level_offline) - - # set pod to offline - result=$(rubik_qos "$data_offline") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - - # create container and check qos level - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cpu_qos=$(cat /sys/fs/cgroup/cpu/"$cgroup_path"/"$cid"/cpu.qos_level) - mem_qos=$(cat /sys/fs/cgroup/memory/"$cgroup_path"/"$cid"/memory.qos_level) - fn_check_result "$cpu_qos" $qos_level_offline - fn_check_result "$mem_qos" $qos_level_offline -} - -function post_fun() { - clean_all - docker rm -f "$container_id" - fn_check_result $? 0 - docker rm -f "$pod_id" - fn_check_result $? 0 - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_offline_0002.sh b/tests/src/test_rubik_offline_0002.sh deleted file mode 100755 index 3eec239..0000000 --- a/tests/src/test_rubik_offline_0002.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: 调整带业务的pod为离线业务 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 - containers=() - total_num=50 -} - -function test_fun() { - cgroup_path="kubepods/${pod_id}" - qos_level_offline=-1 - # generate json data - data_offline=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level_offline) - - # create containers in the pod - for i in $(seq 1 ${total_num}); do - containers[$i]=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - fn_check_result $? 0 - done - - # set pod to offline - result=$(rubik_qos "$data_offline") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - - # check qos level for containers - for i in $(seq 1 ${total_num}); do - cpu_qos=$(cat /sys/fs/cgroup/cpu/"${cgroup_path}"/"${containers[$i]}"/cpu.qos_level) - mem_qos=$(cat /sys/fs/cgroup/memory/"${cgroup_path}"/"${containers[$i]}"/memory.qos_level) - fn_check_result "$cpu_qos" "$qos_level_offline" - fn_check_result "$mem_qos" "$qos_level_offline" - done -} - -function post_fun() { - docker rm -f "$rubik_id" - fn_check_result $? 0 - docker rm -f "${containers[@]}" - # Deleting multiple containers may time out. - [ "$?" -ne "0" ] && docker rm -f "${containers[@]}" - docker rm -f "$pod_id" - fn_check_result $? 0 - docker ps -a | grep -v "CONTAINER" - fn_check_result $? 1 "cleanup" - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_offline_0003.sh b/tests/src/test_rubik_offline_0003.sh deleted file mode 100755 index 277c839..0000000 --- a/tests/src/test_rubik_offline_0003.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: pod与container状态不一致测试.调整为离线 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -function test_fun() { - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cgroup_path="kubepods/${pod_id}" - qos_level=-1 - # generate json data - data=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level) - - # set container's qos to -1 - cpu_qos_path=/sys/fs/cgroup/cpu/${cgroup_path}/${container_id}/cpu.qos_level - mem_qos_path=/sys/fs/cgroup/memory/${cgroup_path}/${container_id}/memory.qos_level - echo -n ${qos_level} > "${cpu_qos_path}" - echo -n ${qos_level} > "${mem_qos_path}" - - # set pod's qos to -1 - result=$(rubik_qos "$data") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - - # check set ok - cpu_qos=$(cat "${cpu_qos_path}") - mem_qos=$(cat "${mem_qos_path}") - fn_check_result "${cpu_qos}" "${qos_level}" - fn_check_result "${mem_qos}" "${qos_level}" -} - -function post_fun() { - docker rm -f "$rubik_id" - fn_check_result $? 0 - docker rm -f "$container_id" - fn_check_result $? 0 - docker rm -f "$pod_id" - fn_check_result $? 0 - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_online_0001.sh b/tests/src/test_rubik_online_0001.sh deleted file mode 100755 index 2543f14..0000000 --- a/tests/src/test_rubik_online_0001.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: 不支持将离线业务转为在线业务 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -function test_fun() { - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cgroup_path="kubepods/${pod_id}" - qos_level_offline=-1 - qos_level_online=0 - # generate json data - data_offline=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level_offline) - data_online=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level_online) - - # set pod to offline - result=$(rubik_qos "$data_offline") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - - # set pod to online - result=$(rubik_qos "$data_online") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" -} - -function post_fun() { - docker rm -f "$rubik_id" - fn_check_result $? 0 - docker rm -f "$container_id" - fn_check_result $? 0 - docker rm -f "$pod_id" - fn_check_result $? 0 - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_cachelimit_abn.sh b/tests/src/test_rubik_reply_cachelimit_abn.sh deleted file mode 100755 index 90ea4c7..0000000 --- a/tests/src/test_rubik_reply_cachelimit_abn.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-05-19 -# Description: rubik cachelimit 0002 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - kernel_check CACHE - if [ $? -ne 0 ]; then - echo "Kernel not supported, skip test" - exit "${SKIP_FLAG}" - fi - run_rubik -} - -# pod not exist -function test_fun() { - local pod_name=podrubiktestpod - local cgroup_path=kubepods/podrubiktestpod - json_data=$(gen_single_pod_json ${pod_name} ${cgroup_path}) - result=$(rubik_qos "${json_data}") - if ! echo "$result" | grep "set qos failed"; then - ((exit_flag++)) - fi - rmdir /sys/fs/resctrl/rubik_* -} - -function post_fun() { - clean_all - if [[ $exit_flag -eq 0 ]]; then - echo "PASS" - else - echo "FAILED" - fi - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_cachelimit_fun.sh b/tests/src/test_rubik_reply_cachelimit_fun.sh deleted file mode 100755 index 8ea83c8..0000000 --- a/tests/src/test_rubik_reply_cachelimit_fun.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-05-19 -# Description: rubik cachelimit 0001 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - kernel_check CACHE - if [ $? -ne 0 ]; then - echo "Kernel not supported, skip test" - exit "${SKIP_FLAG}" - fi - rubik_id=$(run_rubik) - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") -} - -function test_fun() { - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}""${openEuler_image}" bash) - qos_level=-1 - data=$(gen_single_pod_json "${pod_id}" "${cgroup_path}" $qos_level) - - result=$(curl -s -H "Accept: application/json" -H "Content-type: application/json" -X POST --data '{"Pods": {"podrubiktestpod": {"CgroupPath": "kubepods/podrubiktestpod","QosLevel": -1,"CacheLimitLevel": "max"}}}' --unix-socket /run/rubik/rubik.sock http://localhost/) - if [[ $? -ne 0 ]]; then - ((exit_flag++)) - fi - cat /sys/fs/resctrl/rubik_max/tasks | grep $$ - if [[ $? -ne 0 ]]; then - ((exit_flag++)) - fi -} - -function post_fun() { - echo $$ > /sys/fs/cgroup/cpu/cgroup.procs - echo $$ > /sys/fs/resctrl/tasks - rmdir /sys/fs/cgroup/cpu/"${cg}" - rmdir /sys/fs/cgroup/memory/"${cg}" - rmdir /sys/fs/resctrl/rubik_* - clean_all - if [[ $exit_flag -eq 0 ]]; then - echo "PASS" - else - echo "FAILED" - fi - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_healthcheck_0001.sh b/tests/src/test_rubik_reply_healthcheck_0001.sh deleted file mode 100755 index 0e4aa77..0000000 --- a/tests/src/test_rubik_reply_healthcheck_0001.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-15 -# Description: rubik reply healthcheck 0001 - -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -pre_fun() { - run_rubik -} - -test_fun() { - result=$(rubik_ping) - if [[ $? -eq 0 ]] && [[ ${result} =~ "ok" ]]; then - echo "PASS" - else - echo "FAILED" - ((exit_flag++)) - fi -} - -post_fun() { - clean_all - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_healthcheck_0002.sh b/tests/src/test_rubik_reply_healthcheck_0002.sh deleted file mode 100755 index 7ca6698..0000000 --- a/tests/src/test_rubik_reply_healthcheck_0002.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-15 -# Description: rubik reply healthcheck 0002 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -pre_fun() { - # empty, so no rubik working - continue -} - -test_fun() { - result=$(rubik_ping) - if [[ $? -ne 0 ]]; then - echo "PASS" - else - echo "FAILED" - ((exit_flag++)) - fi -} - -post_fun() { - clean_all - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_http_parameter_0001.sh b/tests/src/test_rubik_reply_http_parameter_0001.sh deleted file mode 100755 index e7c50f1..0000000 --- a/tests/src/test_rubik_reply_http_parameter_0001.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: http接口有效参数测试 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -test_fun() { - # generate json data - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cgroup_path="kubepods/${pod_id}" - qos_level=-1 - data=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level) - - # check http://localhost/ - result=$(rubik_qos "$data") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - - # check http://localhost/ping - result=$(rubik_ping) - fn_check_result $? 0 - fn_check_string_contain "ok" "$result" - - # check http://localhost/version - result=$(rubik_version) - fn_check_result $? 0 - fn_check_string_contain "Version" "$result" -} - -post_fun() { - clean_all - docker rm -f "${container_id}" - fn_check_result $? 0 - docker rm -f "${pod_id}" - fn_check_result $? 0 - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_http_parameter_0002.sh b/tests/src/test_rubik_reply_http_parameter_0002.sh deleted file mode 100755 index 5562fd6..0000000 --- a/tests/src/test_rubik_reply_http_parameter_0002.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: http接口无效参数测试 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -function test_fun() { - # generate json data - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cgroup_path="kubepods/${pod_id}" - qos_level=-1 - data=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level) - - # construct wrong handlers - super_long_handler="http://longlonglong/$(long_char 1000)" - https_handler="https://localhost/" - special_char_handler="!@#$!@#!@#" - invalid_handler="http://localhost/not_exist" - debug_handler="http://localhost/debug/pprof/" - profile_handler="http://localhost/debug/pprof/profile" - - result=$(curl_cmd "$super_long_handler" "$data" "POST") - fn_check_string_contain "404 page not found" "$result" - - curl_cmd "$https_handler" "$data" "POST" - fn_check_result_noeq $? 0 - - curl_cmd "$special_char_handler" "$data" "POST" - fn_check_result_noeq $? 0 - - result=$(curl_cmd "$invalid_handler" "$data" "POST") - fn_check_string_contain "404 page not found" "$result" - - result=$(curl_cmd "$debug_handler" "$data" "POST") - fn_check_string_contain "404 page not found" "$result" - - result=$(curl_cmd "$profile_handler" "$data" "POST") - fn_check_string_contain "404 page not found" "$result" -} - -function post_fun() { - docker rm -f "$rubik_id" - fn_check_result $? 0 - docker rm -f "$container_id" - fn_check_result $? 0 - docker rm -f "$pod_id" - fn_check_result $? 0 - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_http_parameter_0003.sh b/tests/src/test_rubik_reply_http_parameter_0003.sh deleted file mode 100755 index 2ba6abd..0000000 --- a/tests/src/test_rubik_reply_http_parameter_0003.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2022-06-28 -# Description: http接口无效参数测试 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -function pre_fun() { - rubik_id=$(run_rubik) - fn_check_result $? 0 - logfile=${RUBIK_TEST_ROOT}/rubik_"$rubik_id"_log - docker logs -f "$rubik_id" > "$logfile" & - pod_id=$(docker run -tid --cgroup-parent /kubepods "${PAUSE_IMAGE}") - fn_check_result $? 0 -} - -function test_fun() { - # generate validate json data - container_id=$(docker run -tid --cgroup-parent /kubepods/"${pod_id}" "${OPENEULER_IMAGE}" bash) - cgroup_path="kubepods/${pod_id}" - qos_level=-1 - validate_data=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level) - - # generate invalid data - # case1: pod id not exist - pod_id_not_exist=$(cat /proc/sys/kernel/random/uuid) - pod_id_not_exist_cgroup="kubepods/${pod_id_not_exist}" - invalid_data1=$(gen_single_pod_json "$pod_id_not_exist" "$pod_id_not_exist_cgroup" $qos_level) - result=$(rubik_qos "$invalid_data1") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" - grep "set qos level error" "$logfile" - fn_check_result $? 0 - echo > "$logfile" - - # case2: cgroup path not exist - cgroup_path_not_exist="kubepods/cgroup/path/not/exist" - invalid_data2=$(gen_single_pod_json "$pod_id" "$cgroup_path_not_exist" $qos_level) - result=$(rubik_qos "$invalid_data2") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" - grep "set qos level error" "$logfile" - fn_check_result $? 0 - echo > "$logfile" - - # case3: super long cgroup path - cgroup_path_super_long="kubepods/$(long_char 10000)" - invalid_data3=$(gen_single_pod_json "$pod_id" "$cgroup_path_super_long" $qos_level) - result=$(rubik_qos "$invalid_data3") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" - grep -i "length of cgroup path exceeds max limit 4096" "$logfile" - fn_check_result $? 0 - echo > "$logfile" - - # case4: invalid qos level - qos_level_invalid=-999 - invalid_data4=$(gen_single_pod_json "$pod_id" "$cgroup_path" $qos_level_invalid) - result=$(rubik_qos "$invalid_data4") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" - grep -i "Invalid qos level number" "$logfile" - fn_check_result $? 0 - echo > "$logfile" - - # generate invalid data - # case5: pod id empty - pid_id_empty="" - pod_id_empty_cgroup="kubepods/${pod_id_empty}" - invalid_data5=$(gen_single_pod_json "$pod_id_empty" "$pod_id_empty_cgroup" $qos_level) - result=$(rubik_qos "$invalid_data5") - fn_check_result $? 0 - fn_check_string_contain "set qos failed" "$result" - grep "invalid cgroup path" "$logfile" - fn_check_result $? 0 - echo > "$logfile" - - # rubik will success with validate data - result=$(rubik_qos "$validate_data") - fn_check_result $? 0 - fn_check_string_not_contain "set qos failed" "$result" - echo > "$logfile" -} - -function post_fun() { - clean_all - docker rm -f "$container_id" - fn_check_result $? 0 - docker rm -f "$pod_id" - fn_check_result $? 0 - rm -f "$logfile" - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/src/test_rubik_reply_version_0001.sh b/tests/src/test_rubik_reply_version_0001.sh deleted file mode 100755 index 4c96a89..0000000 --- a/tests/src/test_rubik_reply_version_0001.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-05-15 -# Description: rubik reply healthcheck 0001 - -set -x -top_dir=$(git rev-parse --show-toplevel) -source "$top_dir"/tests/lib/commonlib.sh - -pre_fun() { - prepare_rubik - run_rubik -} - -test_fun() { - result=$(rubik_version) - if [[ $? -eq 0 ]]; then - field_number=$(echo "${result}" | grep -iE "version|release|commit|buildtime" -o | wc -l) - if [[ $field_number -eq 4 ]]; then - echo "PASS" - else - echo "FAILED" - ((exit_flag++)) - fi - else - echo "FAILED" - ((exit_flag++)) - fi -} - -post_fun() { - clean_all - exit "$exit_flag" -} - -pre_fun -test_fun -post_fun diff --git a/tests/test.sh b/tests/test.sh deleted file mode 100755 index f5ae6ca..0000000 --- a/tests/test.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -# rubik licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Create: 2021-04-15 -# Description: DT test script - -top_dir=$(git rev-parse --show-toplevel) - -# go fuzz test -function fuzz() { - failed=0 - while IFS= read -r testfile; do - printf "%-45s" "test $(basename "$testfile"): " | tee -a "${top_dir}"/tests/fuzz.log - bash "$testfile" "$1" | tee -a "${top_dir}"/tests/fuzz.log - if [ "$PIPESTATUS" -ne 0 ]; then - failed=1 - fi - # delete tmp files to avoid "no space left" problem - find /tmp -maxdepth 1 -iname "*fuzz*" -exec rm -rf {} \; - done < <(find "$top_dir"/tests/src -maxdepth 1 -name "fuzz_*.sh" -type f -print | sort) - exit $failed -} - -# integration test -function normal() { - source "${top_dir}"/tests/lib/commonlib.sh - failed=0 - while IFS= read -r testfile; do - filename=$(basename "$testfile") - DATE=$(date "+%Y%m%d%H%M%S") - export LOGFILE=${RUBIK_TEST_ROOT}/${filename}.${DATE}.log - printf "%-45s" "$filename: " - bash -x "$testfile" > "${LOGFILE}" 2>&1 - result=$? - if [ $result -eq "${SKIP_FLAG}" ]; then - echo -e "\033[33m SKIP \033[0m" - elif [ $result -ne 0 ]; then - echo -e "\033[31m FAIL \033[0m" - failed=1 - else - echo -e "\033[32m PASS \033[0m" - fi - done < <(find "$top_dir"/tests/src -maxdepth 1 -name "test_*" -type f -print | sort) - if [[ ${failed} -ne 0 ]]; then - exit $failed - else - clean_all - fi -} - -# main function to chose which kind of test -function main() { - case "$1" in - fuzz) - fuzz "$2" - ;; - *) - normal - ;; - esac -} - -main "$@" -- Gitee From 999e6a4339b3b540de2a52a28ebeb2be3125e9d3 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Sat, 3 Dec 2022 01:48:43 +0800 Subject: [PATCH 02/73] refactor: reconstruct directory hierarchy Signed-off-by: DCCooper <1866858@gmail.com> --- Makefile | 91 ++++++++ VERSION | 1 + hack/rubik-daemonset.yaml | 127 ++++++++++ hack/rubik.service | 26 +++ hack/static_check.sh | 55 +++++ hack/unit_test.sh | 36 +++ pkg/common/constant/constant.go | 113 +++++++++ pkg/common/tinylog/tinylog.go | 306 +++++++++++++++++++++++++ pkg/common/tinylog/tinylog_test.go | 273 ++++++++++++++++++++++ pkg/common/try/try.go | 127 ++++++++++ pkg/common/try/try_test.go | 43 ++++ pkg/common/typedef/convert.go | 60 +++++ pkg/common/typedef/convert_test.go | 156 +++++++++++++ pkg/common/typedef/types.go | 98 ++++++++ pkg/common/typedef/types_test.go | 142 ++++++++++++ pkg/common/util/file.go | 101 ++++++++ pkg/common/util/file_test.go | 169 ++++++++++++++ pkg/common/util/pod.go | 77 +++++++ pkg/common/util/pod_test.go | 145 ++++++++++++ pkg/modules/config/config.go | 171 ++++++++++++++ pkg/modules/podinformer/podinformer.go | 92 ++++++++ pkg/version/version.go | 49 ++++ 22 files changed, 2458 insertions(+) create mode 100644 Makefile create mode 100644 VERSION create mode 100644 hack/rubik-daemonset.yaml create mode 100644 hack/rubik.service create mode 100644 hack/static_check.sh create mode 100644 hack/unit_test.sh create mode 100644 pkg/common/constant/constant.go create mode 100644 pkg/common/tinylog/tinylog.go create mode 100644 pkg/common/tinylog/tinylog_test.go create mode 100644 pkg/common/try/try.go create mode 100644 pkg/common/try/try_test.go create mode 100644 pkg/common/typedef/convert.go create mode 100644 pkg/common/typedef/convert_test.go create mode 100644 pkg/common/typedef/types.go create mode 100644 pkg/common/typedef/types_test.go create mode 100644 pkg/common/util/file.go create mode 100644 pkg/common/util/file_test.go create mode 100644 pkg/common/util/pod.go create mode 100644 pkg/common/util/pod_test.go create mode 100644 pkg/modules/config/config.go create mode 100644 pkg/modules/podinformer/podinformer.go create mode 100644 pkg/version/version.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5312ed6 --- /dev/null +++ b/Makefile @@ -0,0 +1,91 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. +# rubik licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Xiang Li +# Create: 2021-04-17 +# Description: Makefile for rubik + +CWD=$(realpath .) +TMP_DIR := /tmp/rubik_tmpdir +INSTALL_DIR := /var/lib/rubik +BUILD_DIR=build +VERSION_FILE := ./VERSION +VERSION := $(shell awk -F"-" '{print $$1}' < $(VERSION_FILE)) +RELEASE :=$(if $(shell awk -F"-" '{print $$2}' < $(VERSION_FILE)),$(shell awk -F"-" '{print $$2}' < $(VERSION_FILE)),NA) +BUILD_TIME := $(shell date "+%Y-%m-%d") +GIT_COMMIT := $(if $(shell git rev-parse --short HEAD),$(shell git rev-parse --short HEAD),$(shell cat ./git-commit | head -c 7)) + +DEBUG_FLAGS := -gcflags="all=-N -l" +LD_FLAGS := -ldflags '-buildid=none -tmpdir=$(TMP_DIR) \ + -X isula.org/rubik/pkg/version.GitCommit=$(GIT_COMMIT) \ + -X isula.org/rubik/pkg/version.BuildTime=$(BUILD_TIME) \ + -X isula.org/rubik/pkg/version.Version=$(VERSION) \ + -X isula.org/rubik/pkg/version.Release=$(RELEASE) \ + -extldflags=-ftrapv \ + -extldflags=-Wl,-z,relro,-z,now -linkmode=external -extldflags=-static' + +GO_BUILD=CGO_ENABLED=1 \ + CGO_CFLAGS="-fstack-protector-strong -fPIE" \ + CGO_CPPFLAGS="-fstack-protector-strong -fPIE" \ + CGO_LDFLAGS_ALLOW='-Wl,-z,relro,-z,now' \ + CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ + go build -buildmode=pie + +all: release + +help: + @echo "Usage:" + @echo + @echo "make # build rubik with security build option" + @echo "make image # build container image" + @echo "make check # static check for latest commit" + @echo "make checkall # static check for whole project" + @echo "make tests # run all tests" + @echo "make test-unit # run unit test" + @echo "make cover # generate coverage report" + @echo "make install # install files to /var/lib/rubik" + +release: + mkdir -p $(TMP_DIR) $(BUILD_DIR) + rm -rf $(TMP_DIR) && mkdir -p $(ORG_PATH) $(TMP_DIR) + $(GO_BUILD) -o $(BUILD_DIR)/rubik $(LD_FLAGS) rubik.go + sed 's/__RUBIK_IMAGE__/rubik:$(VERSION)-$(RELEASE)/g' hack/rubik-daemonset.yaml > $(BUILD_DIR)/rubik-daemonset.yaml + cp hack/rubik.service $(BUILD_DIR) + +image: release + docker build -f Dockerfile -t rubik:$(VERSION)-$(RELEASE) . + +check: + @echo "Static check for last commit ..." + @./hack/static_check.sh last + @echo "Static check for last commit finished" + +checkall: + @echo "Static check for all ..." + @./hack/static_check.sh all + @echo "Static check for all finished" + +tests: test-unit test-integration + +test-unit: + @bash ./hack/unit_test.sh + +test-integration: + @bash ./tests/test.sh + +cover: + go test -p 1 -v ./... -coverprofile=cover.out + go tool cover -html=cover.out -o cover.html + python3 -m http.server 8080 + +install: + install -d -m 0750 $(INSTALL_DIR) + cp -f $(BUILD_DIR)/* $(INSTALL_DIR) + cp -f $(BUILD_DIR)/rubik.service /lib/systemd/system/ + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..ee1372d --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2.2 diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml new file mode 100644 index 0000000..df166d0 --- /dev/null +++ b/hack/rubik-daemonset.yaml @@ -0,0 +1,127 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rubik +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rubik +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rubik +subjects: + - kind: ServiceAccount + name: rubik + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rubik + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rubik-config + namespace: kube-system +data: + config.json: | + { + "autoCheck": false, + "logDriver": "stdio", + "logDir": "/var/log/rubik", + "logSize": 1024, + "logLevel": "info", + "cgroupRoot": "/sys/fs/cgroup", + "cacheConfig": { + "enable": false, + "defaultLimitMode": "static", + "adjustInterval": 1000, + "perfDuration": 1000, + "l3Percent": { + "low": 20, + "mid": 30, + "high": 50 + }, + "memBandPercent": { + "low": 10, + "mid": 30, + "high": 50 + } + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: rubik-agent + namespace: kube-system + labels: + k8s-app: rubik-agent +spec: + selector: + matchLabels: + name: rubik-agent + template: + metadata: + namespace: kube-system + labels: + name: rubik-agent + spec: + serviceAccountName: rubik + hostPID: true + containers: + - name: rubik-agent + image: rubik_image_name_and_tag + imagePullPolicy: IfNotPresent + env: + - name: RUBIK_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + capabilities: + add: + - SYS_ADMIN + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: rubiklog + mountPath: /var/log/rubik + readOnly: false + - name: runrubik + mountPath: /run/rubik + readOnly: false + - name: sysfs + mountPath: /sys/fs + readOnly: false + - name: config-volume + mountPath: /var/lib/rubik + terminationGracePeriodSeconds: 30 + volumes: + - name: rubiklog + hostPath: + path: /var/log/rubik + - name: runrubik + hostPath: + path: /run/rubik + - name: sysfs + hostPath: + path: /sys/fs + - name: config-volume + configMap: + name: rubik-config + items: + - key: config.json + path: config.json diff --git a/hack/rubik.service b/hack/rubik.service new file mode 100644 index 0000000..6a885cf --- /dev/null +++ b/hack/rubik.service @@ -0,0 +1,26 @@ +[Unit] +Description=Rubik multi-priority colocation engine +Documentation=https://gitee.com/openeuler/rubik +After=network-online.target firewalld.service +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/local/bin/rubik +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +StartLimitBurst=3 +StartLimitInterval=60s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +Delegate=yes +KillMode=process +OOMScoreAdjust=-500 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/hack/static_check.sh b/hack/static_check.sh new file mode 100644 index 0000000..0d15c2d --- /dev/null +++ b/hack/static_check.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. +# rubik licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2021-04-15 +# Description: shell script for static checking + +workspace=$(cd "$(dirname "$0")" && cd .. && pwd) +config_file=".golangci.yml" +check_type=$1 +export GO111MODULE=off + +# check binary file golangci-lint and it's config exist +function pre() { + # check golangci-lint exist + lint=$(command -v golangci-lint) > /dev/null 2>&1 + if [ -z "${lint}" ]; then + echo "Could not find binary golangci-lint" + exit 1 + fi + + # check config exist + config_path=${workspace}/${config_file} + if [[ ! -f ${config_path} ]]; then + echo "Could not find config file for golangci-lint" + exit 1 + fi +} + +# last: only do static check for the very latest commit +# all : do static check for the whole project +function run() { + case ${check_type} in + last) + # ${lint} run --modules-download-mode vendor + ${lint} run + ;; + all) + ${lint} run --new=false --new-from-rev=false + ;; + *) + return + ;; + esac +} + +pre +run diff --git a/hack/unit_test.sh b/hack/unit_test.sh new file mode 100644 index 0000000..50aee5f --- /dev/null +++ b/hack/unit_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +# rubik licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2021-04-15 +# Description: go test script + +export GO111MODULE=off +export GOPATH="${PWD}"/build + +mkdir -p "${GOPATH}"/src/isula.org +ln -sfn "${PWD}" build/src/isula.org/rubik +cd $GOPATH/src/isula.org/rubik + +test_log=unit_test.log +rm -rf "${test_log}" +touch "${test_log}" + +go_list=$(go list ./...) +for path in ${go_list}; do + echo "Start to test: ${path}" + go test -race -cover -count=1 -timeout 300s -v "${path}" >> "${test_log}" + cat "${test_log}" | grep -E -- "--- FAIL:|^FAIL" + if [ $? -eq 0 ]; then + echo "Testing failed... Please check ${test_log}" + exit 1 + fi + tail -n 1 "${test_log}" +done diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go new file mode 100644 index 0000000..a1d9476 --- /dev/null +++ b/pkg/common/constant/constant.go @@ -0,0 +1,113 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2021-04-17 +// Description: This file contains default constants used in the project + +// Package constant is for constant definition +package constant + +import ( + "errors" + "os" + "time" +) + +const ( + // RubikSock is path for rubik socket file + RubikSock = "/run/rubik/rubik.sock" + // ConfigFile is rubik config file + ConfigFile = "/var/lib/rubik/config.json" + // DefaultLogDir is default log dir + DefaultLogDir = "/var/log/rubik" + // LockFile is rubik lock file + LockFile = "/run/rubik/rubik.lock" + // ReadTimeout is timeout for http read + ReadTimeout = 60 * time.Second + // WriteTimeout is timeout for http write + WriteTimeout = 60 * time.Second + // DefaultSucceedCode is succeed code + DefaultSucceedCode = 0 + // DefaultCgroupRoot is mount point + DefaultCgroupRoot = "/sys/fs/cgroup" + // CPUCgroupFileName is name of cgroup file used for cpu qos level setting + CPUCgroupFileName = "cpu.qos_level" + // MemoryCgroupFileName is name of cgroup file used for memory qos level setting + MemoryCgroupFileName = "memory.qos_level" + // DefaultFileMode is file mode for cgroup files + DefaultFileMode os.FileMode = 0600 + // DefaultDirMode is dir default mode + DefaultDirMode os.FileMode = 0700 + // DefaultUmask is default umask + DefaultUmask = 0077 + // MaxCgroupPathLen is max cgroup path length for pod + MaxCgroupPathLen = 4096 + // MaxPodIDLen is max pod id length + MaxPodIDLen = 256 + // MaxPodsPerRequest is max pods number per http request + MaxPodsPerRequest = 100 + // TmpTestDir is tmp directory for test + TmpTestDir = "/tmp/rubik-test" + // TaskChanCapacity is capacity for task chan + TaskChanCapacity = 1024 + // WorkerNum is number of workers + WorkerNum = 1 + // KubepodsCgroup is kubepods root cgroup + KubepodsCgroup = "kubepods" + // PodCgroupNamePrefix is pod cgroup name prefix + PodCgroupNamePrefix = "pod" + // NodeNameEnvKey is node name environment variable key + NodeNameEnvKey = "RUBIK_NODE_NAME" + // PriorityAnnotationKey is annotation key to mark offline pod + PriorityAnnotationKey = "volcano.sh/preemptable" + // CacheLimitAnnotationKey is annotation key to set L3/Mb resctrl group + CacheLimitAnnotationKey = "volcano.sh/cache-limit" + // QuotaBurstAnnotationKey is annotation key to set cpu.cfs_burst_ns + QuotaBurstAnnotationKey = "volcano.sh/quota-burst-time" + // BlkioKey is annotation key to set blkio limit + BlkioKey = "volcano.sh/blkio-limit" + // DefaultMemCheckInterval indicates the default memory check interval 5s. + DefaultMemCheckInterval = 5 + // DefaultMaxMemCheckInterval indicates the default max memory check interval 30s. + DefaultMaxMemCheckInterval = 30 + // DefaultMemStrategy indicates the default memory strategy. + DefaultMemStrategy = "none" +) + +// LevelType is type definition of qos level +type LevelType int32 + +const ( + // MinLevel is min level for qos level + MinLevel LevelType = -1 + // MaxLevel is max level for qos level + MaxLevel LevelType = 0 +) + +// Int is type casting for type LevelType +func (l LevelType) Int() int { + return int(l) +} + +const ( + // ErrCodeFailed for normal failed + ErrCodeFailed = 1 +) + +// error define ref from src/internal/oserror/errors.go +var ( + // ErrFileTooBig file too big + ErrFileTooBig = errors.New("file too big") +) + +const ( + // InvalidBurst for invalid quota burst + InvalidBurst = -1 +) diff --git a/pkg/common/tinylog/tinylog.go b/pkg/common/tinylog/tinylog.go new file mode 100644 index 0000000..57de2da --- /dev/null +++ b/pkg/common/tinylog/tinylog.go @@ -0,0 +1,306 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Haomin Tsai +// Create: 2021-09-28 +// Description: This file is used for rubik log + +// Package tinylog is for rubik log +package tinylog + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "isula.org/rubik/pkg/common/constant" +) + +// CtxKey used for UUID +type CtxKey string + +const ( + // UUID is log uuid + UUID = "uuid" + + logStdio = 0 + logFile = 1 + logDriverStdio = "stdio" + logDriverFile = "file" + + logDebug = 0 + logInfo = 1 + logError = 2 + logStack = 20 + logStackFrom = 2 + logLevelInfo = "info" + logLevelStack = "stack" + + logFileNum = 10 + logSizeMin int64 = 10 // 10MB + logSizeMax int64 = 1024 * 1024 // 1TB + unitMB int64 = 1024 * 1024 +) + +var ( + logDriver = logStdio + logFname = filepath.Join(constant.DefaultLogDir, "rubik.log") + logLevel = 0 + logSize int64 = 1024 + logFileMaxSize int64 + logFileSize int64 + + lock = sync.Mutex{} +) + +func makeLogDir(logDir string) error { + if !filepath.IsAbs(logDir) { + return fmt.Errorf("log-dir %v must be an absolute path", logDir) + } + + if err := os.MkdirAll(logDir, constant.DefaultDirMode); err != nil { + return fmt.Errorf("create log directory %v failed", logDir) + } + + return nil +} + +// InitConfig init log config +func InitConfig(driver, logdir, level string, size int64) error { + if driver == "" { + driver = logDriverStdio + } + if driver != logDriverStdio && driver != logDriverFile { + return fmt.Errorf("invalid log driver %s", driver) + } + logDriver = logStdio + if driver == logDriverFile { + logDriver = logFile + } + + if level == "" { + level = logLevelInfo + } + levelstr, err := logLevelFromString(level) + if err != nil { + return err + } + logLevel = levelstr + + if size < logSizeMin || size > logSizeMax { + return fmt.Errorf("invalid log size %d", size) + } + logSize = size + logFileMaxSize = logSize / logFileNum + + if driver == "file" { + if err := makeLogDir(logdir); err != nil { + return err + } + logFname = filepath.Join(logdir, "rubik.log") + if f, err := os.Stat(logFname); err == nil { + atomic.StoreInt64(&logFileSize, f.Size()) + } + } + + return nil +} + +// DropError drop unused error +func DropError(args ...interface{}) { + argn := len(args) + if argn == 0 { + return + } + arg := args[argn-1] + if arg != nil { + fmt.Printf("drop error: %v\n", arg) + } +} + +func logLevelToString(level int) string { + switch level { + case logDebug: + return "debug" + case logInfo: + return "info" + case logError: + return "error" + case logStack: + return logLevelStack + default: + return "" + } +} + +func logLevelFromString(level string) (int, error) { + switch level { + case "debug": + return logDebug, nil + case "info", "": + return logInfo, nil + case "error": + return logError, nil + default: + return logInfo, fmt.Errorf("invalid log level %s", level) + } +} + +func logRename() { + for i := logFileNum - 1; i > 1; i-- { + old := logFname + fmt.Sprintf(".%d", i-1) + new := logFname + fmt.Sprintf(".%d", i) + if _, err := os.Stat(old); err == nil { + DropError(os.Rename(old, new)) + } + } + DropError(os.Rename(logFname, logFname+".1")) +} + +func logRotate(line int64) string { + if atomic.AddInt64(&logFileSize, line) > logFileMaxSize*unitMB { + logRename() + atomic.StoreInt64(&logFileSize, line) + } + + return logFname +} + +func writeLine(line string) { + if logDriver == logStdio { + fmt.Printf("%s", line) + return + } + + lock.Lock() + defer lock.Unlock() + + f, err := os.OpenFile(logRotate(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) + if err != nil { + return + } + + DropError(f.WriteString(line)) + DropError(f.Close()) +} + +func logf(level string, format string, args ...interface{}) { + tag := fmt.Sprintf("%s [rubik] level=%s ", time.Now().Format("2006-01-02 15:04:05.000"), level) + raw := fmt.Sprintf(format, args...) + "\n" + + depth := 1 + if level == logLevelStack { + depth = logStack + } + + for i := logStackFrom; i < logStackFrom+depth; i++ { + line := tag + raw + pc, file, linum, ok := runtime.Caller(i) + if ok { + fs := strings.Split(runtime.FuncForPC(pc).Name(), "/") + fs = strings.Split("."+fs[len(fs)-1], ".") + fn := fs[len(fs)-1] + line = tag + fmt.Sprintf("%s:%d:%s() ", file, linum, fn) + raw + } else if level == logLevelStack { + break + } + writeLine(line) + } +} + +// Logf log info level +func Logf(format string, args ...interface{}) { + if logInfo >= logLevel { + logf(logLevelToString(logInfo), format, args...) + } +} + +// Infof log info level +func Infof(format string, args ...interface{}) { + if logInfo >= logLevel { + logf(logLevelToString(logInfo), format, args...) + } +} + +// Debugf log debug level +func Debugf(format string, args ...interface{}) { + if logDebug >= logLevel { + logf(logLevelToString(logDebug), format, args...) + } +} + +// Errorf log error level +func Errorf(format string, args ...interface{}) { + if logError >= logLevel { + logf(logLevelToString(logError), format, args...) + } +} + +// Stackf log stack dump +func Stackf(format string, args ...interface{}) { + logf("stack", format, args...) +} + +// Entry is log entry +type Entry struct { + Ctx context.Context +} + +// WithCtx create entry with ctx +func WithCtx(ctx context.Context) *Entry { + return &Entry{ + Ctx: ctx, + } +} + +func (e *Entry) level(l int) string { + uuid, ok := e.Ctx.Value(CtxKey(UUID)).(string) + if ok { + return logLevelToString(l) + " UUID=" + uuid + } + return logLevelToString(l) +} + +// Logf write logs +func (e *Entry) Logf(f string, args ...interface{}) { + if logInfo < logLevel { + return + } + logf(e.level(logInfo), f, args...) +} + +// Infof write logs +func (e *Entry) Infof(f string, args ...interface{}) { + if logInfo < logLevel { + return + } + logf(e.level(logInfo), f, args...) +} + +// Debugf write verbose logs +func (e *Entry) Debugf(f string, args ...interface{}) { + if logDebug < logLevel { + return + } + logf(e.level(logDebug), f, args...) +} + +// Errorf write error logs +func (e *Entry) Errorf(f string, args ...interface{}) { + if logError < logLevel { + return + } + logf(e.level(logError), f, args...) +} diff --git a/pkg/common/tinylog/tinylog_test.go b/pkg/common/tinylog/tinylog_test.go new file mode 100644 index 0000000..f5c3719 --- /dev/null +++ b/pkg/common/tinylog/tinylog_test.go @@ -0,0 +1,273 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Danni Xia +// Create: 2021-05-24 +// Description: This file is used for testing tinylog + +package tinylog + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/try" +) + +// test_rubik_set_logdriver_0001 +func TestInitConfigLogDriver(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + // case: rubik.log already exist. + try.WriteFile(logFilePath, []byte(""), constant.DefaultFileMode) + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + + err = os.RemoveAll(logDir) + assert.NoError(t, err) + + // logDriver is file + err = InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, logFile, logDriver) + logString := "Test InitConfig with logDriver file" + Logf(logString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, true, strings.Contains(string(b), logString)) + + // logDriver is stdio + os.Remove(logFilePath) + err = InitConfig("stdio", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, logStdio, logDriver) + logString = "Test InitConfig with logDriver stdio" + Logf(logString) + b, err = ioutil.ReadFile(logFilePath) + assert.Equal(t, true, err != nil) + + // logDriver invalid + err = InitConfig("std", logDir, "", logSize) + assert.Equal(t, true, err != nil) + + // logDriver is null + err = InitConfig("", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, logStdio, logDriver) +} + +// test_rubik_set_logdir_0001 +func TestInitConfigLogDir(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + // LogDir valid + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + logString := "Test InitConfig with logDir valid" + Logf(logString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, true, strings.Contains(string(b), logString)) + + // logDir invalid + err = InitConfig("file", "invalid/log", "", logSize) + assert.Equal(t, true, err != nil) +} + +type logTC struct { + name, logLevel string + wantErr, debug, info, error bool +} + +func createLogTC() []logTC { + return []logTC{ + { + name: "TC1-logLevel debug", + logLevel: "debug", + wantErr: false, + debug: true, + info: true, + error: true, + }, + { + name: "TC2-logLevel info", + logLevel: "info", + wantErr: false, + debug: false, + info: true, + error: true, + }, + { + name: "TC3-logLevel error", + logLevel: "error", + wantErr: false, + debug: false, + info: false, + error: true, + }, + { + name: "TC4-logLevel null", + logLevel: "", + wantErr: false, + debug: false, + info: true, + error: true, + }, + { + name: "TC5-logLevel invalid", + logLevel: "inf", + wantErr: true, + }, + } +} + +// test_rubik_set_loglevel_0001 +func TestInitConfigLogLevel(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + debugLogSting, infoLogSting, errorLogSting, logLogString := "Test InitConfig debug log", + "Test InitConfig info log", "Test InitConfig error log", "Test InitConfig log log" + for _, tt := range createLogTC() { + t.Run(tt.name, func(t *testing.T) { + err := InitConfig("file", logDir, tt.logLevel, logSize) + if (err != nil) != tt.wantErr { + t.Errorf("InitConfig() = %v, want %v", err, tt.wantErr) + } else if tt.wantErr == false { + Debugf(debugLogSting) + Infof(infoLogSting) + Errorf(errorLogSting) + Logf(logLogString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) + assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) + os.Remove(logFilePath) + + ctx := context.WithValue(context.Background(), CtxKey(UUID), "abc123") + WithCtx(ctx).Debugf(debugLogSting) + WithCtx(ctx).Infof(infoLogSting) + WithCtx(ctx).Errorf(errorLogSting) + WithCtx(ctx).Logf(logLogString) + b, err = ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) + assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) + assert.Equal(t, true, strings.Contains(string(b), "abc123")) + err = os.RemoveAll(logDir) + assert.NoError(t, err) + } + }) + } +} + +// test_rubik_set_logsize_0001 +func TestInitConfigLogSize(t *testing.T) { + logDir := try.GenTestDir().String() + // LogSize invalid + err := InitConfig("file", logDir, "", logSizeMin-1) + assert.Equal(t, true, err != nil) + err = InitConfig("file", logDir, "", logSizeMax+1) + assert.Equal(t, true, err != nil) + + // logSize valid + testSize, printLine, repeat := 100, 50000, 100 + err = InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + for i := 0; i < printLine; i++ { + Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) + } + err = InitConfig("file", logDir, "", int64(testSize)) + assert.NoError(t, err) + for i := 0; i < printLine; i++ { + Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) + } + var size int64 + err = filepath.Walk(logDir, func(_ string, f os.FileInfo, _ error) error { + size += f.Size() + return nil + }) + assert.NoError(t, err) + assert.Equal(t, true, size < int64(testSize)*unitMB) + err = os.RemoveAll(constant.TmpTestDir) + assert.NoError(t, err) +} + +// TestLogStack is Stackf function test +func TestLogStack(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + Stackf("test stack log") + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + fmt.Println(string(b)) + assert.Equal(t, true, strings.Contains(string(b), t.Name())) + line := strings.Split(string(b), "\n") + maxLineNum := 5 + assert.Equal(t, true, len(line) < maxLineNum) +} + +// TestDropError is DropError function test +func TestDropError(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + DropError() + dropError := "test drop error" + DropError(dropError) + DropError(nil) + _, err = ioutil.ReadFile(logFilePath) + assert.Equal(t, true, err != nil) +} + +// TestLogOthers is log other tests +func TestLogOthers(t *testing.T) { + logDir := filepath.Join(try.GenTestDir().String(), "regular-file") + try.WriteFile(logDir, []byte{}, constant.DefaultFileMode) + + err := makeLogDir(logDir) + assert.Equal(t, true, err != nil) + + level1 := 3 + s := logLevelToString(level1) + assert.Equal(t, "", s) + level2 := 20 + s = logLevelToString(level2) + assert.Equal(t, "stack", s) + + logDriver = 1 + logFname = filepath.Join(constant.TmpTestDir, "log-not-exist") + os.MkdirAll(logFname, constant.DefaultDirMode) + writeLine("abc") + + s = WithCtx(context.Background()).level(1) + assert.Equal(t, "info", s) + + logLevel = logError + 1 + WithCtx(context.Background()).Errorf("abc") +} diff --git a/pkg/common/try/try.go b/pkg/common/try/try.go new file mode 100644 index 0000000..5f84dfa --- /dev/null +++ b/pkg/common/try/try.go @@ -0,0 +1,127 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: jingrui +// Create: 2022-04-17 +// Description: try provide some helper functions for unit-test. +// +// Package try provide some helper function for unit-test, if you want +// to use try outside unit-test, please add notes. +// +// 1. Try some function quiet|log|die on error. because some test +// function does not care the error returned, but the code checker +// always generate unuseful warnings. This method can suppress the +// noisy warnings. +// +// 2. Provide testdir helper to generate tmpdir for unitest. +// +package try + +import ( + "fmt" + "io/ioutil" + "os" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/google/uuid" + + "isula.org/rubik/pkg/common/constant" +) + +// Ret provide some action for error. +type Ret struct { + val interface{} + err error +} + +func newRet(err error) Ret { + return Ret{ + val: nil, + err: err, + } +} + +// OrDie try func or die with fail. +func (r Ret) OrDie() { + if r.err == nil { + return + } + fmt.Printf("try failed, die with error %v", r.err) + os.Exit(-1) +} + +// ErrMessage get ret error string format +func (r Ret) ErrMessage() string { + if r.err == nil { + return "" + } + return fmt.Sprintf("%v", r.err) +} + +// String get ret val and convert to string +func (r Ret) String() string { + val, ok := r.val.(string) + if ok { + return val + } + return "" +} + +// SecureJoin wrap error to Ret. +func SecureJoin(root, unsafe string) Ret { + name, err := securejoin.SecureJoin(root, unsafe) + ret := newRet(err) + if err == nil { + ret.val = name + } + return ret +} + +// MkdirAll wrap error to Ret. +func MkdirAll(path string, perm os.FileMode) Ret { + if err := os.MkdirAll(path, perm); err != nil { + return newRet(err) + } + return newRet(nil) +} + +// RemoveAll wrap error to Ret. +func RemoveAll(path string) Ret { + if err := os.RemoveAll(path); err != nil { + return newRet(err) + } + return newRet(nil) +} + +// WriteFile wrap error to Ret. +func WriteFile(filename string, data []byte, perm os.FileMode) Ret { + ret := newRet(nil) + ret.val = filename + if err := ioutil.WriteFile(filename, data, perm); err != nil { + ret.err = err + } + return ret +} + +const ( + testdir = "/tmp/rubik-test" +) + +// GenTestDir gen testdir +func GenTestDir() Ret { + name := fmt.Sprintf("%s/%s", testdir, uuid.New().String()) + ret := MkdirAll(name, constant.DefaultDirMode) + ret.val = name + return ret +} + +// DelTestDir del testdir, this function only need call once. +func DelTestDir() Ret { + return RemoveAll(testdir) +} diff --git a/pkg/common/try/try_test.go b/pkg/common/try/try_test.go new file mode 100644 index 0000000..9fbbc33 --- /dev/null +++ b/pkg/common/try/try_test.go @@ -0,0 +1,43 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: jingrui +// Create: 2022-04-17 +// Description: try provide some helper functions for unit-test. +// +// Package try provide some helper function for unit-test, if you want +// to use try outside unit-test, please add notes. + +package try + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" +) + +// Test_OrDie test try some-func or die. +func Test_OrDie(t *testing.T) { + ret := GenTestDir() + ret.OrDie() + dname := ret.String() + WriteFile(SecureJoin(dname, "die.txt").String(), []byte("ok"), constant.DefaultFileMode).OrDie() + RemoveAll(dname).OrDie() +} + +// Test_ErrMessage test try some-func or check the error. +func Test_ErrMessage(t *testing.T) { + ret := GenTestDir() + assert.Equal(t, ret.ErrMessage(), "") + dname := ret.String() + WriteFile(SecureJoin(dname, "log.txt").String(), []byte("ok"), constant.DefaultFileMode).ErrMessage() + assert.Equal(t, RemoveAll(dname).ErrMessage(), "") +} diff --git a/pkg/common/typedef/convert.go b/pkg/common/typedef/convert.go new file mode 100644 index 0000000..77af888 --- /dev/null +++ b/pkg/common/typedef/convert.go @@ -0,0 +1,60 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2022-03-25 +// Description: This package stores basic type conversion functions. + +package typedef + +import ( + "crypto/rand" + "math/big" + "strconv" +) + +// FormatInt64 convert the int 64 type to a string +func FormatInt64(n int64) string { + const base = 10 + return strconv.FormatInt(n, base) +} + +// ParseInt64 convert the string type to Int64 +func ParseInt64(str string) (int64, error) { + const ( + base = 10 + bitSize = 64 + ) + return strconv.ParseInt(str, base, bitSize) +} + +// ParseFloat64 convert the string type to Float64 +func ParseFloat64(str string) (float64, error) { + const bitSize = 64 + return strconv.ParseFloat(str, bitSize) +} + +// FormatFloat64 convert the Float64 type to string +func FormatFloat64(f float64) string { + const ( + precision = -1 + bitSize = 64 + format = 'f' + ) + return strconv.FormatFloat(f, format, precision, bitSize) +} + +// RandInt provide safe rand int in range [0, max) +func RandInt(max int) int { + n, err := rand.Int(rand.Reader, big.NewInt(int64(max))) + if err != nil { + return 0 + } + return int(n.Int64()) +} diff --git a/pkg/common/typedef/convert_test.go b/pkg/common/typedef/convert_test.go new file mode 100644 index 0000000..35cc50c --- /dev/null +++ b/pkg/common/typedef/convert_test.go @@ -0,0 +1,156 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2022-07-10 +// Description: This package stores basic type conversion functions. + +// Package typedef provides a common type conversion method. +package typedef + +import ( + "math" + "testing" +) + +// TestFormatInt64 is testcase for FormatInt64 +func TestFormatInt64(t *testing.T) { + type args struct { + n int64 + } + validNum := 100 + tests := []struct { + name string + args args + want string + }{ + { + name: "TC-convert the int 64 to string", + args: args{n: int64(validNum)}, + want: "100", + }, + { + name: "TC-convert the big int", + args: args{math.MaxInt64}, + want: "9223372036854775807", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FormatInt64(tt.args.n); got != tt.want { + t.Errorf("FormatInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestParseInt64 is testcase for ParseInt64 +func TestParseInt64(t *testing.T) { + type args struct { + str string + } + validNum := 100 + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + { + name: "TC-convert the int 64 to string", + args: args{str: "100"}, + want: int64(validNum), + }, + { + name: "TC-convert the big int", + args: args{str: "9223372036854775807"}, + want: math.MaxInt64, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseInt64(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("ParseInt64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestFormatFloat64 is testcase for FormatFloat64 +func TestFormatFloat64(t *testing.T) { + type args struct { + f float64 + } + validNum := 100.0 + tests := []struct { + name string + args args + want string + }{ + { + name: "TC-convert the float64 to string", + args: args{f: validNum}, + want: "100", + }, + { + name: "TC-convert the big float", + args: args{math.MaxFloat64}, + want: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FormatFloat64(tt.args.f); got != tt.want { + t.Errorf("FormatFloat64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestParseFloat64 is testcase for ParseFloat64 +func TestParseFloat64(t *testing.T) { + type args struct { + str string + } + validNum := 100.0 + tests := []struct { + name string + args args + want float64 + wantErr bool + }{ + { + name: "TC-convert the string to float64", + args: args{str: "100"}, + want: validNum, + }, + { + name: "TC-convert the big float", + args: args{str: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + want: math.MaxFloat64, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseFloat64(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("ParseFloat64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseFloat64() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/common/typedef/types.go b/pkg/common/typedef/types.go new file mode 100644 index 0000000..a9a1e28 --- /dev/null +++ b/pkg/common/typedef/types.go @@ -0,0 +1,98 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jing Rui +// Create: 2021-04-27 +// Description: This file contains default constants used in the project + +// Package typedef is general used types. +package typedef + +import ( + "path/filepath" + + corev1 "k8s.io/api/core/v1" +) + +// ContainerInfo represent container +type ContainerInfo struct { + // Basic Information + Name string `json:"name"` + ID string `json:"id"` + PodID string `json:"podID"` + CgroupRoot string `json:"cgroupRoot"` + CgroupAddr string `json:"cgroupAddr"` +} + +// NewContainerInfo create container info +func NewContainerInfo(container corev1.Container, podID, conID, cgroupRoot, podCgroupPath string) *ContainerInfo { + c := ContainerInfo{ + Name: container.Name, + ID: conID, + PodID: podID, + CgroupRoot: cgroupRoot, + CgroupAddr: filepath.Join(podCgroupPath, conID), + } + return &c +} + +// CgroupPath return full cgroup path +func (ci *ContainerInfo) CgroupPath(subsys string) string { + if ci == nil || ci.Name == "" { + return "" + } + return filepath.Join(ci.CgroupRoot, subsys, ci.CgroupAddr) +} + +// Clone return deepcopy object. +func (ci *ContainerInfo) Clone() *ContainerInfo { + copy := *ci + return © +} + +// PodInfo represent pod +type PodInfo struct { + // Basic Information + Containers map[string]*ContainerInfo `json:"containers,omitempty"` + Name string `json:"name"` + UID string `json:"uid"` + CgroupPath string `json:"cgroupPath"` + Namespace string `json:"namespace"` + CgroupRoot string `json:"cgroupRoot"` + + // Service Information + Offline bool `json:"offline"` + CacheLimitLevel string `json:"cacheLimitLevel,omitempty"` + + // value of quota burst + QuotaBurst int64 `json:"quotaBurst"` +} + +// Clone return deepcopy object +func (pi *PodInfo) Clone() *PodInfo { + if pi == nil { + return nil + } + copy := *pi + // deepcopy reference object + copy.Containers = make(map[string]*ContainerInfo, len(pi.Containers)) + for _, c := range pi.Containers { + copy.Containers[c.Name] = c.Clone() + } + return © +} + +// AddContainerInfo store container info to checkpoint +func (pi *PodInfo) AddContainerInfo(containerInfo *ContainerInfo) { + // key should not be empty + if containerInfo.Name == "" { + return + } + pi.Containers[containerInfo.Name] = containerInfo +} diff --git a/pkg/common/typedef/types_test.go b/pkg/common/typedef/types_test.go new file mode 100644 index 0000000..ca5f28d --- /dev/null +++ b/pkg/common/typedef/types_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jing Rui +// Create: 2022-07-10 +// Description: This file contains default constants used in the project + +// Package typedef is general used types. +package typedef + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func init() { + err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) + if err != nil { + log.Fatalf("Failed to create tmp test dir for testing!") + } +} + +func genContainer() corev1.Container { + c := corev1.Container{} + c.Name = "testContainer" + c.Resources.Requests = make(corev1.ResourceList) + c.Resources.Limits = make(corev1.ResourceList) + c.Resources.Requests["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) + c.Resources.Limits["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) + c.Resources.Limits["memory"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) + + return c +} + +// TestNewContainerInfo is testcase for NewContainerInfo +func TestNewContainerInfo(t *testing.T) { + cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") + assert.NoError(t, err) + defer os.RemoveAll(cgRoot) + podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") + assert.NoError(t, err) + defer os.RemoveAll(cgRoot) + + c := genContainer() + type args struct { + container corev1.Container + podID string + conID string + cgroupRoot string + podCgroupPath string + } + tests := []struct { + want *ContainerInfo + name string + args args + }{ + { + name: "TC", + args: args{container: c, podID: "podID", cgroupRoot: cgRoot, conID: "cID", podCgroupPath: podCGPath}, + want: &ContainerInfo{ + Name: "testContainer", + ID: "cID", + PodID: "podID", + CgroupRoot: cgRoot, + CgroupAddr: filepath.Join(podCGPath, "cID"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewContainerInfo(tt.args.container, tt.args.podID, tt.args.conID, tt.args.cgroupRoot, tt.args.podCgroupPath); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewContainerInfo() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestContainerInfo_CgroupPath is testcase for ContainerInfo.CgroupPath +func TestContainerInfo_CgroupPath(t *testing.T) { + cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") + assert.NoError(t, err) + defer os.RemoveAll(cgRoot) + podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") + assert.NoError(t, err) + defer os.RemoveAll(podCGPath) + + emptyCi := &ContainerInfo{} + assert.Equal(t, "", emptyCi.CgroupPath("cpu")) + + ci := emptyCi.Clone() + + ci.Name = "testContainer" + ci.ID = "cID" + ci.PodID = "podID" + ci.CgroupRoot = cgRoot + ci.CgroupAddr = filepath.Join(podCGPath, "cID") + assert.Equal(t, ci.CgroupPath("cpu"), + filepath.Join(cgRoot, "cpu", filepath.Join(podCGPath, "cID"))) +} + +// TestPodInfo_Clone is testcase for PodInfo.Clone +func TestPodInfo_Clone(t *testing.T) { + cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") + assert.NoError(t, err) + defer os.RemoveAll(cgRoot) + podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") + assert.NoError(t, err) + defer os.RemoveAll(podCGPath) + emptyPI := &PodInfo{} + pi := emptyPI.Clone() + pi.Containers = make(map[string]*ContainerInfo) + pi.Name = "testPod" + pi.UID = "abcd" + pi.CgroupPath = cgRoot + + containerWithOutName := genContainer() + containerWithOutName.Name = "" + + emptyNameCI := NewContainerInfo(containerWithOutName, "testPod", "cID", cgRoot, podCGPath) + pi.AddContainerInfo(emptyNameCI) + assert.Equal(t, len(pi.Containers), 0) + + ci := NewContainerInfo(genContainer(), "testPod", "cID", cgRoot, podCGPath) + pi.AddContainerInfo(ci) + newPi := pi.Clone() + assert.Equal(t, len(newPi.Containers), 1) +} diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go new file mode 100644 index 0000000..15de5e6 --- /dev/null +++ b/pkg/common/util/file.go @@ -0,0 +1,101 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2021-04-17 +// Description: filepath related common functions + +package util + +import ( + "io/ioutil" + "os" + "path/filepath" + "syscall" + + "isula.org/rubik/pkg/common/constant" + log "isula.org/rubik/pkg/common/tinylog" +) + +const ( + fileMaxSize = 10 * 1024 * 1024 // 10MB +) + +// CreateFile create full path including dir and file. +func CreateFile(path string) error { + if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + + return f.Close() +} + +// IsDirectory returns true if the file exists and it is a dir +func IsDirectory(path string) bool { + fi, err := os.Lstat(path) + if err != nil { + return false + } + + return fi.IsDir() +} + +// ReadSmallFile read small file less than 10MB +func ReadSmallFile(path string) ([]byte, error) { + st, err := os.Lstat(path) + if err != nil { + return nil, err + } + if st.Size() > fileMaxSize { + return nil, constant.ErrFileTooBig + } + return ioutil.ReadFile(path) // nolint: gosec +} + +// PathExist returns true if the path exists +func PathExist(path string) bool { + if _, err := os.Lstat(path); err != nil { + return false + } + + return true +} + +// CreateLockFile creates a lock file +func CreateLockFile(p string) (*os.File, error) { + path := filepath.Clean(p) + if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { + return nil, err + } + + lock, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DefaultFileMode) + if err != nil { + return nil, err + } + + if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + log.DropError(lock.Close()) + return nil, err + } + + return lock, nil +} + +// RemoveLockFile removes lock file - this function used cleanup resource, +// errors will ignored to make sure more source is cleaned. +func RemoveLockFile(lock *os.File, path string) { + log.DropError(syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)) + log.DropError(lock.Close()) + log.DropError(os.Remove(path)) +} diff --git a/pkg/common/util/file_test.go b/pkg/common/util/file_test.go new file mode 100644 index 0000000..63fc637 --- /dev/null +++ b/pkg/common/util/file_test.go @@ -0,0 +1,169 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2021-04-17 +// Description: filepath related common functions testing + +package util + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" +) + +// TestIsDirectory is IsDirectory function test +func TestIsDirectory(t *testing.T) { + directory, err := ioutil.TempDir(constant.TmpTestDir, t.Name()) + assert.NoError(t, err) + defer os.RemoveAll(directory) + + filePath, err := ioutil.TempFile(directory, t.Name()) + assert.NoError(t, err) + + type args struct { + path string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "TC1-directory is exist", + args: args{path: directory}, + want: true, + }, + { + name: "TC2-directory is not exist", + args: args{path: "/directory/is/not/exist"}, + want: false, + }, + { + name: "TC3-test file path", + args: args{path: filePath.Name()}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsDirectory(tt.args.path); got != tt.want { + t.Errorf("IsDirectory() = %v, want %v", got, tt.want) + } + }) + } + err = filePath.Close() + assert.NoError(t, err) +} + +// TestPathIsExist is PathExist function test +func TestPathIsExist(t *testing.T) { + filePath, err := ioutil.TempDir(constant.TmpTestDir, "file_exist") + assert.NoError(t, err) + defer os.RemoveAll(filePath) + + type args struct { + path string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "TC1-path is exist", + args: args{path: filePath}, + want: true, + }, + { + name: "TC2-path is not exist", + args: args{path: "/path/is/not/exist"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PathExist(tt.args.path); got != tt.want { + t.Errorf("PathExist() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestReadSmallFile is test for read file +func TestReadSmallFile(t *testing.T) { + filePath, err := ioutil.TempDir(constant.TmpTestDir, "read_file") + assert.NoError(t, err) + defer os.RemoveAll(filePath) + + // case1: ok + err = ioutil.WriteFile(filepath.Join(filePath, "ok"), []byte{}, constant.DefaultFileMode) + assert.NoError(t, err) + _, err = ReadSmallFile(filepath.Join(filePath, "ok")) + assert.NoError(t, err) + + // case2: too big + size := 20000000 + big := make([]byte, size, size) + err = ioutil.WriteFile(filepath.Join(filePath, "big"), big, constant.DefaultFileMode) + assert.NoError(t, err) + _, err = ReadSmallFile(filepath.Join(filePath, "big")) + assert.Error(t, err) + + // case3: file not exist + _, err = ReadSmallFile(filepath.Join(filePath, "missing")) + assert.Error(t, err) +} + +// TestCreateLockFile is CreateLockFile function test +func TestCreateLockFile(t *testing.T) { + lockFile := filepath.Join(constant.TmpTestDir, "rubik.lock") + err := os.RemoveAll(lockFile) + assert.NoError(t, err) + + lock, err := CreateLockFile(lockFile) + assert.NoError(t, err) + RemoveLockFile(lock, lockFile) +} + +// TestLockFail is CreateLockFile fail test +func TestLockFail(t *testing.T) { + lockFile := filepath.Join(constant.TmpTestDir, "rubik.lock") + err := os.RemoveAll(constant.TmpTestDir) + assert.NoError(t, err) + os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) + + _, err = os.Create(filepath.Join(constant.TmpTestDir, "rubik-lock")) + assert.NoError(t, err) + _, err = CreateLockFile(filepath.Join(constant.TmpTestDir, "rubik-lock", "rubik.lock")) + assert.Equal(t, true, err != nil) + err = os.RemoveAll(filepath.Join(constant.TmpTestDir, "rubik-lock")) + assert.NoError(t, err) + + err = os.MkdirAll(lockFile, constant.DefaultDirMode) + assert.NoError(t, err) + _, err = CreateLockFile(lockFile) + assert.Equal(t, true, err != nil) + err = os.RemoveAll(lockFile) + assert.NoError(t, err) + + _, err = CreateLockFile(lockFile) + assert.NoError(t, err) + _, err = CreateLockFile(lockFile) + assert.Equal(t, true, err != nil) + err = os.RemoveAll(lockFile) + assert.NoError(t, err) +} diff --git a/pkg/common/util/pod.go b/pkg/common/util/pod.go new file mode 100644 index 0000000..a0e061e --- /dev/null +++ b/pkg/common/util/pod.go @@ -0,0 +1,77 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Danni Xia +// Create: 2022-05-25 +// Description: Pod related common functions + +package util + +import ( + "path/filepath" + "strings" + + corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/common/constant" + log "isula.org/rubik/pkg/common/tinylog" + "isula.org/rubik/pkg/common/typedef" +) + +const configHashAnnotationKey = "kubernetes.io/config.hash" + +// IsOffline judges whether pod is offline pod +func IsOffline(pod *corev1.Pod) bool { + return pod.Annotations[constant.PriorityAnnotationKey] == "true" +} + +func GetPodCacheLimit(pod *corev1.Pod) string { + return pod.Annotations[constant.CacheLimitAnnotationKey] +} + +// GetQuotaBurst checks CPU quota burst annotation value. +func GetQuotaBurst(pod *corev1.Pod) int64 { + quota := pod.Annotations[constant.QuotaBurstAnnotationKey] + if quota == "" { + return constant.InvalidBurst + } + + quotaBurst, err := typedef.ParseInt64(quota) + if err != nil { + log.Errorf("pod %s burst quota annotation value %v is invalid, expect integer", pod.Name, quotaBurst) + return constant.InvalidBurst + } + if quotaBurst < 0 { + log.Errorf("pod %s burst quota annotation value %v is invalid, expect positive", pod.Name, quotaBurst) + return constant.InvalidBurst + } + return quotaBurst +} + +// GetPodCgroupPath returns cgroup path of pod +func GetPodCgroupPath(pod *corev1.Pod) string { + var cgroupPath string + id := string(pod.UID) + if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { + id = configHash + } + + switch pod.Status.QOSClass { + case corev1.PodQOSGuaranteed: + cgroupPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBurstable: + cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), + constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBestEffort: + cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), + constant.PodCgroupNamePrefix+id) + } + + return cgroupPath +} diff --git a/pkg/common/util/pod_test.go b/pkg/common/util/pod_test.go new file mode 100644 index 0000000..bc66202 --- /dev/null +++ b/pkg/common/util/pod_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jingxiao Lu +// Create: 2022-05-25 +// Description: tests for pod.go + +package util + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/common/constant" +) + +const ( + trueStr = "true" +) + +func TestIsOffline(t *testing.T) { + var pod = &corev1.Pod{} + pod.Annotations = make(map[string]string) + pod.Annotations[constant.PriorityAnnotationKey] = trueStr + if !IsOffline(pod) { + t.Fatalf("%s failed for Annotations is %s", t.Name(), trueStr) + } + + delete(pod.Annotations, constant.PriorityAnnotationKey) + if IsOffline(pod) { + t.Fatalf("%s failed for Annotations no such key", t.Name()) + } +} + +// TestGetQuotaBurst is testcase for GetQuotaBurst +func TestGetQuotaBurst(t *testing.T) { + pod := &corev1.Pod{} + pod.Annotations = make(map[string]string) + maxInt64PlusOne := "9223372036854775808" + tests := []struct { + name string + quotaBurst string + want int64 + }{ + { + name: "TC1-valid quota burst", + quotaBurst: "1", + want: 1, + }, + { + name: "TC2-empty quota burst", + quotaBurst: "", + want: -1, + }, + { + name: "TC3-zero quota burst", + quotaBurst: "0", + want: 0, + }, + { + name: "TC4-negative quota burst", + quotaBurst: "-100", + want: -1, + }, + { + name: "TC5-float quota burst", + quotaBurst: "100.34", + want: -1, + }, + { + name: "TC6-nonnumerical quota burst", + quotaBurst: "nonnumerical", + want: -1, + }, + { + name: "TC7-exceed max int64", + quotaBurst: maxInt64PlusOne, + want: -1, + }, + } + for _, tt := range tests { + pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.quotaBurst + assert.Equal(t, GetQuotaBurst(pod), tt.want) + } +} + +func TestGetPodCgroupPath(t *testing.T) { + var pod = &corev1.Pod{} + pod.UID = "AAA" + var guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+string(pod.UID)) + var burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+string(pod.UID)) + var besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+string(pod.UID)) + pod.Annotations = make(map[string]string) + + // no pod.Annotations[configHashAnnotationKey] + pod.Status.QOSClass = corev1.PodQOSGuaranteed + if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { + t.Fatalf("%s failed for PodQOSGuaranteed without configHash", t.Name()) + } + pod.Status.QOSClass = corev1.PodQOSBurstable + if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { + t.Fatalf("%s failed for PodQOSBurstable without configHash", t.Name()) + } + pod.Status.QOSClass = corev1.PodQOSBestEffort + if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { + t.Fatalf("%s failed for PodQOSBestEffort without configHash", t.Name()) + } + pod.Status.QOSClass = "" + if !assert.Equal(t, GetPodCgroupPath(pod), "") { + t.Fatalf("%s failed for not setting QOSClass without configHash", t.Name()) + } + + // has pod.Annotations[configHashAnnotationKey] + pod.Annotations[configHashAnnotationKey] = "BBB" + var id = pod.Annotations[configHashAnnotationKey] + guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) + burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+id) + besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+id) + pod.Status.QOSClass = corev1.PodQOSGuaranteed + if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { + t.Fatalf("%s failed for PodQOSGuaranteed with configHash", t.Name()) + } + pod.Status.QOSClass = corev1.PodQOSBurstable + if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { + t.Fatalf("%s failed for PodQOSBurstable with configHash", t.Name()) + } + pod.Status.QOSClass = corev1.PodQOSBestEffort + if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { + t.Fatalf("%s failed for PodQOSBestEffort with configHash", t.Name()) + } + pod.Status.QOSClass = "" + if !assert.Equal(t, GetPodCgroupPath(pod), "") { + t.Fatalf("%s failed for not setting QOSClass with configHash", t.Name()) + } +} diff --git a/pkg/modules/config/config.go b/pkg/modules/config/config.go new file mode 100644 index 0000000..c1edc6c --- /dev/null +++ b/pkg/modules/config/config.go @@ -0,0 +1,171 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Danni Xia +// Create: 2021-04-26 +// Description: config load + +package config + +import ( + "bytes" + "encoding/json" + "path/filepath" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" +) + +var ( + // CgroupRoot is cgroup mount point + CgroupRoot = constant.DefaultCgroupRoot + // ShutdownFlag is rubik shutdown flag + ShutdownFlag int32 + // ShutdownChan is rubik shutdown channel + ShutdownChan = make(chan struct{}) +) + +// Config defines the configuration for rubik +type Config struct { + AutoCheck bool `json:"autoCheck,omitempty"` + LogDriver string `json:"logDriver,omitempty"` + LogDir string `json:"logDir,omitempty"` + LogSize int `json:"logSize,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + CgroupRoot string `json:"cgroupRoot,omitempty"` + CacheCfg CacheConfig `json:"cacheConfig,omitempty"` + BlkioCfg BlkioConfig `json:"blkioConfig,omitempty"` + MemCfg MemoryConfig `json:"memoryConfig,omitempty"` + NodeConfig []NodeConfig `json:"nodeConfig,omitempty"` +} + +// CacheConfig define cache limit related config +type CacheConfig struct { + Enable bool `json:"enable,omitempty"` + DefaultLimitMode string `json:"defaultLimitMode,omitempty"` + DefaultResctrlDir string `json:"-"` + AdjustInterval int `json:"adjustInterval,omitempty"` + PerfDuration int `json:"perfDuration,omitempty"` + L3Percent MultiLvlPercent `json:"l3Percent,omitempty"` + MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` +} + +// BlkioConfig defines blkio related configurations. +type BlkioConfig struct { + Enable bool `json:"enable,omitempty"` +} + +// MultiLvlPercent define multi level percentage +type MultiLvlPercent struct { + Low int `json:"low,omitempty"` + Mid int `json:"mid,omitempty"` + High int `json:"high,omitempty"` +} + +type MemoryConfig struct { + Enable bool `json:"enable,omitempty"` + Strategy string `json:"strategy,omitempty"` + CheckInterval int `json:"checkInterval,omitempty"` +} + +// NodeConfig define node configuration for each node +type NodeConfig struct { + NodeName string `json:"nodeName,omitempty"` + IOcostEnable bool `json:"iocostEnable,omitempty"` + IOcostConfig []IOcostConfig `json:"iocostConfig,omitempty"` +} + +// IOcostConfig define iocost for node +type IOcostConfig struct { + Dev string `json:"dev,omitempty"` + Enable bool `json:"enable,omitempty"` + Model string `json:"model,omitempty"` + Param Param `json:"param,omitempty"` +} + +// Param for linear model +type Param struct { + Rbps int64 `json:"rbps,omitempty"` + Rseqiops int64 `json:"rseqiops,omitempty"` + Rrandiops int64 `json:"rrandiops,omitempty"` + Wbps int64 `json:"wbps,omitempty"` + Wseqiops int64 `json:"wseqiops,omitempty"` + Wrandiops int64 `json:"wrandiops,omitempty"` +} + +// NewConfig returns new config load from config file +func NewConfig(path string) (*Config, error) { + if path == "" { + path = constant.ConfigFile + } + + defaultLogSize, defaultAdInt, defaultPerfDur := 1024, 1000, 1000 + defaultLowL3, defaultMidL3, defaultHighL3, defaultLowMB, defaultMidMB, defaultHighMB := 20, 30, 50, 10, 30, 50 + cfg := Config{ + LogDriver: "stdio", + LogDir: constant.DefaultLogDir, + LogSize: defaultLogSize, + LogLevel: "info", + CgroupRoot: constant.DefaultCgroupRoot, + CacheCfg: CacheConfig{ + Enable: false, + DefaultLimitMode: "static", + DefaultResctrlDir: "/sys/fs/resctrl", + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{ + Low: defaultLowL3, + Mid: defaultMidL3, + High: defaultHighL3, + }, + MemBandPercent: MultiLvlPercent{ + Low: defaultLowMB, + Mid: defaultMidMB, + High: defaultHighMB, + }, + }, + BlkioCfg: BlkioConfig{ + Enable: false, + }, + MemCfg: MemoryConfig{ + Enable: false, + Strategy: constant.DefaultMemStrategy, + CheckInterval: constant.DefaultMemCheckInterval, + }, + } + + defer func() { + CgroupRoot = cfg.CgroupRoot + }() + + if !util.PathExist(path) { + return &cfg, nil + } + + b, err := util.ReadSmallFile(filepath.Clean(path)) + if err != nil { + return nil, err + } + + reader := bytes.NewReader(b) + if err := json.NewDecoder(reader).Decode(&cfg); err != nil { + return nil, err + } + + return &cfg, nil +} + +// String return string format. +func (cfg *Config) String() string { + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "{}" + } + return string(data) +} diff --git a/pkg/modules/podinformer/podinformer.go b/pkg/modules/podinformer/podinformer.go new file mode 100644 index 0000000..90e1c20 --- /dev/null +++ b/pkg/modules/podinformer/podinformer.go @@ -0,0 +1,92 @@ +package podinformer + +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Danni Xia +// Create: 2021-07-22 +// Description: qos auto config + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "isula.org/rubik/pkg/modules/config" + log "isula.org/rubik/pkg/tools/tinylog" +) + +const invalidErr = "Auto config error: invalid pod type" + +// EventHandler is used to process pod events pushed by Kubernetes APIServer. +type EventHandler interface { + AddEvent(pod *corev1.Pod) + UpdateEvent(oldPod *corev1.Pod, newPod *corev1.Pod) + DeleteEvent(pod *corev1.Pod) +} + +// Backend is Rubik struct. +var Backend EventHandler + +// Init initializes the callback function for the pod event. +func Init(kubeClient *kubernetes.Clientset, nodeName string) error { + const ( + reSyncTime = 30 + specNodeNameField = "spec.nodeName" + ) + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, + time.Duration(reSyncTime)*time.Second, + informers.WithTweakListOptions(func(options *metav1.ListOptions) { + // set Options to return only pods on the current node. + options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, nodeName).String() + })) + kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: addHandler, + UpdateFunc: updateHandler, + DeleteFunc: deleteHandler, + }) + kubeInformerFactory.Start(config.ShutdownChan) + return nil +} + +func addHandler(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + log.Errorf(invalidErr) + return + } + + Backend.AddEvent(pod) +} + +func updateHandler(old, new interface{}) { + oldPod, ok1 := old.(*corev1.Pod) + newPod, ok2 := new.(*corev1.Pod) + if !ok1 || !ok2 { + log.Errorf(invalidErr) + return + } + + Backend.UpdateEvent(oldPod, newPod) +} + +func deleteHandler(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + log.Errorf(invalidErr) + return + } + + Backend.DeleteEvent(pod) +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..e285a9b --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,49 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2021-04-25 +// Description: version releated + +// Package version is for version check +package version + +import ( + "fmt" + "os" + "runtime" +) + +var ( + // Version represents rubik version + Version string + // Release represents rubik release number + Release string + // GitCommit represents git commit number + GitCommit string + // BuildTime represents build time + BuildTime string +) + +func init() { + var showVersion bool + if len(os.Args) == 2 && os.Args[1] == "-v" { + showVersion = true + } + + if showVersion { + fmt.Println("Version: ", Version) + fmt.Println("Release: ", Release) + fmt.Println("Go Version: ", runtime.Version()) + fmt.Println("Git Commit: ", GitCommit) + fmt.Println("Built: ", BuildTime) + fmt.Println("OS/Arch: ", runtime.GOOS+"/"+runtime.GOARCH) + os.Exit(0) + } +} -- Gitee From 02b1a3b9c935f6adadd5fdd98ffe009de2366b60 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Sat, 3 Dec 2022 01:49:50 +0800 Subject: [PATCH 03/73] refactor: redesign api 1. introduce plugin registry mechanism between main flow and each modules 2. introduce observer mode Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/api/api.go | 45 +++++++++++++++++ pkg/modules/blkio/blkio.go | 61 +++++++++++++++++++++++ pkg/modules/cachelimit/cachelimit.go | 61 +++++++++++++++++++++++ pkg/modules/checkpoint/checkpoint.go | 68 ++++++++++++++++++++++++++ pkg/modules/cpu/cpu.go | 61 +++++++++++++++++++++++ pkg/modules/iocost/iocost.go | 61 +++++++++++++++++++++++ pkg/modules/memory/memory.go | 61 +++++++++++++++++++++++ pkg/modules/podinformer/podinformer.go | 2 +- pkg/modules/quota/cpuBurst.go | 1 + pkg/modules/quota/quotaTurbo.go | 1 + pkg/registry/registry.go | 56 +++++++++++++++++++++ pkg/rubik/rubik.go | 18 +++++++ rubik.go | 11 +++++ 13 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 pkg/api/api.go create mode 100644 pkg/modules/blkio/blkio.go create mode 100644 pkg/modules/cachelimit/cachelimit.go create mode 100644 pkg/modules/checkpoint/checkpoint.go create mode 100644 pkg/modules/cpu/cpu.go create mode 100644 pkg/modules/iocost/iocost.go create mode 100644 pkg/modules/memory/memory.go create mode 100644 pkg/modules/quota/cpuBurst.go create mode 100644 pkg/modules/quota/quotaTurbo.go create mode 100644 pkg/registry/registry.go create mode 100644 pkg/rubik/rubik.go create mode 100644 rubik.go diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..a0810ea --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,45 @@ +package api + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Registry provides an interface for service discovery +type Registry interface { + Init() error + Register(*Service, string) error + Deregister(*Service, string) error + GetService(string) (*Service, error) + ListServices() ([]*Service, error) +} + +// Service contains progress that all services(modules) need to have +type Service interface { + Init() error + Setup() error + Run() error + TearDown() error +} + +// PodEventSubscriber control pod activities +type PodEventSubscriber interface { + AddPod(pod *corev1.Pod) + UpdatePod(pod *corev1.Pod) + DeletePod(podID types.UID) + ID() string +} + +// Publisher publish pod event to all subscribers +type Publisher interface { + Subscribe(s PodEventSubscriber) + Unsubscribe(s PodEventSubscriber) + NotifySubscribers() + ReceivePodEvent(string, *corev1.Pod) +} + +// Viewer collect on/offline pods info +type Viewer interface { + ListOnlinePods() ([]*corev1.Pod, error) + ListOfflinePods() ([]*corev1.Pod, error) +} diff --git a/pkg/modules/blkio/blkio.go b/pkg/modules/blkio/blkio.go new file mode 100644 index 0000000..a3fedc8 --- /dev/null +++ b/pkg/modules/blkio/blkio.go @@ -0,0 +1,61 @@ +package blkio + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/registry" +) + +type Blkio struct { + Name string + api.Service + api.PodEventSubscriber +} + +func init() { + registry.DefaultRegister.Register(&NewBlkio().Service, "blkio") +} + +func NewBlkio() *Blkio { + return &Blkio{Name: "blkio"} +} + +func (b *Blkio) Init() error { + fmt.Println("blkio Init()") + return nil +} + +func (b *Blkio) Setup() error { + fmt.Println("blkio Setup()") + return nil +} + +func (b *Blkio) Run() error { + fmt.Println("blkio Run()") + return nil +} + +func (b *Blkio) TearDown() error { + fmt.Println("blkio TearDown()") + return nil +} + +func (b *Blkio) AddPod(pod *corev1.Pod) { + fmt.Println("blkio AddPod()") +} + +func (b *Blkio) UpdatePod(pod *corev1.Pod) { + fmt.Println("blkio UpdatePod()") +} + +func (b *Blkio) DeletePod(podID types.UID) { + fmt.Println("blkio DeletePod()") +} + +func (b *Blkio) ID() string { + return b.Name +} diff --git a/pkg/modules/cachelimit/cachelimit.go b/pkg/modules/cachelimit/cachelimit.go new file mode 100644 index 0000000..4a67990 --- /dev/null +++ b/pkg/modules/cachelimit/cachelimit.go @@ -0,0 +1,61 @@ +package cachelimit + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/registry" +) + +type CacheLimit struct { + Name string + api.Service + api.PodEventSubscriber +} + +func init() { + registry.DefaultRegister.Register(&NewCacheLimit().Service, "cacheLimit") +} + +func NewCacheLimit() *CacheLimit { + return &CacheLimit{Name: "cacheLimit"} +} + +func (c *CacheLimit) Init() error { + fmt.Println("cache limit Init()") + return nil +} + +func (c *CacheLimit) Setup() error { + fmt.Println("cache limit Setup()") + return nil +} + +func (c *CacheLimit) Run() error { + fmt.Println("cache limit Run()") + return nil +} + +func (c *CacheLimit) TearDown() error { + fmt.Println("cache limit TearDown()") + return nil +} + +func (c *CacheLimit) AddPod(pod *corev1.Pod) { + fmt.Println("cache limit AddPod()") +} + +func (c *CacheLimit) UpdatePod(pod *corev1.Pod) { + fmt.Println("cache limit UpdatePod()") +} + +func (c *CacheLimit) DeletePod(podID types.UID) { + fmt.Println("cache limit DeletePod()") +} + +func (c *CacheLimit) ID() string { + return c.Name +} diff --git a/pkg/modules/checkpoint/checkpoint.go b/pkg/modules/checkpoint/checkpoint.go new file mode 100644 index 0000000..eab7e2e --- /dev/null +++ b/pkg/modules/checkpoint/checkpoint.go @@ -0,0 +1,68 @@ +package checkpoint + +import ( + "fmt" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/typedef" + corev1 "k8s.io/api/core/v1" +) + +type CheckPoint struct { + Pods map[string]*typedef.PodInfo `json:"pods,omitempty"` +} + +type CheckPointViewer struct { + checkpoint CheckPoint + subscribers []api.PodEventSubscriber +} + +type CheckPointPublisher struct { + checkpoint *CheckPoint + subscribers []api.PodEventSubscriber + viewer CheckPointViewer +} + +func NewCheckPointPublisher() *CheckPointPublisher { + return &CheckPointPublisher{ + checkpoint: &CheckPoint{ + Pods: make(map[string]*typedef.PodInfo), + }, + subscribers: make([]api.PodEventSubscriber, 0), + } +} + +func (cpp *CheckPointPublisher) Subscribe(s api.PodEventSubscriber) { + fmt.Printf("CheckPointPublisher subscribe()\n") + cpp.subscribers = append(cpp.subscribers, s) +} + +func (cpp *CheckPointPublisher) Unsubscribe(s api.PodEventSubscriber) { + fmt.Printf("CheckPointPublisher unsubscribe()\n") + subscribersLength := len(cpp.subscribers) + for i, subscriber := range cpp.subscribers { + if s.ID() == subscriber.ID() { + cpp.subscribers[subscribersLength-1], cpp.subscribers[i] = cpp.subscribers[i], cpp.subscribers[subscribersLength-1] + cpp.subscribers = cpp.subscribers[:subscribersLength-1] + break + } + } +} + +func (cpp *CheckPointPublisher) NotifySubscribers() { + fmt.Printf("CheckPointPublisher notifyAll()\n") +} + +func (cpp *CheckPointPublisher) ReceivePodEvent(eventType string, pod *corev1.Pod) { + fmt.Printf("CheckPointPublisher ReceivePodEvent(%s)\n", eventType) +} + +func (cv *CheckPointViewer) ListOnlinePods() ([]*corev1.Pod, error) { + fmt.Printf("CheckPointViewer ListOnlinePods()\n") + return nil, nil +} + +func (cv *CheckPointViewer) ListOfflinePods() ([]*corev1.Pod, error) { + fmt.Printf("CheckPointViewer ListOfflinePods()\n") + return nil, nil +} diff --git a/pkg/modules/cpu/cpu.go b/pkg/modules/cpu/cpu.go new file mode 100644 index 0000000..bbede9c --- /dev/null +++ b/pkg/modules/cpu/cpu.go @@ -0,0 +1,61 @@ +package cpu + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/registry" +) + +type CPU struct { + Name string + api.Service + api.PodEventSubscriber +} + +func init() { + registry.DefaultRegister.Register(&NewCPU().Service, "cpu") +} + +func NewCPU() *CPU { + return &CPU{Name: "cpu"} +} + +func (c *CPU) Init() error { + fmt.Println("cpu Init()") + return nil +} + +func (c *CPU) Setup() error { + fmt.Println("cpu Setup()") + return nil +} + +func (c *CPU) Run() error { + fmt.Println("cpu Run()") + return nil +} + +func (c *CPU) TearDown() error { + fmt.Println("cpu TearDown()") + return nil +} + +func (c *CPU) AddPod(pod *corev1.Pod) { + fmt.Println("cpu AddPod()") +} + +func (c *CPU) UpdatePod(pod *corev1.Pod) { + fmt.Println("cpu UpdatePod()") +} + +func (c *CPU) DeletePod(podID types.UID) { + fmt.Println("cpu DeletePod()") +} + +func (c *CPU) ID() string { + return c.Name +} diff --git a/pkg/modules/iocost/iocost.go b/pkg/modules/iocost/iocost.go new file mode 100644 index 0000000..09e3358 --- /dev/null +++ b/pkg/modules/iocost/iocost.go @@ -0,0 +1,61 @@ +package iocost + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/registry" +) + +type IOCost struct { + Name string + api.Service + api.PodEventSubscriber +} + +func init() { + registry.DefaultRegister.Register(&NewIOCost().Service, "iocost") +} + +func NewIOCost() *IOCost { + return &IOCost{Name: "iocost"} +} + +func (i *IOCost) Init() error { + fmt.Println("iocost Init()") + return nil +} + +func (i *IOCost) Setup() error { + fmt.Println("iocost Setup()") + return nil +} + +func (i *IOCost) Run() error { + fmt.Println("iocost Run()") + return nil +} + +func (i *IOCost) TearDown() error { + fmt.Println("iocost TearDown()") + return nil +} + +func (i *IOCost) AddPod(pod *corev1.Pod) { + fmt.Println("cpu AddPod()") +} + +func (i *IOCost) UpdatePod(pod *corev1.Pod) { + fmt.Println("cpu UpdatePod()") +} + +func (i *IOCost) DeletePod(podID types.UID) { + fmt.Println("cpu DeletePod()") +} + +func (i *IOCost) ID() string { + return i.Name +} diff --git a/pkg/modules/memory/memory.go b/pkg/modules/memory/memory.go new file mode 100644 index 0000000..4dd3f13 --- /dev/null +++ b/pkg/modules/memory/memory.go @@ -0,0 +1,61 @@ +package memory + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/registry" +) + +type Memory struct { + Name string + api.Service + api.PodEventSubscriber +} + +func init() { + registry.DefaultRegister.Register(&NewMemory().Service, "memory") +} + +func NewMemory() *Memory { + return &Memory{Name: "memory"} +} + +func (m *Memory) Init() error { + fmt.Println("memory Init()") + return nil +} + +func (m *Memory) Setup() error { + fmt.Println("memory Setup()") + return nil +} + +func (m *Memory) Run() error { + fmt.Println("memory Run()") + return nil +} + +func (m *Memory) TearDown() error { + fmt.Println("memory TearDown()") + return nil +} + +func (m *Memory) AddPod(pod *corev1.Pod) { + fmt.Println("memory AddPod()") +} + +func (m *Memory) UpdatePod(pod *corev1.Pod) { + fmt.Println("memory UpdatePod()") +} + +func (m *Memory) DeletePod(podID types.UID) { + fmt.Println("memory DeletePod()") +} + +func (m *Memory) ID() string { + return m.Name +} diff --git a/pkg/modules/podinformer/podinformer.go b/pkg/modules/podinformer/podinformer.go index 90e1c20..90de756 100644 --- a/pkg/modules/podinformer/podinformer.go +++ b/pkg/modules/podinformer/podinformer.go @@ -23,8 +23,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + log "isula.org/rubik/pkg/common/tinylog" "isula.org/rubik/pkg/modules/config" - log "isula.org/rubik/pkg/tools/tinylog" ) const invalidErr = "Auto config error: invalid pod type" diff --git a/pkg/modules/quota/cpuBurst.go b/pkg/modules/quota/cpuBurst.go new file mode 100644 index 0000000..0dcd561 --- /dev/null +++ b/pkg/modules/quota/cpuBurst.go @@ -0,0 +1 @@ +package quota \ No newline at end of file diff --git a/pkg/modules/quota/quotaTurbo.go b/pkg/modules/quota/quotaTurbo.go new file mode 100644 index 0000000..e63c4f8 --- /dev/null +++ b/pkg/modules/quota/quotaTurbo.go @@ -0,0 +1 @@ +package quota diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 0000000..aa6194f --- /dev/null +++ b/pkg/registry/registry.go @@ -0,0 +1,56 @@ +package registry + +import ( + "fmt" + + "isula.org/rubik/pkg/api" +) + +type RubikRegistry struct { + services map[string]*api.Service +} + +var DefaultRegister = NewRegistry() + +func NewRegistry() *RubikRegistry { + return &RubikRegistry{ + services: make(map[string]*api.Service), + } +} + +func (r *RubikRegistry) Init() error { + fmt.Println("rubik registry Init()") + return nil +} + +func (r *RubikRegistry) Register(s *api.Service, name string) error { + fmt.Printf("rubik registry Register(%s)\n", name) + if _, ok := r.services[name]; !ok { + r.services[name] = s + } + return nil +} + +func (r *RubikRegistry) Deregister(s *api.Service, name string) error { + fmt.Printf("rubik register Deregister(%s)\n", name) + delete(r.services, name) + return nil +} + +func (r *RubikRegistry) GetService(name string) (*api.Service, error) { + fmt.Printf("rubik register GetService(%s)\n", name) + if s, ok := r.services[name]; ok { + return s, nil + } else { + return nil, fmt.Errorf("service %s did not registered", name) + } +} + +func (r *RubikRegistry) ListServices() ([]*api.Service, error) { + fmt.Println("rubik register ListServices()") + var services []*api.Service + for _, s := range r.services { + services = append(services, s) + } + return services, nil +} diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go new file mode 100644 index 0000000..4ac7ac7 --- /dev/null +++ b/pkg/rubik/rubik.go @@ -0,0 +1,18 @@ +package rubik + +import ( + "fmt" + + // import packages to auto register service + _ "isula.org/rubik/pkg/modules/blkio" + _ "isula.org/rubik/pkg/modules/cachelimit" + _ "isula.org/rubik/pkg/modules/cpu" + _ "isula.org/rubik/pkg/modules/iocost" + _ "isula.org/rubik/pkg/modules/memory" + _ "isula.org/rubik/pkg/version" +) + +func Run() int { + fmt.Println("rubik running") + return 0 +} diff --git a/rubik.go b/rubik.go new file mode 100644 index 0000000..3c7676b --- /dev/null +++ b/rubik.go @@ -0,0 +1,11 @@ +package main + +import ( + "os" + + "isula.org/rubik/pkg/rubik" +) + +func main() { + os.Exit(rubik.Run()) +} -- Gitee From 70153eacc28c1ca06e766f095863b12452796f4a Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 8 Dec 2022 10:12:29 +0800 Subject: [PATCH 04/73] refactor: move implicit import to import.go file Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/rubik/import.go | 11 +++++++++++ pkg/rubik/rubik.go | 8 -------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 pkg/rubik/import.go diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go new file mode 100644 index 0000000..2b846e0 --- /dev/null +++ b/pkg/rubik/import.go @@ -0,0 +1,11 @@ +package rubik + +import ( + // import packages to auto register service + _ "isula.org/rubik/pkg/modules/blkio" + _ "isula.org/rubik/pkg/modules/cachelimit" + _ "isula.org/rubik/pkg/modules/cpu" + _ "isula.org/rubik/pkg/modules/iocost" + _ "isula.org/rubik/pkg/modules/memory" + _ "isula.org/rubik/pkg/version" +) diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 4ac7ac7..b95c1a6 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -2,14 +2,6 @@ package rubik import ( "fmt" - - // import packages to auto register service - _ "isula.org/rubik/pkg/modules/blkio" - _ "isula.org/rubik/pkg/modules/cachelimit" - _ "isula.org/rubik/pkg/modules/cpu" - _ "isula.org/rubik/pkg/modules/iocost" - _ "isula.org/rubik/pkg/modules/memory" - _ "isula.org/rubik/pkg/version" ) func Run() int { -- Gitee From 5d3d4cc094e58d1a2ad1d336e4be280c8eafc7d5 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 8 Dec 2022 17:35:58 +0800 Subject: [PATCH 05/73] refactor: starting writing rubik main flow Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/api/api.go | 1 + pkg/modules/blkio/blkio.go | 12 ++++++++---- pkg/modules/cachelimit/cachelimit.go | 12 ++++++++---- pkg/modules/checkpoint/checkpoint.go | 6 ++++-- pkg/modules/cpu/cpu.go | 12 ++++++++---- pkg/modules/iocost/iocost.go | 18 +++++++++++------- pkg/modules/memory/memory.go | 12 ++++++++---- pkg/registry/registry.go | 18 +++++++++--------- pkg/rubik/rubik.go | 24 ++++++++++++++++++++++++ 9 files changed, 81 insertions(+), 34 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index a0810ea..63185c0 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -20,6 +20,7 @@ type Service interface { Setup() error Run() error TearDown() error + PodEventHandler() error } // PodEventSubscriber control pod activities diff --git a/pkg/modules/blkio/blkio.go b/pkg/modules/blkio/blkio.go index a3fedc8..e976678 100644 --- a/pkg/modules/blkio/blkio.go +++ b/pkg/modules/blkio/blkio.go @@ -6,18 +6,16 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/modules/checkpoint" "isula.org/rubik/pkg/registry" ) type Blkio struct { Name string - api.Service - api.PodEventSubscriber } func init() { - registry.DefaultRegister.Register(&NewBlkio().Service, "blkio") + registry.DefaultRegister.Register(NewBlkio(), "blkio") } func NewBlkio() *Blkio { @@ -59,3 +57,9 @@ func (b *Blkio) DeletePod(podID types.UID) { func (b *Blkio) ID() string { return b.Name } + +func (b *Blkio) PodEventHandler() error { + fmt.Println("blkio PodEventHandler") + checkpoint.DefaultCheckPointPublisher.Subscribe(b) + return nil +} diff --git a/pkg/modules/cachelimit/cachelimit.go b/pkg/modules/cachelimit/cachelimit.go index 4a67990..3545cee 100644 --- a/pkg/modules/cachelimit/cachelimit.go +++ b/pkg/modules/cachelimit/cachelimit.go @@ -6,18 +6,16 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/modules/checkpoint" "isula.org/rubik/pkg/registry" ) type CacheLimit struct { Name string - api.Service - api.PodEventSubscriber } func init() { - registry.DefaultRegister.Register(&NewCacheLimit().Service, "cacheLimit") + registry.DefaultRegister.Register(NewCacheLimit(), "cacheLimit") } func NewCacheLimit() *CacheLimit { @@ -59,3 +57,9 @@ func (c *CacheLimit) DeletePod(podID types.UID) { func (c *CacheLimit) ID() string { return c.Name } + +func (c *CacheLimit) PodEventHandler() error { + fmt.Println("cache limit PodEventHandler") + checkpoint.DefaultCheckPointPublisher.Subscribe(c) + return nil +} diff --git a/pkg/modules/checkpoint/checkpoint.go b/pkg/modules/checkpoint/checkpoint.go index eab7e2e..263923c 100644 --- a/pkg/modules/checkpoint/checkpoint.go +++ b/pkg/modules/checkpoint/checkpoint.go @@ -23,6 +23,8 @@ type CheckPointPublisher struct { viewer CheckPointViewer } +var DefaultCheckPointPublisher = NewCheckPointPublisher() + func NewCheckPointPublisher() *CheckPointPublisher { return &CheckPointPublisher{ checkpoint: &CheckPoint{ @@ -33,12 +35,12 @@ func NewCheckPointPublisher() *CheckPointPublisher { } func (cpp *CheckPointPublisher) Subscribe(s api.PodEventSubscriber) { - fmt.Printf("CheckPointPublisher subscribe()\n") + fmt.Printf("CheckPointPublisher subscribe(%s)\n", s.ID()) cpp.subscribers = append(cpp.subscribers, s) } func (cpp *CheckPointPublisher) Unsubscribe(s api.PodEventSubscriber) { - fmt.Printf("CheckPointPublisher unsubscribe()\n") + fmt.Printf("CheckPointPublisher unsubscribe(%s)\n", s.ID()) subscribersLength := len(cpp.subscribers) for i, subscriber := range cpp.subscribers { if s.ID() == subscriber.ID() { diff --git a/pkg/modules/cpu/cpu.go b/pkg/modules/cpu/cpu.go index bbede9c..395fe91 100644 --- a/pkg/modules/cpu/cpu.go +++ b/pkg/modules/cpu/cpu.go @@ -6,18 +6,16 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/modules/checkpoint" "isula.org/rubik/pkg/registry" ) type CPU struct { Name string - api.Service - api.PodEventSubscriber } func init() { - registry.DefaultRegister.Register(&NewCPU().Service, "cpu") + registry.DefaultRegister.Register(NewCPU(), "cpu") } func NewCPU() *CPU { @@ -59,3 +57,9 @@ func (c *CPU) DeletePod(podID types.UID) { func (c *CPU) ID() string { return c.Name } + +func (c *CPU) PodEventHandler() error { + fmt.Println("cpu PodEventHandler") + checkpoint.DefaultCheckPointPublisher.Subscribe(c) + return nil +} diff --git a/pkg/modules/iocost/iocost.go b/pkg/modules/iocost/iocost.go index 09e3358..ef2b118 100644 --- a/pkg/modules/iocost/iocost.go +++ b/pkg/modules/iocost/iocost.go @@ -6,18 +6,16 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/modules/checkpoint" "isula.org/rubik/pkg/registry" ) type IOCost struct { Name string - api.Service - api.PodEventSubscriber } func init() { - registry.DefaultRegister.Register(&NewIOCost().Service, "iocost") + registry.DefaultRegister.Register(NewIOCost(), "iocost") } func NewIOCost() *IOCost { @@ -45,17 +43,23 @@ func (i *IOCost) TearDown() error { } func (i *IOCost) AddPod(pod *corev1.Pod) { - fmt.Println("cpu AddPod()") + fmt.Println("iocost AddPod()") } func (i *IOCost) UpdatePod(pod *corev1.Pod) { - fmt.Println("cpu UpdatePod()") + fmt.Println("iocost UpdatePod()") } func (i *IOCost) DeletePod(podID types.UID) { - fmt.Println("cpu DeletePod()") + fmt.Println("iocost DeletePod()") } func (i *IOCost) ID() string { return i.Name } + +func (i *IOCost) PodEventHandler() error { + fmt.Println("iosoct PodEventHandler") + checkpoint.DefaultCheckPointPublisher.Subscribe(i) + return nil +} diff --git a/pkg/modules/memory/memory.go b/pkg/modules/memory/memory.go index 4dd3f13..1646b02 100644 --- a/pkg/modules/memory/memory.go +++ b/pkg/modules/memory/memory.go @@ -6,18 +6,16 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/modules/checkpoint" "isula.org/rubik/pkg/registry" ) type Memory struct { Name string - api.Service - api.PodEventSubscriber } func init() { - registry.DefaultRegister.Register(&NewMemory().Service, "memory") + registry.DefaultRegister.Register(NewMemory(), "memory") } func NewMemory() *Memory { @@ -59,3 +57,9 @@ func (m *Memory) DeletePod(podID types.UID) { func (m *Memory) ID() string { return m.Name } + +func (m *Memory) PodEventHandler() error { + fmt.Println("memory PodEventHandler()") + checkpoint.DefaultCheckPointPublisher.Subscribe(m) + return nil +} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index aa6194f..fcc5195 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -7,14 +7,14 @@ import ( ) type RubikRegistry struct { - services map[string]*api.Service + services map[string]api.Service } var DefaultRegister = NewRegistry() -func NewRegistry() *RubikRegistry { - return &RubikRegistry{ - services: make(map[string]*api.Service), +func NewRegistry() RubikRegistry { + return RubikRegistry{ + services: make(map[string]api.Service), } } @@ -23,7 +23,7 @@ func (r *RubikRegistry) Init() error { return nil } -func (r *RubikRegistry) Register(s *api.Service, name string) error { +func (r *RubikRegistry) Register(s api.Service, name string) error { fmt.Printf("rubik registry Register(%s)\n", name) if _, ok := r.services[name]; !ok { r.services[name] = s @@ -31,13 +31,13 @@ func (r *RubikRegistry) Register(s *api.Service, name string) error { return nil } -func (r *RubikRegistry) Deregister(s *api.Service, name string) error { +func (r *RubikRegistry) Deregister(s api.Service, name string) error { fmt.Printf("rubik register Deregister(%s)\n", name) delete(r.services, name) return nil } -func (r *RubikRegistry) GetService(name string) (*api.Service, error) { +func (r *RubikRegistry) GetService(name string) (api.Service, error) { fmt.Printf("rubik register GetService(%s)\n", name) if s, ok := r.services[name]; ok { return s, nil @@ -46,9 +46,9 @@ func (r *RubikRegistry) GetService(name string) (*api.Service, error) { } } -func (r *RubikRegistry) ListServices() ([]*api.Service, error) { +func (r *RubikRegistry) ListServices() ([]api.Service, error) { fmt.Println("rubik register ListServices()") - var services []*api.Service + var services []api.Service for _, s := range r.services { services = append(services, s) } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index b95c1a6..30cedcc 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -2,9 +2,33 @@ package rubik import ( "fmt" + + "isula.org/rubik/pkg/registry" ) func Run() int { fmt.Println("rubik running") + + // 0. services automatic registration + // done in import.go + + // 1. enable autoconfig(informer) + // podinformer.Init() + + // 2. enable checkpoint + services, err := registry.DefaultRegister.ListServices() + if err != nil { + return -1 + } + for _, s := range services { + if err := s.PodEventHandler(); err != nil { + continue + } + s.Init() + s.Setup() + s.Run() + defer s.TearDown() + } + return 0 } -- Gitee From 2daa4b64f908f6cb59c17ee4e1cbffc46c9f0cc4 Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 10 Jan 2023 11:58:53 +0800 Subject: [PATCH 06/73] add publisher&subscriber framework 1. design publisher and subscriber interface(api.go) 2. implement generic publisher (using publisherfactory) & generic subscriber(core) 3. move log to common dir(common/log) --- pkg/api/api.go | 50 +++- pkg/common/log/log.go | 316 +++++++++++++++++++++++ pkg/common/log/log_test.go | 273 ++++++++++++++++++++ pkg/core/publisher/genericpublisher.go | 113 ++++++++ pkg/core/publisher/publisherFactory.go | 43 +++ pkg/core/subscriber/genericsubscriber.go | 34 +++ pkg/core/typedef/event.go | 25 ++ 7 files changed, 846 insertions(+), 8 deletions(-) create mode 100644 pkg/common/log/log.go create mode 100644 pkg/common/log/log_test.go create mode 100644 pkg/core/publisher/genericpublisher.go create mode 100644 pkg/core/publisher/publisherFactory.go create mode 100644 pkg/core/subscriber/genericsubscriber.go create mode 100644 pkg/core/typedef/event.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 63185c0..70f6934 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -1,8 +1,24 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file contains important interfaces used in the project + +// Package api is interface collection package api import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + + "isula.org/rubik/pkg/core/typedef" ) // Registry provides an interface for service discovery @@ -31,16 +47,34 @@ type PodEventSubscriber interface { ID() string } -// Publisher publish pod event to all subscribers -type Publisher interface { - Subscribe(s PodEventSubscriber) - Unsubscribe(s PodEventSubscriber) - NotifySubscribers() - ReceivePodEvent(string, *corev1.Pod) -} - // Viewer collect on/offline pods info type Viewer interface { ListOnlinePods() ([]*corev1.Pod, error) ListOfflinePods() ([]*corev1.Pod, error) } + +// Publisher is a generic interface for Observables +type Publisher interface { + Subscribe(s Subscriber) error + Unsubscribe(s Subscriber) error + Publish(topic typedef.EventType, event typedef.Event) +} + +// Subscriber is a common interface for subscribers +type Subscriber interface { + ID() string + NotifyFunc(eventType typedef.EventType, event typedef.Event) + TopicsFunc() []typedef.EventType +} + +// EventHandler is the processing interface for change events +type EventHandler interface { + HandleEvent(eventType typedef.EventType, event typedef.Event) + EventTypes() []typedef.EventType +} + +// Informer is an interface for external pod data sources to interact with rubik +type Informer interface { + Publisher + Start(stopCh <-chan struct{}) +} diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go new file mode 100644 index 0000000..0b8a53b --- /dev/null +++ b/pkg/common/log/log.go @@ -0,0 +1,316 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Haomin Tsai +// Create: 2021-09-28 +// Description: This file is used for rubik log + +// Package tinylog is for rubik log +package log + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "isula.org/rubik/pkg/common/constant" +) + +// CtxKey used for UUID +type CtxKey string + +const ( + // UUID is log uuid + UUID = "uuid" + + logDriverStdio = "stdio" + logDriverFile = "file" + logStack = 20 + logStackFrom = 2 + logLevelInfo = "info" + logLevelStack = "stack" + + logFileNum = 10 + logSizeMin int64 = 10 // 10MB + logSizeMax int64 = 1024 * 1024 // 1TB + unitMB int64 = 1024 * 1024 +) + +const ( + stdio int = iota + file +) + +const ( + logDebug int = iota + logInfo + logWarn + logError +) + +var ( + logDriver = stdio + logFname = filepath.Join(constant.DefaultLogDir, "rubik.log") + logLevel = 0 + logSize int64 = 1024 + logFileMaxSize int64 + logFileSize int64 + + lock = sync.Mutex{} +) + +func makeLogDir(logDir string) error { + if !filepath.IsAbs(logDir) { + return fmt.Errorf("log-dir %v must be an absolute path", logDir) + } + + if err := os.MkdirAll(logDir, constant.DefaultDirMode); err != nil { + return fmt.Errorf("create log directory %v failed", logDir) + } + + return nil +} + +// InitConfig init log config +func InitConfig(driver, logdir, level string, size int64) error { + if driver == "" { + driver = logDriverStdio + } + if driver != logDriverStdio && driver != logDriverFile { + return fmt.Errorf("invalid log driver %s", driver) + } + logDriver = stdio + if driver == logDriverFile { + logDriver = file + } + + if level == "" { + level = logLevelInfo + } + levelstr, err := logLevelFromString(level) + if err != nil { + return err + } + logLevel = levelstr + + if size < logSizeMin || size > logSizeMax { + return fmt.Errorf("invalid log size %d", size) + } + logSize = size + logFileMaxSize = logSize / logFileNum + + if driver == "file" { + if err := makeLogDir(logdir); err != nil { + return err + } + logFname = filepath.Join(logdir, "rubik.log") + if f, err := os.Stat(logFname); err == nil { + atomic.StoreInt64(&logFileSize, f.Size()) + } + } + + return nil +} + +// DropError drop unused error +func DropError(args ...interface{}) { + argn := len(args) + if argn == 0 { + return + } + arg := args[argn-1] + if arg != nil { + fmt.Printf("drop error: %v\n", arg) + } +} + +func logLevelToString(level int) string { + switch level { + case logDebug: + return "debug" + case logInfo: + return "info" + case logWarn: + return "warn" + case logError: + return "error" + case logStack: + return logLevelStack + default: + return "" + } +} + +func logLevelFromString(level string) (int, error) { + switch level { + case "debug": + return logDebug, nil + case "info", "": + return logInfo, nil + case "warn": + return logWarn, nil + case "error": + return logError, nil + default: + return logInfo, fmt.Errorf("invalid log level %s", level) + } +} + +func logRename() { + for i := logFileNum - 1; i > 1; i-- { + old := logFname + fmt.Sprintf(".%d", i-1) + new := logFname + fmt.Sprintf(".%d", i) + if _, err := os.Stat(old); err == nil { + DropError(os.Rename(old, new)) + } + } + DropError(os.Rename(logFname, logFname+".1")) +} + +func logRotate(line int64) string { + if atomic.AddInt64(&logFileSize, line) > logFileMaxSize*unitMB { + logRename() + atomic.StoreInt64(&logFileSize, line) + } + + return logFname +} + +func writeLine(line string) { + if logDriver == stdio { + fmt.Printf("%s", line) + return + } + + lock.Lock() + defer lock.Unlock() + + f, err := os.OpenFile(logRotate(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) + if err != nil { + return + } + + DropError(f.WriteString(line)) + DropError(f.Close()) +} + +func logf(level string, format string, args ...interface{}) { + tag := fmt.Sprintf("%s [rubik] level=%s ", time.Now().Format("2006-01-02 15:04:05.000"), level) + raw := fmt.Sprintf(format, args...) + "\n" + + depth := 1 + if level == logLevelStack { + depth = logStack + } + + for i := logStackFrom; i < logStackFrom+depth; i++ { + line := tag + raw + pc, file, linum, ok := runtime.Caller(i) + if ok { + fs := strings.Split(runtime.FuncForPC(pc).Name(), "/") + fs = strings.Split("."+fs[len(fs)-1], ".") + fn := fs[len(fs)-1] + line = tag + fmt.Sprintf("%s:%d:%s() ", file, linum, fn) + raw + } else if level == logLevelStack { + break + } + writeLine(line) + } +} + +// Warnf log warn level +func Warnf(format string, args ...interface{}) { + if logWarn >= logLevel { + logf(logLevelToString(logInfo), format, args...) + } +} + +// Infof log info level +func Infof(format string, args ...interface{}) { + if logInfo >= logLevel { + logf(logLevelToString(logInfo), format, args...) + } +} + +// Debugf log debug level +func Debugf(format string, args ...interface{}) { + if logDebug >= logLevel { + logf(logLevelToString(logDebug), format, args...) + } +} + +// Errorf log error level +func Errorf(format string, args ...interface{}) { + if logError >= logLevel { + logf(logLevelToString(logError), format, args...) + } +} + +// Stackf log stack dump +func Stackf(format string, args ...interface{}) { + logf("stack", format, args...) +} + +// Entry is log entry +type Entry struct { + Ctx context.Context +} + +// WithCtx create entry with ctx +func WithCtx(ctx context.Context) *Entry { + return &Entry{ + Ctx: ctx, + } +} + +func (e *Entry) level(l int) string { + uuid, ok := e.Ctx.Value(CtxKey(UUID)).(string) + if ok { + return logLevelToString(l) + " UUID=" + uuid + } + return logLevelToString(l) +} + +// Logf write logs +func (e *Entry) Logf(f string, args ...interface{}) { + if logInfo < logLevel { + return + } + logf(e.level(logInfo), f, args...) +} + +// Infof write logs +func (e *Entry) Infof(f string, args ...interface{}) { + if logInfo < logLevel { + return + } + logf(e.level(logInfo), f, args...) +} + +// Debugf write verbose logs +func (e *Entry) Debugf(f string, args ...interface{}) { + if logDebug < logLevel { + return + } + logf(e.level(logDebug), f, args...) +} + +// Errorf write error logs +func (e *Entry) Errorf(f string, args ...interface{}) { + if logError < logLevel { + return + } + logf(e.level(logError), f, args...) +} diff --git a/pkg/common/log/log_test.go b/pkg/common/log/log_test.go new file mode 100644 index 0000000..40be768 --- /dev/null +++ b/pkg/common/log/log_test.go @@ -0,0 +1,273 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Danni Xia +// Create: 2021-05-24 +// Description: This file is used for testing tinylog + +package log + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/try" +) + +// test_rubik_set_logdriver_0001 +func TestInitConfigLogDriver(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + // case: rubik.log already exist. + try.WriteFile(logFilePath, []byte(""), constant.DefaultFileMode) + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + + err = os.RemoveAll(logDir) + assert.NoError(t, err) + + // logDriver is file + err = InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, file, logDriver) + logString := "Test InitConfig with logDriver file" + Infof(logString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, true, strings.Contains(string(b), logString)) + + // logDriver is stdio + os.Remove(logFilePath) + err = InitConfig("stdio", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, stdio, logDriver) + logString = "Test InitConfig with logDriver stdio" + Infof(logString) + b, err = ioutil.ReadFile(logFilePath) + assert.Equal(t, true, err != nil) + + // logDriver invalid + err = InitConfig("std", logDir, "", logSize) + assert.Equal(t, true, err != nil) + + // logDriver is null + err = InitConfig("", logDir, "", logSize) + assert.NoError(t, err) + assert.Equal(t, stdio, logDriver) +} + +// test_rubik_set_logdir_0001 +func TestInitConfigLogDir(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + // LogDir valid + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + logString := "Test InitConfig with logDir valid" + Infof(logString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, true, strings.Contains(string(b), logString)) + + // logDir invalid + err = InitConfig("file", "invalid/log", "", logSize) + assert.Equal(t, true, err != nil) +} + +type logTC struct { + name, logLevel string + wantErr, debug, info, error bool +} + +func createLogTC() []logTC { + return []logTC{ + { + name: "TC1-logLevel debug", + logLevel: "debug", + wantErr: false, + debug: true, + info: true, + error: true, + }, + { + name: "TC2-logLevel info", + logLevel: "info", + wantErr: false, + debug: false, + info: true, + error: true, + }, + { + name: "TC3-logLevel error", + logLevel: "error", + wantErr: false, + debug: false, + info: false, + error: true, + }, + { + name: "TC4-logLevel null", + logLevel: "", + wantErr: false, + debug: false, + info: true, + error: true, + }, + { + name: "TC5-logLevel invalid", + logLevel: "inf", + wantErr: true, + }, + } +} + +// test_rubik_set_loglevel_0001 +func TestInitConfigLogLevel(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + debugLogSting, infoLogSting, errorLogSting, logLogString := "Test InitConfig debug log", + "Test InitConfig info log", "Test InitConfig error log", "Test InitConfig log log" + for _, tt := range createLogTC() { + t.Run(tt.name, func(t *testing.T) { + err := InitConfig("file", logDir, tt.logLevel, logSize) + if (err != nil) != tt.wantErr { + t.Errorf("InitConfig() = %v, want %v", err, tt.wantErr) + } else if tt.wantErr == false { + Debugf(debugLogSting) + Infof(infoLogSting) + Errorf(errorLogSting) + Infof(logLogString) + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) + assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) + os.Remove(logFilePath) + + ctx := context.WithValue(context.Background(), CtxKey(UUID), "abc123") + WithCtx(ctx).Debugf(debugLogSting) + WithCtx(ctx).Infof(infoLogSting) + WithCtx(ctx).Errorf(errorLogSting) + WithCtx(ctx).Logf(logLogString) + b, err = ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) + assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) + assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) + assert.Equal(t, true, strings.Contains(string(b), "abc123")) + err = os.RemoveAll(logDir) + assert.NoError(t, err) + } + }) + } +} + +// test_rubik_set_logsize_0001 +func TestInitConfigLogSize(t *testing.T) { + logDir := try.GenTestDir().String() + // LogSize invalid + err := InitConfig("file", logDir, "", logSizeMin-1) + assert.Equal(t, true, err != nil) + err = InitConfig("file", logDir, "", logSizeMax+1) + assert.Equal(t, true, err != nil) + + // logSize valid + testSize, printLine, repeat := 100, 50000, 100 + err = InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + for i := 0; i < printLine; i++ { + Infof(strings.Repeat("TestInitConfigLogSize log", repeat)) + } + err = InitConfig("file", logDir, "", int64(testSize)) + assert.NoError(t, err) + for i := 0; i < printLine; i++ { + Infof(strings.Repeat("TestInitConfigLogSize log", repeat)) + } + var size int64 + err = filepath.Walk(logDir, func(_ string, f os.FileInfo, _ error) error { + size += f.Size() + return nil + }) + assert.NoError(t, err) + assert.Equal(t, true, size < int64(testSize)*unitMB) + err = os.RemoveAll(constant.TmpTestDir) + assert.NoError(t, err) +} + +// TestLogStack is Stackf function test +func TestLogStack(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + Stackf("test stack log") + b, err := ioutil.ReadFile(logFilePath) + assert.NoError(t, err) + fmt.Println(string(b)) + assert.Equal(t, true, strings.Contains(string(b), t.Name())) + line := strings.Split(string(b), "\n") + maxLineNum := 5 + assert.Equal(t, true, len(line) < maxLineNum) +} + +// TestDropError is DropError function test +func TestDropError(t *testing.T) { + logDir := try.GenTestDir().String() + logFilePath := filepath.Join(logDir, "rubik.log") + + err := InitConfig("file", logDir, "", logSize) + assert.NoError(t, err) + DropError() + dropError := "test drop error" + DropError(dropError) + DropError(nil) + _, err = ioutil.ReadFile(logFilePath) + assert.Equal(t, true, err != nil) +} + +// TestLogOthers is log other tests +func TestLogOthers(t *testing.T) { + logDir := filepath.Join(try.GenTestDir().String(), "regular-file") + try.WriteFile(logDir, []byte{}, constant.DefaultFileMode) + + err := makeLogDir(logDir) + assert.Equal(t, true, err != nil) + + const outOfRangeLogLevel = 100 + s := logLevelToString(outOfRangeLogLevel) + assert.Equal(t, "", s) + const stackLoglevel = 20 + s = logLevelToString(stackLoglevel) + assert.Equal(t, "stack", s) + + logDriver = 1 + logFname = filepath.Join(constant.TmpTestDir, "log-not-exist") + os.MkdirAll(logFname, constant.DefaultDirMode) + writeLine("abc") + + s = WithCtx(context.Background()).level(1) + assert.Equal(t, "info", s) + + logLevel = logError + 1 + WithCtx(context.Background()).Errorf("abc") +} diff --git a/pkg/core/publisher/genericpublisher.go b/pkg/core/publisher/genericpublisher.go new file mode 100644 index 0000000..bd170a4 --- /dev/null +++ b/pkg/core/publisher/genericpublisher.go @@ -0,0 +1,113 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file implements pod publisher + +// Package publisher implement publisher interface +package publisher + +import ( + "fmt" + "sync" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" +) + +type ( + // subscriberIDs records subscriber's ID + subscriberIDs map[string]struct{} + // NotifyFunc is used to notify subscribers of events + NotifyFunc func(typedef.EventType, typedef.Event) +) + +// defaultPublisher is the default is a globally unique generic publisher entity +var defaultPublisher *genericPublisher + +// genericPublisher is the structure to publish Event +type genericPublisher struct { + sync.RWMutex + // topicSubscribersMap is a collection of subscribers organized by interested topics + topicSubscribersMap map[typedef.EventType]subscriberIDs + // subscribers is the set of notification methods divided by ID + subscribers map[string]NotifyFunc +} + +// newGenericPublisher creates the genericPublisher instance +func newGenericPublisher() *genericPublisher { + pub := &genericPublisher{ + subscribers: make(map[string]NotifyFunc, 0), + topicSubscribersMap: make(map[typedef.EventType]subscriberIDs, 0), + } + return pub +} + +// getGenericPublisher initializes via lazy mode and return generic publisher entity +func getGenericPublisher() *genericPublisher { + if defaultPublisher == nil { + defaultPublisher = newGenericPublisher() + } + return defaultPublisher +} + +// subscriberExisted confirms the existence of the subscriber based on the ID +func (pub *genericPublisher) subscriberExisted(id string) bool { + pub.RLock() + _, ok := pub.subscribers[id] + pub.RUnlock() + return ok +} + +// Subscribe registers a api.Subscriber +func (pub *genericPublisher) Subscribe(s api.Subscriber) error { + id := s.ID() + if pub.subscriberExisted(id) { + return fmt.Errorf("subscriber %v has registered", id) + } + pub.Lock() + for _, topic := range s.TopicsFunc() { + if _, ok := pub.topicSubscribersMap[topic]; !ok { + pub.topicSubscribersMap[topic] = make(subscriberIDs, 0) + } + pub.topicSubscribersMap[topic][id] = struct{}{} + log.Debugf("%s subscribes topic %s\n", id, topic) + } + pub.subscribers[id] = s.NotifyFunc + pub.Unlock() + return nil +} + +// Unsubscribe unsubscribes the indicated subscriber +func (pub *genericPublisher) Unsubscribe(s api.Subscriber) error { + id := s.ID() + if !pub.subscriberExisted(id) { + return fmt.Errorf("subscriber %v has not registered", id) + } + pub.Lock() + for _, topic := range s.TopicsFunc() { + delete(pub.topicSubscribersMap[topic], id) + log.Debugf("%s unsubscribes topic %s\n", id, topic) + } + delete(pub.subscribers, id) + pub.Unlock() + return nil +} + +// Publish publishes Event to subscribers interested in specified topic +func (pub *genericPublisher) Publish(eventType typedef.EventType, data typedef.Event) { + pub.RLock() + for id := range pub.topicSubscribersMap[eventType] { + pub.subscribers[id](eventType, data) + } + pub.RUnlock() + log.Debugf("publish %s event", eventType.String()) +} diff --git a/pkg/core/publisher/publisherFactory.go b/pkg/core/publisher/publisherFactory.go new file mode 100644 index 0000000..c582956 --- /dev/null +++ b/pkg/core/publisher/publisherFactory.go @@ -0,0 +1,43 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file contains publisher factory + +// Package publisher implement publisher interface +package publisher + +import "isula.org/rubik/pkg/api" + +type publihserType int8 + +const ( + // TYPE_GENERIC indicates the generic publisher type + TYPE_GENERIC publihserType = iota +) + +// PublisherFactory is the factory class of the publisher entity +type PublisherFactory struct { +} + +// NewPublisherFactory creates a publisher factory class +func NewPublisherFactory() *PublisherFactory { + return &PublisherFactory{} +} + +// GetPublisher returns the publisher entity according to the publisher type +func (f *PublisherFactory) GetPublisher(publisherType publihserType) api.Publisher { + switch publisherType { + case TYPE_GENERIC: + return getGenericPublisher() + default: + return nil + } +} diff --git a/pkg/core/subscriber/genericsubscriber.go b/pkg/core/subscriber/genericsubscriber.go new file mode 100644 index 0000000..901cba0 --- /dev/null +++ b/pkg/core/subscriber/genericsubscriber.go @@ -0,0 +1,34 @@ +package subscriber + +import ( + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/core/typedef" +) + +type genericSubscriber struct { + id string + api.EventHandler +} + +// NewGenericSubscriber returns the generic subscriber entity +func NewGenericSubscriber(handler api.EventHandler, id string) *genericSubscriber { + return &genericSubscriber{ + id: id, + EventHandler: handler, + } +} + +// ID returns the unique ID of the subscriber +func (pub *genericSubscriber) ID() string { + return pub.id +} + +// NotifyFunc notifys subscriber event +func (pub *genericSubscriber) NotifyFunc(eventType typedef.EventType, event typedef.Event) { + pub.HandleEvent(eventType, event) +} + +// TopicsFunc returns the topics that the subscriber is interested in +func (pub *genericSubscriber) TopicsFunc() []typedef.EventType { + return pub.EventTypes() +} diff --git a/pkg/core/typedef/event.go b/pkg/core/typedef/event.go new file mode 100644 index 0000000..0417765 --- /dev/null +++ b/pkg/core/typedef/event.go @@ -0,0 +1,25 @@ +package typedef + +type ( + EventType int8 + Event interface{} +) + +const ( + ADD EventType = iota + UPDATE + DELETE +) + +func (t EventType) String() string { + switch t { + case ADD: + return "add" + case UPDATE: + return "update" + case DELETE: + return "delete" + default: + return "unknown" + } +} -- Gitee From 698a2062a242ece3c256b5c56faae204335b8ec1 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 12 Jan 2023 09:21:01 +0800 Subject: [PATCH 07/73] add informer & implement kubeinformer interacting with apiserver --- pkg/informer/kubeinformer.go | 109 +++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 pkg/informer/kubeinformer.go diff --git a/pkg/informer/kubeinformer.go b/pkg/informer/kubeinformer.go new file mode 100644 index 0000000..652815c --- /dev/null +++ b/pkg/informer/kubeinformer.go @@ -0,0 +1,109 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines apiinformer which interact with kubernetes apiserver + +// Package typedef implement informer interface +package informer + +import ( + "fmt" + "os" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef" +) + +// KubeInformer interacts with k8s api server and forward data to the internal +type kubeInformer struct { + api.Publisher + client *kubernetes.Clientset + nodeName string +} + +// NewKubeInformer creates an KubeInformer instance +func NewKubeInformer(publisher api.Publisher) (*kubeInformer, error) { + informer := &kubeInformer{ + Publisher: publisher, + } + + // interact with apiserver + client, err := initKubeClient() + if err != nil { + return nil, fmt.Errorf("fail to init kubenetes client: %v", err) + } + informer.client = client + + // filter pods on current nodes + nodeName := os.Getenv(constant.NodeNameEnvKey) + if nodeName == "" { + return nil, fmt.Errorf("missing %s", constant.NodeNameEnvKey) + } + informer.nodeName = nodeName + + return informer, nil +} + +// initKubeClient initializes kubeClient +func initKubeClient() (*kubernetes.Clientset, error) { + conf, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + kubeClient, err := kubernetes.NewForConfig(conf) + if err != nil { + return nil, err + } + + return kubeClient, nil +} + +// Start starts and enables KubeInfomer +func (ki *kubeInformer) Start(stopCh <-chan struct{}) { + const ( + reSyncTime = 30 + specNodeNameField = "spec.nodeName" + ) + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(ki.client, + time.Duration(reSyncTime)*time.Second, + informers.WithTweakListOptions(func(options *metav1.ListOptions) { + // set Options to return only pods on the current node. + options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, ki.nodeName).String() + })) + kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: ki.addFunc, + UpdateFunc: ki.updateFunc, + DeleteFunc: ki.deleteFunc, + }) + kubeInformerFactory.Start(stopCh) +} + +func (ki *kubeInformer) addFunc(obj interface{}) { + ki.Publish(typedef.ADD, obj) +} + +func (ki *kubeInformer) updateFunc(oldObj, newObj interface{}) { + ki.Publish(typedef.UPDATE, newObj) +} + +func (ki *kubeInformer) deleteFunc(obj interface{}) { + ki.Publish(typedef.DELETE, obj) +} -- Gitee From bd2c195422c3fbcbfe6eb25e4cb1fe6e3dddef4f Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 12 Jan 2023 20:10:19 +0800 Subject: [PATCH 08/73] implement pod cache The Cache processes and stores the data when receiving event. And then, it transfers pod information to it's subscribers. --- pkg/core/publisher/genericpublisher.go | 2 +- pkg/core/typedef/containerinfo.go | 53 +++++++ pkg/core/typedef/event.go | 57 ++++++-- pkg/core/typedef/podinfo.go | 192 +++++++++++++++++++++++++ pkg/core/typedef/rawpod.go | 53 +++++++ pkg/informer/kubeinformer.go | 6 +- pkg/podmanager/podcache.go | 88 ++++++++++++ pkg/podmanager/podmanager.go | 157 ++++++++++++++++++++ 8 files changed, 591 insertions(+), 17 deletions(-) create mode 100644 pkg/core/typedef/containerinfo.go create mode 100644 pkg/core/typedef/podinfo.go create mode 100644 pkg/core/typedef/rawpod.go create mode 100644 pkg/podmanager/podcache.go create mode 100644 pkg/podmanager/podmanager.go diff --git a/pkg/core/publisher/genericpublisher.go b/pkg/core/publisher/genericpublisher.go index bd170a4..caff433 100644 --- a/pkg/core/publisher/genericpublisher.go +++ b/pkg/core/publisher/genericpublisher.go @@ -104,10 +104,10 @@ func (pub *genericPublisher) Unsubscribe(s api.Subscriber) error { // Publish publishes Event to subscribers interested in specified topic func (pub *genericPublisher) Publish(eventType typedef.EventType, data typedef.Event) { + log.Debugf("publish %s event", eventType.String()) pub.RLock() for id := range pub.topicSubscribersMap[eventType] { pub.subscribers[id](eventType, data) } pub.RUnlock() - log.Debugf("publish %s event", eventType.String()) } diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go new file mode 100644 index 0000000..dc94a75 --- /dev/null +++ b/pkg/core/typedef/containerinfo.go @@ -0,0 +1,53 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines ContainerInfo + +// Package typedef defines core struct and methods for rubik +package typedef + +import "path/filepath" + +// ContainerInfo contains the interested information of container +type ContainerInfo struct { + // Basic Information + Name string `json:"name"` + ID string `json:"id"` + PodID string `json:"podID"` + CgroupRoot string `json:"cgroupRoot"` + CgroupAddr string `json:"cgroupAddr"` +} + +// NewContainerInfo creates a ContainerInfo instance +func NewContainerInfo(container RawContainer, podID, conID, cgroupRoot, podCgroupPath string) *ContainerInfo { + c := ContainerInfo{ + Name: container.Name, + ID: conID, + PodID: podID, + CgroupRoot: cgroupRoot, + CgroupAddr: filepath.Join(podCgroupPath, conID), + } + return &c +} + +// CgroupPath returns cgroup path of specified subsystem of a container +func (ci *ContainerInfo) CgroupPath(subsys string) string { + if ci == nil || ci.Name == "" { + return "" + } + return filepath.Join(ci.CgroupRoot, subsys, ci.CgroupAddr) +} + +// Clone returns deepcopy object. +func (ci *ContainerInfo) Clone() *ContainerInfo { + copy := *ci + return © +} diff --git a/pkg/core/typedef/event.go b/pkg/core/typedef/event.go index 0417765..b25565c 100644 --- a/pkg/core/typedef/event.go +++ b/pkg/core/typedef/event.go @@ -1,25 +1,56 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines event type and event + +// Package typedef defines core struct and methods for rubik package typedef type ( + // the type of event published by generic publisher EventType int8 - Event interface{} + // the event published by generic publisher + Event interface{} ) const ( - ADD EventType = iota - UPDATE - DELETE + // Kubernetes starts a new Pod event + RAW_POD_ADD EventType = iota + // Kubernetes updates Pod event + RAW_POD_UPDATE + // Kubernetes deletes Pod event + RAW_POD_DELETE + // PodManager adds pod information event + INFO_ADD + // PodManager updates pod information event + INFO_UPDATE + // PodManager deletes pod information event + INFO_DELETE ) +const unknownType = "unknown" + +var eventTypeToString = map[EventType]string{ + RAW_POD_ADD: "addrawpod", + RAW_POD_UPDATE: "updaterawpod", + RAW_POD_DELETE: "deleterawpod", + INFO_ADD: "addinfo", + INFO_UPDATE: "updateinfo", + INFO_DELETE: "deleteinfo", +} + +// String returns the string of the current event type func (t EventType) String() string { - switch t { - case ADD: - return "add" - case UPDATE: - return "update" - case DELETE: - return "delete" - default: - return "unknown" + if str, ok := eventTypeToString[t]; ok { + return str } + return unknownType } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go new file mode 100644 index 0000000..b9d50e8 --- /dev/null +++ b/pkg/core/typedef/podinfo.go @@ -0,0 +1,192 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines podInfo + +// Package typedef defines core struct and methods for rubik +package typedef + +import ( + "path/filepath" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/common/constant" +) + +// PodInfo represents pod +type PodInfo struct { + // Basic Information + Containers map[string]*ContainerInfo `json:"containers,omitempty"` + Name string `json:"name"` + UID string `json:"uid"` + CgroupPath string `json:"cgroupPath"` + Namespace string `json:"namespace"` + // TODO: elimate cgroupRoot + CgroupRoot string `json:"cgroupRoot"` + + // TODO: use map[string]interface to replace annotations + // Service Information + Offline bool `json:"offline"` + CacheLimitLevel string `json:"cacheLimitLevel,omitempty"` + + // value of quota burst + QuotaBurst int64 `json:"quotaBurst"` +} + +// NewPodInfo creates the PodInfo instance +func NewPodInfo(pod *RawPod, cgroupRoot string) *PodInfo { + pi := &PodInfo{ + Name: pod.Name, + UID: string(pod.UID), + Containers: make(map[string]*ContainerInfo, 0), + CgroupPath: GetPodCgroupPath(pod), + Namespace: pod.Namespace, + CgroupRoot: cgroupRoot, + } + updatePodInfoNoLock(pi, pod) + return pi +} + +// updatePodInfoNoLock updates PodInfo from the pod of Kubernetes. +// UpdatePodInfoNoLock does not lock pods during the modification. +// Therefore, ensure that the pod is being used only by this function. +// Currently, the checkpoint manager variable is locked when this function is invoked. +func updatePodInfoNoLock(pi *PodInfo, pod *RawPod) { + const ( + dockerPrefix = "docker://" + containerdPrefix = "containerd://" + ) + pi.Name = pod.Name + pi.Offline = IsOffline(pod) + pi.CacheLimitLevel = GetPodCacheLimit(pod) + pi.QuotaBurst = GetQuotaBurst(pod) + + nameID := make(map[string]string, len(pod.Status.ContainerStatuses)) + for _, c := range pod.Status.ContainerStatuses { + // rubik is compatible with dockerd and containerd container engines. + cid := strings.TrimPrefix(c.ContainerID, dockerPrefix) + cid = strings.TrimPrefix(cid, containerdPrefix) + + // the container may be in the creation or deletion phase. + if len(cid) == 0 { + // log.Debugf("no container id found of container %v", c.Name) + continue + } + nameID[c.Name] = cid + } + // update ContainerInfo in a PodInfo + for _, c := range pod.Spec.Containers { + ci, ok := pi.Containers[c.Name] + // add a container + if !ok { + // log.Debugf("add new container %v", c.Name) + pi.AddContainerInfo(NewContainerInfo(&c, string(pod.UID), nameID[c.Name], + pi.CgroupRoot, pi.CgroupPath)) + continue + } + // The container name remains unchanged, and other information about the container is updated. + ci.ID = nameID[c.Name] + ci.CgroupAddr = filepath.Join(pi.CgroupPath, ci.ID) + } + // delete a container that does not exist + for name := range pi.Containers { + if _, ok := nameID[name]; !ok { + // log.Debugf("delete container %v", name) + delete(pi.Containers, name) + } + } +} + +// Clone returns deepcopy object +func (pi *PodInfo) Clone() *PodInfo { + if pi == nil { + return nil + } + copy := *pi + // deepcopy reference object + copy.Containers = make(map[string]*ContainerInfo, len(pi.Containers)) + for _, c := range pi.Containers { + copy.Containers[c.Name] = c.Clone() + } + return © +} + +// AddContainerInfo add container info to pod +func (pi *PodInfo) AddContainerInfo(containerInfo *ContainerInfo) { + // key should not be empty + if containerInfo.Name == "" { + return + } + pi.Containers[containerInfo.Name] = containerInfo +} + +const configHashAnnotationKey = "kubernetes.io/config.hash" + +// IsOffline judges whether pod is offline pod +func IsOffline(pod *RawPod) bool { + return pod.Annotations[constant.PriorityAnnotationKey] == "true" +} + +// GetPodCacheLimit returns cachelimit annotation +func GetPodCacheLimit(pod *RawPod) string { + return pod.Annotations[constant.CacheLimitAnnotationKey] +} + +// ParseInt64 converts the string type to Int64 +func ParseInt64(str string) (int64, error) { + const ( + base = 10 + bitSize = 64 + ) + return strconv.ParseInt(str, base, bitSize) +} + +// GetQuotaBurst checks CPU quota burst annotation value. +func GetQuotaBurst(pod *RawPod) int64 { + quota := pod.Annotations[constant.QuotaBurstAnnotationKey] + if quota == "" { + return constant.InvalidBurst + } + + quotaBurst, err := ParseInt64(quota) + if err != nil { + return constant.InvalidBurst + } + if quotaBurst < 0 { + return constant.InvalidBurst + } + return quotaBurst +} + +// GetPodCgroupPath returns cgroup path of pod +func GetPodCgroupPath(pod *RawPod) string { + var cgroupPath string + id := string(pod.UID) + if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { + id = configHash + } + + switch pod.Status.QOSClass { + case corev1.PodQOSGuaranteed: + cgroupPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBurstable: + cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), + constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBestEffort: + cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), + constant.PodCgroupNamePrefix+id) + } + + return cgroupPath +} diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go new file mode 100644 index 0000000..2d5becb --- /dev/null +++ b/pkg/core/typedef/rawpod.go @@ -0,0 +1,53 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines RawPod which encapsulate kubernetes pods + +// Package typedef defines core struct and methods for rubik +package typedef + +import ( + corev1 "k8s.io/api/core/v1" +) + +// RUNNING means the Pod is in the running phase +const RUNNING = corev1.PodRunning + +type ( + // RawContainer is kubernetes contaienr structure + RawContainer *corev1.Container + // RawPod represents kubernetes pod structure + RawPod corev1.Pod +) + +// StripInfo strips podInfo from RawPod instance +func (pod *RawPod) StripInfo() *PodInfo { + if pod == nil { + return nil + } + return NewPodInfo(pod, "") +} + +// Running return true when pod is in the running phase +func (pod *RawPod) Running() bool { + if pod == nil { + return false + } + return pod.Status.Phase == RUNNING +} + +// ID returns the unique identity of pod +func (pod *RawPod) ID() string { + if pod == nil { + return "" + } + return string(pod.UID) +} diff --git a/pkg/informer/kubeinformer.go b/pkg/informer/kubeinformer.go index 652815c..ec3bc83 100644 --- a/pkg/informer/kubeinformer.go +++ b/pkg/informer/kubeinformer.go @@ -97,13 +97,13 @@ func (ki *kubeInformer) Start(stopCh <-chan struct{}) { } func (ki *kubeInformer) addFunc(obj interface{}) { - ki.Publish(typedef.ADD, obj) + ki.Publish(typedef.RAW_POD_ADD, obj) } func (ki *kubeInformer) updateFunc(oldObj, newObj interface{}) { - ki.Publish(typedef.UPDATE, newObj) + ki.Publish(typedef.RAW_POD_UPDATE, newObj) } func (ki *kubeInformer) deleteFunc(obj interface{}) { - ki.Publish(typedef.DELETE, obj) + ki.Publish(typedef.RAW_POD_DELETE, obj) } diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go new file mode 100644 index 0000000..04a9e79 --- /dev/null +++ b/pkg/podmanager/podcache.go @@ -0,0 +1,88 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-12 +// Description: This file defines pod cache storing pod information + +// Package podmanager implements cache connecting informer and module manager +package podmanager + +import ( + "sync" + + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" +) + +// podCache is used to store PodInfo +type podCache struct { + sync.RWMutex + Pods map[string]*typedef.PodInfo +} + +// NewPodCache returns a PodCache object (pointer) +func NewPodCache() *podCache { + return &podCache{ + Pods: make(map[string]*typedef.PodInfo, 0), + } +} + +// getPod returns the deepcopy object of pod +func (cache *podCache) getPod(podID string) *typedef.PodInfo { + cache.RLock() + defer cache.RUnlock() + return cache.Pods[podID].Clone() +} + +// podExist returns true if there is a pod whose key is podID in the pods +func (cache *podCache) podExist(podID string) bool { + cache.RLock() + _, ok := cache.Pods[podID] + cache.RUnlock() + return ok +} + +// addPod adds pod information +func (cache *podCache) addPod(pod *typedef.PodInfo) { + if pod == nil || pod.UID == "" { + return + } + if ok := cache.podExist(pod.UID); ok { + log.Debugf("pod %v is existed", string(pod.UID)) + return + } + cache.Lock() + cache.Pods[pod.UID] = pod + cache.Unlock() + log.Debugf("add pod %v", string(pod.UID)) +} + +// delPod deletes pod information +func (cache *podCache) delPod(podID string) { + if ok := cache.podExist(podID); !ok { + log.Debugf("pod %v is not existed", string(podID)) + return + } + cache.Lock() + delete(cache.Pods, podID) + cache.Unlock() + log.Debugf("delete pod %v", podID) +} + +// updatePod updates pod information +func (cache *podCache) updatePod(pod *typedef.PodInfo) { + if pod == nil || pod.UID == "" { + return + } + cache.Lock() + cache.Pods[pod.UID] = pod + cache.Unlock() + log.Debugf("update pod %v", pod.UID) +} diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go new file mode 100644 index 0000000..ba777f8 --- /dev/null +++ b/pkg/podmanager/podmanager.go @@ -0,0 +1,157 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-12 +// Description: This file defines PodManager passing and processing raw pod data + +// Package podmanager implements manager connecting informer and module manager +package podmanager + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/subscriber" + "isula.org/rubik/pkg/core/typedef" +) + +// PodManagerName is the unique identity of PodManager +const PodManagerName = "DefaultPodManager" + +// PodManager manages pod cache and pushes cache change events based on external input +type PodManager struct { + api.Subscriber + api.Publisher + pods *podCache +} + +// NewPodManager returns a PodManager pointer +func NewPodManager(publisher api.Publisher) *PodManager { + manager := &PodManager{ + pods: NewPodCache(), + Publisher: publisher, + } + manager.Subscriber = subscriber.NewGenericSubscriber(manager, PodManagerName) + return manager +} + +// HandleEvent handles the event from publisher +func (manager *PodManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { + pod, err := eventToRawPod(event) + if err != nil { + log.Warnf(err.Error()) + return + } + switch eventType { + case typedef.RAW_POD_ADD: + manager.addFunc(pod) + case typedef.RAW_POD_UPDATE: + manager.updateFunc(pod) + case typedef.RAW_POD_DELETE: + manager.deleteFunc(pod) + default: + log.Infof("fail to process %s type event", eventType.String()) + } +} + +// EventTypes returns the intersted event types +func (manager *PodManager) EventTypes() []typedef.EventType { + return []typedef.EventType{typedef.RAW_POD_ADD, typedef.RAW_POD_UPDATE, typedef.RAW_POD_DELETE} +} + +// eventToRawPod converts the event interface to RawPod pointer +func eventToRawPod(e typedef.Event) (*typedef.RawPod, error) { + pod, ok := e.(*corev1.Pod) + if !ok { + return nil, fmt.Errorf("fail to get *typedef.RawPod which type is %T", e) + } + rawPod := typedef.RawPod(*pod) + return &rawPod, nil +} + +// addFunc handles the pod add event +func (manager *PodManager) addFunc(pod *typedef.RawPod) { + // condition 1: only add running pod + if !pod.Running() { + log.Debugf("pod %v is not running", pod.UID) + return + } + // condition2: pod is not existed + if manager.pods.podExist(pod.ID()) { + log.Debugf("pod %v has added", pod.UID) + return + } + // step1: get pod information + podInfo := pod.StripInfo() + if podInfo == nil { + log.Errorf("fail to strip info from raw pod") + return + } + // step2. add pod information + manager.tryAdd(podInfo) +} + +// updateFunc handles the pod update event +func (manager *PodManager) updateFunc(pod *typedef.RawPod) { + // step1: delete existed but not running pod + if !pod.Running() { + manager.tryDelete(pod.ID()) + return + } + + // add or update information for running pod + podInfo := pod.StripInfo() + if podInfo == nil { + log.Errorf("fail to strip info from raw pod") + return + } + // The calling order must be updated first and then added + // step2: process exsited and running pod + manager.tryUpdate(podInfo) + // step3: process not exsited and running pod + manager.tryAdd(podInfo) +} + +// deleteFunc handles the pod delete event +func (manager *PodManager) deleteFunc(pod *typedef.RawPod) { + manager.tryDelete(pod.ID()) +} + +// tryAdd tries to add pod info which is not added +func (manager *PodManager) tryAdd(podInfo *typedef.PodInfo) { + // only add when pod is not existed + if !manager.pods.podExist(podInfo.UID) { + manager.pods.addPod(podInfo) + manager.Publish(typedef.INFO_ADD, podInfo) + } +} + +// tryUpdate tries to update podinfo which is existed +func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { + // only update when pod is existed + if manager.pods.podExist(podInfo.UID) { + oldPod := manager.pods.getPod(podInfo.UID) + manager.pods.updatePod(podInfo) + manager.Publish(typedef.INFO_UPDATE, []interface{}{oldPod, podInfo.Clone()}) + } +} + +// tryDelete tries to delete podinfo which is existed +func (manager *PodManager) tryDelete(id string) { + // only delete when pod is existed + oldPod := manager.pods.getPod(id) + if oldPod != nil { + manager.pods.delPod(id) + manager.Publish(typedef.INFO_DELETE, oldPod) + } +} -- Gitee From 3ca4930cc7d00d1f9edd05d8ab28e2014b8f52c2 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 13 Jan 2023 20:44:55 +0800 Subject: [PATCH 09/73] Decoupling configuration and service (original module) Reason: The globalization of service configuration does not conform to the principle of opening and closing 1. The sub-features are uniformly named as services instead of modules 2. Rewrite Config to support key-value parsing configuration 1). The service name is uniquely identified, 2). Support multi-parser extension. 3.services add registry for registering services 4.services add serviceManager to manage services --- pkg/api/api.go | 12 +- pkg/common/constant/constant.go | 19 +- pkg/config/config.go | 111 ++++++++++++ pkg/config/config_test.go | 79 ++++++++ pkg/config/jsonparser.go | 45 +++++ pkg/config/parserfactory.go | 28 +++ pkg/modules/checkpoint/checkpoint.go | 70 ------- pkg/modules/config/config.go | 171 ------------------ pkg/modules/cpu/cpu.go | 65 ------- pkg/modules/iocost/iocost.go | 65 ------- pkg/modules/memory/memory.go | 65 ------- pkg/modules/podinformer/podinformer.go | 92 ---------- pkg/modules/quota/cpuBurst.go | 1 - pkg/modules/quota/quotaTurbo.go | 1 - pkg/rubik/import.go | 6 +- pkg/{modules => services}/blkio/blkio.go | 8 +- .../cachelimit/cachelimit.go | 53 +++++- pkg/services/registry.go | 42 +++++ pkg/services/servicemanager.go | 92 ++++++++++ 19 files changed, 476 insertions(+), 549 deletions(-) create mode 100644 pkg/config/config.go create mode 100644 pkg/config/config_test.go create mode 100644 pkg/config/jsonparser.go create mode 100644 pkg/config/parserfactory.go delete mode 100644 pkg/modules/checkpoint/checkpoint.go delete mode 100644 pkg/modules/config/config.go delete mode 100644 pkg/modules/cpu/cpu.go delete mode 100644 pkg/modules/iocost/iocost.go delete mode 100644 pkg/modules/memory/memory.go delete mode 100644 pkg/modules/podinformer/podinformer.go delete mode 100644 pkg/modules/quota/cpuBurst.go delete mode 100644 pkg/modules/quota/quotaTurbo.go rename pkg/{modules => services}/blkio/blkio.go (83%) rename pkg/{modules => services}/cachelimit/cachelimit.go (41%) create mode 100644 pkg/services/registry.go create mode 100644 pkg/services/servicemanager.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 70f6934..7b4575f 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -30,7 +30,7 @@ type Registry interface { ListServices() ([]*Service, error) } -// Service contains progress that all services(modules) need to have +// Service contains progress that all services need to have type Service interface { Init() error Setup() error @@ -39,6 +39,16 @@ type Service interface { PodEventHandler() error } +type PersistentService interface { + Init() error + Start(stopCh <-chan struct{}) +} + +type ConfigParser interface { + ParseConfig(data []byte) (map[string]interface{}, error) + UnmarshalSubConfig(data interface{}, v interface{}) error +} + // PodEventSubscriber control pod activities type PodEventSubscriber interface { AddPod(pod *corev1.Pod) diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index a1d9476..f016532 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -25,8 +25,6 @@ const ( RubikSock = "/run/rubik/rubik.sock" // ConfigFile is rubik config file ConfigFile = "/var/lib/rubik/config.json" - // DefaultLogDir is default log dir - DefaultLogDir = "/var/log/rubik" // LockFile is rubik lock file LockFile = "/run/rubik/rubik.lock" // ReadTimeout is timeout for http read @@ -81,6 +79,23 @@ const ( DefaultMemStrategy = "none" ) +// log config +const ( + LogDriverStdio = "stdio" + LogDriverFile = "file" + + LogLevelDebug = "debug" + LogLevelInfo = "info" + LogLevelWarn = "warn" + LogLevelError = "error" + LogLevelStack = "stack" + + // DefaultLogDir is default log dir + DefaultLogDir = "/var/log/rubik" + DefaultLogLevel = LogLevelInfo + DefaultLogSize = 1024 +) + // LevelType is type definition of qos level type LevelType int32 diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..02945c0 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,111 @@ +package config + +import ( + "fmt" + "io/ioutil" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/services" +) + +const agentKey = "agent" + +// sysConfKeys saves the system configuration key, which is the service name except +var sysConfKeys = map[string]struct{}{ + agentKey: {}, +} + +// Config saves all configuration information of rubik +type Config struct { + api.ConfigParser + Agent *AgentConfig + Fields map[string]interface{} +} + +// AgentConfig is the configuration of rubik, including important basic configurations such as logs +type AgentConfig struct { + LogDriver string `json:"logDriver,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + LogSize int64 `json:"logSize,omitempty"` + LogDir string `json:"logDir,omitempty"` + CgroupRoot string `json:"cgroupRoot,omitempty"` +} + +// NewConfig returns an config object pointer +func NewConfig(pType parserType) *Config { + c := &Config{ + ConfigParser: defaultParserFactory.getParser(pType), + Agent: &AgentConfig{ + LogDriver: constant.DefaultLogDir, + LogSize: constant.DefaultLogSize, + LogLevel: constant.DefaultLogLevel, + LogDir: constant.DefaultLogDir, + CgroupRoot: constant.DefaultCgroupRoot, + }, + } + return c +} + +// loadConfigFile loads data from configuration file +func loadConfigFile(config string) ([]byte, error) { + buffer, err := ioutil.ReadFile(config) + if err != nil { + return nil, err + } + return buffer, nil +} + +// parseAgentConfig parses config as AgentConfig +func (c *Config) parseAgentConfig() { + content, ok := c.Fields[agentKey] + if !ok { + return + } + c.UnmarshalSubConfig(content, c.Agent) +} + +// LoadConfig loads and parses configuration data from the file, and save it to the Config +func (c *Config) LoadConfig(path string) error { + if path == "" { + path = constant.ConfigFile + } + data, err := loadConfigFile(path) + if err != nil { + return fmt.Errorf("error loading config file %s: %w", path, err) + } + fields, err := c.ParseConfig(data) + if err != nil { + return fmt.Errorf("error parsing data: %s", err) + } + c.Fields = fields + c.parseAgentConfig() + return nil +} + +// filterNonServiceKeys returns true when inputting a non-service name +func (c *Config) filterNonServiceKeys(name string) bool { + // 1. ignore system configured key values + _, ok := sysConfKeys[name] + return ok +} + +// PrepareServices parses the to-be-run services config and loads them to the ServiceManager +func (c *Config) PrepareServices() error { + // TODO: Later, consider placing the function book in rubik.go + for name, config := range c.Fields { + if c.filterNonServiceKeys(name) { + continue + } + creator := services.GetServiceCreator(name) + if creator == nil { + return fmt.Errorf("no corresponding module %v, please check the module name", name) + } + service := creator() + if err := c.UnmarshalSubConfig(config, service); err != nil { + return fmt.Errorf("error unmarshaling %s config: %v", name, err) + } + services.AddRunningService(name, service) + } + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000..94a0dc8 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,79 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/services" + _ "isula.org/rubik/pkg/services/blkio" + _ "isula.org/rubik/pkg/services/cachelimit" +) + +var rubikConfig string = ` +{ + "agent": { + "logDriver": "stdio", + "logDir": "/var/log/rubik", + "logSize": 2048, + "logLevel": "info" + }, + "blkio":{}, + "cacheLimit": { + "defaultLimitMode": "static", + "adjustInterval": 1000, + "perfDuration": 1000, + "l3Percent": { + "low": 20, + "mid": 30, + "high": 50 + }, + "memBandPercent": { + "low": 10, + "mid": 30, + "high": 50 + } + } +} +` + +func TestPrepareServices(t *testing.T) { + + if !util.PathExist(constant.TmpTestDir) { + if err := os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode); err != nil { + assert.NoError(t, err) + } + } + + defer os.RemoveAll(constant.TmpTestDir) + + tmpConfigFile := filepath.Join(constant.TmpTestDir, "config.json") + defer os.Remove(tmpConfigFile) + if err := ioutil.WriteFile(tmpConfigFile, []byte(rubikConfig), constant.DefaultFileMode); err != nil { + assert.NoError(t, err) + return + } + + c := NewConfig(JSON) + if err := c.LoadConfig(tmpConfigFile); err != nil { + assert.NoError(t, err) + return + } + if err := c.PrepareServices(); err != nil { + assert.NoError(t, err) + return + } + fmt.Printf("agent: %v\n", c.Agent) + for name, service := range services.GetServiceManager().RunningServices { + fmt.Printf("name: %s, service: %v\n", name, service) + } + for name, service := range services.GetServiceManager().RunningPersistentServices { + fmt.Printf("name: %s, persistent service: %v\n", name, service) + } +} diff --git a/pkg/config/jsonparser.go b/pkg/config/jsonparser.go new file mode 100644 index 0000000..7c37a11 --- /dev/null +++ b/pkg/config/jsonparser.go @@ -0,0 +1,45 @@ +package config + +import ( + "encoding/json" + "fmt" +) + +// defaultJsonParser is globally unique json parser +var defaultJsonParser *jsonParser + +// jsonParser is used to parse json +type jsonParser struct{} + +//getJsonParser gets the globally unique json parser +func getJsonParser() *jsonParser { + if defaultJsonParser == nil { + defaultJsonParser = &jsonParser{} + } + return defaultJsonParser +} + +// ParseConfig parses json data as map[string]interface{} +func (parser *jsonParser) ParseConfig(data []byte) (map[string]interface{}, error) { + m := make(map[string]interface{}) + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + return m, nil +} + +// UnmarshalSubConfig deserializes interface to structure +func (p *jsonParser) UnmarshalSubConfig(data interface{}, v interface{}) error { + // 1. convert map[string]interface to json string + val, ok := data.(map[string]interface{}) + if !ok { + fmt.Printf("invalid type %T\n", data) + return fmt.Errorf("invalid type %T", data) + } + jsonString, err := json.Marshal(val) + if err != nil { + return err + } + // 2. convert json string to struct + return json.Unmarshal(jsonString, v) +} diff --git a/pkg/config/parserfactory.go b/pkg/config/parserfactory.go new file mode 100644 index 0000000..9f3d22b --- /dev/null +++ b/pkg/config/parserfactory.go @@ -0,0 +1,28 @@ +package config + +import "isula.org/rubik/pkg/api" + +type ( + // parserType represents the parser type + parserType int8 + // parserFactory is the factory class of the parser + parserFactory struct{} +) + +const ( + // JSON represents the json type parser + JSON parserType = iota +) + +// defaultParserFactory is globally unique parser factory +var defaultParserFactory = &parserFactory{} + +// getParser gets parser instance according to the parser type passed in +func (factory *parserFactory) getParser(pType parserType) api.ConfigParser { + switch pType { + case JSON: + return getJsonParser() + default: + return getJsonParser() + } +} diff --git a/pkg/modules/checkpoint/checkpoint.go b/pkg/modules/checkpoint/checkpoint.go deleted file mode 100644 index 263923c..0000000 --- a/pkg/modules/checkpoint/checkpoint.go +++ /dev/null @@ -1,70 +0,0 @@ -package checkpoint - -import ( - "fmt" - - "isula.org/rubik/pkg/api" - "isula.org/rubik/pkg/common/typedef" - corev1 "k8s.io/api/core/v1" -) - -type CheckPoint struct { - Pods map[string]*typedef.PodInfo `json:"pods,omitempty"` -} - -type CheckPointViewer struct { - checkpoint CheckPoint - subscribers []api.PodEventSubscriber -} - -type CheckPointPublisher struct { - checkpoint *CheckPoint - subscribers []api.PodEventSubscriber - viewer CheckPointViewer -} - -var DefaultCheckPointPublisher = NewCheckPointPublisher() - -func NewCheckPointPublisher() *CheckPointPublisher { - return &CheckPointPublisher{ - checkpoint: &CheckPoint{ - Pods: make(map[string]*typedef.PodInfo), - }, - subscribers: make([]api.PodEventSubscriber, 0), - } -} - -func (cpp *CheckPointPublisher) Subscribe(s api.PodEventSubscriber) { - fmt.Printf("CheckPointPublisher subscribe(%s)\n", s.ID()) - cpp.subscribers = append(cpp.subscribers, s) -} - -func (cpp *CheckPointPublisher) Unsubscribe(s api.PodEventSubscriber) { - fmt.Printf("CheckPointPublisher unsubscribe(%s)\n", s.ID()) - subscribersLength := len(cpp.subscribers) - for i, subscriber := range cpp.subscribers { - if s.ID() == subscriber.ID() { - cpp.subscribers[subscribersLength-1], cpp.subscribers[i] = cpp.subscribers[i], cpp.subscribers[subscribersLength-1] - cpp.subscribers = cpp.subscribers[:subscribersLength-1] - break - } - } -} - -func (cpp *CheckPointPublisher) NotifySubscribers() { - fmt.Printf("CheckPointPublisher notifyAll()\n") -} - -func (cpp *CheckPointPublisher) ReceivePodEvent(eventType string, pod *corev1.Pod) { - fmt.Printf("CheckPointPublisher ReceivePodEvent(%s)\n", eventType) -} - -func (cv *CheckPointViewer) ListOnlinePods() ([]*corev1.Pod, error) { - fmt.Printf("CheckPointViewer ListOnlinePods()\n") - return nil, nil -} - -func (cv *CheckPointViewer) ListOfflinePods() ([]*corev1.Pod, error) { - fmt.Printf("CheckPointViewer ListOfflinePods()\n") - return nil, nil -} diff --git a/pkg/modules/config/config.go b/pkg/modules/config/config.go deleted file mode 100644 index c1edc6c..0000000 --- a/pkg/modules/config/config.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-04-26 -// Description: config load - -package config - -import ( - "bytes" - "encoding/json" - "path/filepath" - - "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/util" -) - -var ( - // CgroupRoot is cgroup mount point - CgroupRoot = constant.DefaultCgroupRoot - // ShutdownFlag is rubik shutdown flag - ShutdownFlag int32 - // ShutdownChan is rubik shutdown channel - ShutdownChan = make(chan struct{}) -) - -// Config defines the configuration for rubik -type Config struct { - AutoCheck bool `json:"autoCheck,omitempty"` - LogDriver string `json:"logDriver,omitempty"` - LogDir string `json:"logDir,omitempty"` - LogSize int `json:"logSize,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - CgroupRoot string `json:"cgroupRoot,omitempty"` - CacheCfg CacheConfig `json:"cacheConfig,omitempty"` - BlkioCfg BlkioConfig `json:"blkioConfig,omitempty"` - MemCfg MemoryConfig `json:"memoryConfig,omitempty"` - NodeConfig []NodeConfig `json:"nodeConfig,omitempty"` -} - -// CacheConfig define cache limit related config -type CacheConfig struct { - Enable bool `json:"enable,omitempty"` - DefaultLimitMode string `json:"defaultLimitMode,omitempty"` - DefaultResctrlDir string `json:"-"` - AdjustInterval int `json:"adjustInterval,omitempty"` - PerfDuration int `json:"perfDuration,omitempty"` - L3Percent MultiLvlPercent `json:"l3Percent,omitempty"` - MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` -} - -// BlkioConfig defines blkio related configurations. -type BlkioConfig struct { - Enable bool `json:"enable,omitempty"` -} - -// MultiLvlPercent define multi level percentage -type MultiLvlPercent struct { - Low int `json:"low,omitempty"` - Mid int `json:"mid,omitempty"` - High int `json:"high,omitempty"` -} - -type MemoryConfig struct { - Enable bool `json:"enable,omitempty"` - Strategy string `json:"strategy,omitempty"` - CheckInterval int `json:"checkInterval,omitempty"` -} - -// NodeConfig define node configuration for each node -type NodeConfig struct { - NodeName string `json:"nodeName,omitempty"` - IOcostEnable bool `json:"iocostEnable,omitempty"` - IOcostConfig []IOcostConfig `json:"iocostConfig,omitempty"` -} - -// IOcostConfig define iocost for node -type IOcostConfig struct { - Dev string `json:"dev,omitempty"` - Enable bool `json:"enable,omitempty"` - Model string `json:"model,omitempty"` - Param Param `json:"param,omitempty"` -} - -// Param for linear model -type Param struct { - Rbps int64 `json:"rbps,omitempty"` - Rseqiops int64 `json:"rseqiops,omitempty"` - Rrandiops int64 `json:"rrandiops,omitempty"` - Wbps int64 `json:"wbps,omitempty"` - Wseqiops int64 `json:"wseqiops,omitempty"` - Wrandiops int64 `json:"wrandiops,omitempty"` -} - -// NewConfig returns new config load from config file -func NewConfig(path string) (*Config, error) { - if path == "" { - path = constant.ConfigFile - } - - defaultLogSize, defaultAdInt, defaultPerfDur := 1024, 1000, 1000 - defaultLowL3, defaultMidL3, defaultHighL3, defaultLowMB, defaultMidMB, defaultHighMB := 20, 30, 50, 10, 30, 50 - cfg := Config{ - LogDriver: "stdio", - LogDir: constant.DefaultLogDir, - LogSize: defaultLogSize, - LogLevel: "info", - CgroupRoot: constant.DefaultCgroupRoot, - CacheCfg: CacheConfig{ - Enable: false, - DefaultLimitMode: "static", - DefaultResctrlDir: "/sys/fs/resctrl", - AdjustInterval: defaultAdInt, - PerfDuration: defaultPerfDur, - L3Percent: MultiLvlPercent{ - Low: defaultLowL3, - Mid: defaultMidL3, - High: defaultHighL3, - }, - MemBandPercent: MultiLvlPercent{ - Low: defaultLowMB, - Mid: defaultMidMB, - High: defaultHighMB, - }, - }, - BlkioCfg: BlkioConfig{ - Enable: false, - }, - MemCfg: MemoryConfig{ - Enable: false, - Strategy: constant.DefaultMemStrategy, - CheckInterval: constant.DefaultMemCheckInterval, - }, - } - - defer func() { - CgroupRoot = cfg.CgroupRoot - }() - - if !util.PathExist(path) { - return &cfg, nil - } - - b, err := util.ReadSmallFile(filepath.Clean(path)) - if err != nil { - return nil, err - } - - reader := bytes.NewReader(b) - if err := json.NewDecoder(reader).Decode(&cfg); err != nil { - return nil, err - } - - return &cfg, nil -} - -// String return string format. -func (cfg *Config) String() string { - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return "{}" - } - return string(data) -} diff --git a/pkg/modules/cpu/cpu.go b/pkg/modules/cpu/cpu.go deleted file mode 100644 index 395fe91..0000000 --- a/pkg/modules/cpu/cpu.go +++ /dev/null @@ -1,65 +0,0 @@ -package cpu - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - - "isula.org/rubik/pkg/modules/checkpoint" - "isula.org/rubik/pkg/registry" -) - -type CPU struct { - Name string -} - -func init() { - registry.DefaultRegister.Register(NewCPU(), "cpu") -} - -func NewCPU() *CPU { - return &CPU{Name: "cpu"} -} - -func (c *CPU) Init() error { - fmt.Println("cpu Init()") - return nil -} - -func (c *CPU) Setup() error { - fmt.Println("cpu Setup()") - return nil -} - -func (c *CPU) Run() error { - fmt.Println("cpu Run()") - return nil -} - -func (c *CPU) TearDown() error { - fmt.Println("cpu TearDown()") - return nil -} - -func (c *CPU) AddPod(pod *corev1.Pod) { - fmt.Println("cpu AddPod()") -} - -func (c *CPU) UpdatePod(pod *corev1.Pod) { - fmt.Println("cpu UpdatePod()") -} - -func (c *CPU) DeletePod(podID types.UID) { - fmt.Println("cpu DeletePod()") -} - -func (c *CPU) ID() string { - return c.Name -} - -func (c *CPU) PodEventHandler() error { - fmt.Println("cpu PodEventHandler") - checkpoint.DefaultCheckPointPublisher.Subscribe(c) - return nil -} diff --git a/pkg/modules/iocost/iocost.go b/pkg/modules/iocost/iocost.go deleted file mode 100644 index ef2b118..0000000 --- a/pkg/modules/iocost/iocost.go +++ /dev/null @@ -1,65 +0,0 @@ -package iocost - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - - "isula.org/rubik/pkg/modules/checkpoint" - "isula.org/rubik/pkg/registry" -) - -type IOCost struct { - Name string -} - -func init() { - registry.DefaultRegister.Register(NewIOCost(), "iocost") -} - -func NewIOCost() *IOCost { - return &IOCost{Name: "iocost"} -} - -func (i *IOCost) Init() error { - fmt.Println("iocost Init()") - return nil -} - -func (i *IOCost) Setup() error { - fmt.Println("iocost Setup()") - return nil -} - -func (i *IOCost) Run() error { - fmt.Println("iocost Run()") - return nil -} - -func (i *IOCost) TearDown() error { - fmt.Println("iocost TearDown()") - return nil -} - -func (i *IOCost) AddPod(pod *corev1.Pod) { - fmt.Println("iocost AddPod()") -} - -func (i *IOCost) UpdatePod(pod *corev1.Pod) { - fmt.Println("iocost UpdatePod()") -} - -func (i *IOCost) DeletePod(podID types.UID) { - fmt.Println("iocost DeletePod()") -} - -func (i *IOCost) ID() string { - return i.Name -} - -func (i *IOCost) PodEventHandler() error { - fmt.Println("iosoct PodEventHandler") - checkpoint.DefaultCheckPointPublisher.Subscribe(i) - return nil -} diff --git a/pkg/modules/memory/memory.go b/pkg/modules/memory/memory.go deleted file mode 100644 index 1646b02..0000000 --- a/pkg/modules/memory/memory.go +++ /dev/null @@ -1,65 +0,0 @@ -package memory - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - - "isula.org/rubik/pkg/modules/checkpoint" - "isula.org/rubik/pkg/registry" -) - -type Memory struct { - Name string -} - -func init() { - registry.DefaultRegister.Register(NewMemory(), "memory") -} - -func NewMemory() *Memory { - return &Memory{Name: "memory"} -} - -func (m *Memory) Init() error { - fmt.Println("memory Init()") - return nil -} - -func (m *Memory) Setup() error { - fmt.Println("memory Setup()") - return nil -} - -func (m *Memory) Run() error { - fmt.Println("memory Run()") - return nil -} - -func (m *Memory) TearDown() error { - fmt.Println("memory TearDown()") - return nil -} - -func (m *Memory) AddPod(pod *corev1.Pod) { - fmt.Println("memory AddPod()") -} - -func (m *Memory) UpdatePod(pod *corev1.Pod) { - fmt.Println("memory UpdatePod()") -} - -func (m *Memory) DeletePod(podID types.UID) { - fmt.Println("memory DeletePod()") -} - -func (m *Memory) ID() string { - return m.Name -} - -func (m *Memory) PodEventHandler() error { - fmt.Println("memory PodEventHandler()") - checkpoint.DefaultCheckPointPublisher.Subscribe(m) - return nil -} diff --git a/pkg/modules/podinformer/podinformer.go b/pkg/modules/podinformer/podinformer.go deleted file mode 100644 index 90de756..0000000 --- a/pkg/modules/podinformer/podinformer.go +++ /dev/null @@ -1,92 +0,0 @@ -package podinformer - -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-07-22 -// Description: qos auto config - -import ( - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - log "isula.org/rubik/pkg/common/tinylog" - "isula.org/rubik/pkg/modules/config" -) - -const invalidErr = "Auto config error: invalid pod type" - -// EventHandler is used to process pod events pushed by Kubernetes APIServer. -type EventHandler interface { - AddEvent(pod *corev1.Pod) - UpdateEvent(oldPod *corev1.Pod, newPod *corev1.Pod) - DeleteEvent(pod *corev1.Pod) -} - -// Backend is Rubik struct. -var Backend EventHandler - -// Init initializes the callback function for the pod event. -func Init(kubeClient *kubernetes.Clientset, nodeName string) error { - const ( - reSyncTime = 30 - specNodeNameField = "spec.nodeName" - ) - kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, - time.Duration(reSyncTime)*time.Second, - informers.WithTweakListOptions(func(options *metav1.ListOptions) { - // set Options to return only pods on the current node. - options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, nodeName).String() - })) - kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: addHandler, - UpdateFunc: updateHandler, - DeleteFunc: deleteHandler, - }) - kubeInformerFactory.Start(config.ShutdownChan) - return nil -} - -func addHandler(obj interface{}) { - pod, ok := obj.(*corev1.Pod) - if !ok { - log.Errorf(invalidErr) - return - } - - Backend.AddEvent(pod) -} - -func updateHandler(old, new interface{}) { - oldPod, ok1 := old.(*corev1.Pod) - newPod, ok2 := new.(*corev1.Pod) - if !ok1 || !ok2 { - log.Errorf(invalidErr) - return - } - - Backend.UpdateEvent(oldPod, newPod) -} - -func deleteHandler(obj interface{}) { - pod, ok := obj.(*corev1.Pod) - if !ok { - log.Errorf(invalidErr) - return - } - - Backend.DeleteEvent(pod) -} diff --git a/pkg/modules/quota/cpuBurst.go b/pkg/modules/quota/cpuBurst.go deleted file mode 100644 index 0dcd561..0000000 --- a/pkg/modules/quota/cpuBurst.go +++ /dev/null @@ -1 +0,0 @@ -package quota \ No newline at end of file diff --git a/pkg/modules/quota/quotaTurbo.go b/pkg/modules/quota/quotaTurbo.go deleted file mode 100644 index e63c4f8..0000000 --- a/pkg/modules/quota/quotaTurbo.go +++ /dev/null @@ -1 +0,0 @@ -package quota diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index 2b846e0..0eb7826 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -2,10 +2,6 @@ package rubik import ( // import packages to auto register service - _ "isula.org/rubik/pkg/modules/blkio" - _ "isula.org/rubik/pkg/modules/cachelimit" - _ "isula.org/rubik/pkg/modules/cpu" - _ "isula.org/rubik/pkg/modules/iocost" - _ "isula.org/rubik/pkg/modules/memory" + _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/version" ) diff --git a/pkg/modules/blkio/blkio.go b/pkg/services/blkio/blkio.go similarity index 83% rename from pkg/modules/blkio/blkio.go rename to pkg/services/blkio/blkio.go index e976678..7a87241 100644 --- a/pkg/modules/blkio/blkio.go +++ b/pkg/services/blkio/blkio.go @@ -6,8 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/modules/checkpoint" - "isula.org/rubik/pkg/registry" + "isula.org/rubik/pkg/services" ) type Blkio struct { @@ -15,7 +14,9 @@ type Blkio struct { } func init() { - registry.DefaultRegister.Register(NewBlkio(), "blkio") + services.Register("blkio", func() interface{} { + return NewBlkio() + }) } func NewBlkio() *Blkio { @@ -60,6 +61,5 @@ func (b *Blkio) ID() string { func (b *Blkio) PodEventHandler() error { fmt.Println("blkio PodEventHandler") - checkpoint.DefaultCheckPointPublisher.Subscribe(b) return nil } diff --git a/pkg/modules/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go similarity index 41% rename from pkg/modules/cachelimit/cachelimit.go rename to pkg/services/cachelimit/cachelimit.go index 3545cee..97bb9b6 100644 --- a/pkg/modules/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -6,20 +6,61 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "isula.org/rubik/pkg/modules/checkpoint" - "isula.org/rubik/pkg/registry" + "isula.org/rubik/pkg/services" ) +const ( + defaultLogSize = 1024 + defaultAdInt = 1000 + defaultPerfDur = 1000 + defaultLowL3 = 20 + defaultMidL3 = 30 + defaultHighL3 = 50 + defaultLowMB = 10 + defaultMidMB = 30 + defaultHighMB = 50 +) + +// MultiLvlPercent define multi level percentage +type MultiLvlPercent struct { + Low int `json:"low,omitempty"` + Mid int `json:"mid,omitempty"` + High int `json:"high,omitempty"` +} + type CacheLimit struct { - Name string + Name string `json:"-"` + DefaultLimitMode string `json:"defaultLimitMode,omitempty"` + DefaultResctrlDir string `json:"-"` + AdjustInterval int `json:"adjustInterval,omitempty"` + PerfDuration int `json:"perfDuration,omitempty"` + L3Percent MultiLvlPercent `json:"l3Percent,omitempty"` + MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` } func init() { - registry.DefaultRegister.Register(NewCacheLimit(), "cacheLimit") + services.Register("cacheLimit", func() interface{} { + return NewCacheLimit() + }) } func NewCacheLimit() *CacheLimit { - return &CacheLimit{Name: "cacheLimit"} + return &CacheLimit{ + DefaultLimitMode: "static", + DefaultResctrlDir: "/sys/fs/resctrl", + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{ + Low: defaultLowL3, + Mid: defaultMidL3, + High: defaultHighL3, + }, + MemBandPercent: MultiLvlPercent{ + Low: defaultLowMB, + Mid: defaultMidMB, + High: defaultHighMB, + }, + } } func (c *CacheLimit) Init() error { @@ -59,7 +100,5 @@ func (c *CacheLimit) ID() string { } func (c *CacheLimit) PodEventHandler() error { - fmt.Println("cache limit PodEventHandler") - checkpoint.DefaultCheckPointPublisher.Subscribe(c) return nil } diff --git a/pkg/services/registry.go b/pkg/services/registry.go new file mode 100644 index 0000000..001dda8 --- /dev/null +++ b/pkg/services/registry.go @@ -0,0 +1,42 @@ +package services + +import ( + "sync" + + "isula.org/rubik/pkg/common/log" +) + +type ( + // Creator creates Service objects + Creator func() interface{} + // registry is used for service registration + registry struct { + sync.RWMutex + // services is a collection of all registered service + services map[string]Creator + } +) + +// servicesRegistry is the globally unique registry +var servicesRegistry = ®istry{ + services: make(map[string]Creator, 0), +} + +// Register is used to register the service creators +func Register(name string, creator Creator) { + servicesRegistry.Lock() + servicesRegistry.services[name] = creator + servicesRegistry.Unlock() + log.Debugf("func register (%s)\n", name) +} + +// GetServiceCreator returns the service creator based on the incoming service name +func GetServiceCreator(name string) Creator { + servicesRegistry.RLock() + creator, ok := servicesRegistry.services[name] + servicesRegistry.RUnlock() + if !ok { + return nil + } + return creator +} diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go new file mode 100644 index 0000000..8772806 --- /dev/null +++ b/pkg/services/servicemanager.go @@ -0,0 +1,92 @@ +package services + +import ( + "sync" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/subscriber" + "isula.org/rubik/pkg/core/typedef" +) + +// serviceManagerName is the unique ID of the service manager +const serviceManagerName = "serviceManager" + +// AddRunningService adds a to-be-run service +func AddRunningService(name string, service interface{}) { + serviceManager.RLock() + _, ok1 := serviceManager.RunningServices[name] + _, ok2 := serviceManager.RunningPersistentServices[name] + serviceManager.RUnlock() + if ok1 || ok2 { + log.Errorf("service name conflict : \"%s\"", name) + return + } + + if !serviceManager.tryAddService(name, service) && !serviceManager.tryAddPersistentService(name, service) { + log.Errorf("invalid service : \"%s\", %T", name, service) + return + } + log.Debugf("pre-start service %s", name) +} + +type ServiceManager struct { + api.Subscriber + sync.RWMutex + RunningServices map[string]api.Service + RunningPersistentServices map[string]api.PersistentService +} + +var serviceManager = newServiceManager() + +func newServiceManager() *ServiceManager { + manager := &ServiceManager{ + RunningServices: make(map[string]api.Service), + RunningPersistentServices: make(map[string]api.PersistentService), + } + manager.Subscriber = subscriber.NewGenericSubscriber(manager, serviceManagerName) + return manager +} + +// GetServiceManager returns the globally unique ServiceManager +func GetServiceManager() *ServiceManager { + return serviceManager +} + +func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { + switch eventType { + case typedef.INFO_ADD: + case typedef.INFO_UPDATE: + case typedef.INFO_DELETE: + default: + log.Infof("service manager fail to process %s type", eventType.String()) + } +} + +func (cmanager *ServiceManager) EventTypes() []typedef.EventType { + return []typedef.EventType{typedef.INFO_ADD, typedef.INFO_UPDATE, typedef.INFO_DELETE} +} + +// tryAddService determines whether it is a api.Service and adds it to the queue to be run +func (manager *ServiceManager) tryAddService(name string, service interface{}) bool { + s, ok := service.(api.Service) + if ok { + serviceManager.Lock() + manager.RunningServices[name] = s + serviceManager.Unlock() + log.Debugf("service %s will run", name) + } + return ok +} + +// tryAddPersistentService determines whether it is a api.PersistentService and adds it to the queue to be run +func (manager *ServiceManager) tryAddPersistentService(name string, service interface{}) bool { + s, ok := service.(api.PersistentService) + if ok { + serviceManager.Lock() + manager.RunningPersistentServices[name] = s + serviceManager.Unlock() + log.Debugf("persistent service %s will run", name) + } + return ok +} -- Gitee From 56e6716872720f3f026030b29d862d9d63d72789 Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 17 Jan 2023 09:30:52 +0800 Subject: [PATCH 10/73] optimize the constants in log --- pkg/common/log/log.go | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index 0b8a53b..f5f8fb9 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -35,12 +35,8 @@ const ( // UUID is log uuid UUID = "uuid" - logDriverStdio = "stdio" - logDriverFile = "file" - logStack = 20 - logStackFrom = 2 - logLevelInfo = "info" - logLevelStack = "stack" + logStack = 20 + logStackFrom = 2 logFileNum = 10 logSizeMin int64 = 10 // 10MB @@ -83,21 +79,21 @@ func makeLogDir(logDir string) error { return nil } -// InitConfig init log config +// InitConfig initializes log config func InitConfig(driver, logdir, level string, size int64) error { if driver == "" { - driver = logDriverStdio + driver = constant.LogDriverStdio } - if driver != logDriverStdio && driver != logDriverFile { + if driver != constant.LogDriverStdio && driver != constant.LogDriverFile { return fmt.Errorf("invalid log driver %s", driver) } logDriver = stdio - if driver == logDriverFile { + if driver == constant.LogDriverFile { logDriver = file } if level == "" { - level = logLevelInfo + level = constant.LogLevelInfo } levelstr, err := logLevelFromString(level) if err != nil { @@ -111,7 +107,7 @@ func InitConfig(driver, logdir, level string, size int64) error { logSize = size logFileMaxSize = logSize / logFileNum - if driver == "file" { + if driver == constant.LogDriverFile { if err := makeLogDir(logdir); err != nil { return err } @@ -139,15 +135,15 @@ func DropError(args ...interface{}) { func logLevelToString(level int) string { switch level { case logDebug: - return "debug" + return constant.LogLevelDebug case logInfo: - return "info" + return constant.LogLevelInfo case logWarn: - return "warn" + return constant.LogLevelWarn case logError: - return "error" + return constant.LogLevelError case logStack: - return logLevelStack + return constant.LogLevelStack default: return "" } @@ -155,13 +151,13 @@ func logLevelToString(level int) string { func logLevelFromString(level string) (int, error) { switch level { - case "debug": + case constant.LogLevelDebug: return logDebug, nil - case "info", "": + case constant.LogLevelInfo, "": return logInfo, nil - case "warn": + case constant.LogLevelWarn: return logWarn, nil - case "error": + case constant.LogLevelError: return logError, nil default: return logInfo, fmt.Errorf("invalid log level %s", level) @@ -211,7 +207,7 @@ func logf(level string, format string, args ...interface{}) { raw := fmt.Sprintf(format, args...) + "\n" depth := 1 - if level == logLevelStack { + if level == constant.LogLevelStack { depth = logStack } @@ -223,7 +219,7 @@ func logf(level string, format string, args ...interface{}) { fs = strings.Split("."+fs[len(fs)-1], ".") fn := fs[len(fs)-1] line = tag + fmt.Sprintf("%s:%d:%s() ", file, linum, fn) + raw - } else if level == logLevelStack { + } else if level == constant.LogLevelStack { break } writeLine(line) -- Gitee From 9b7037e8d391509b1b939fbd51b1df101fbe39a0 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Tue, 17 Jan 2023 17:39:29 +0800 Subject: [PATCH 11/73] refactor:add service manager implementation Signed-off-by: DCCooper <1866858@gmail.com> --- Makefile | 18 ++++-- pkg/api/api.go | 28 +++++---- pkg/podmanager/podmanager.go | 2 +- pkg/rubik/import.go | 1 + pkg/rubik/rubik.go | 52 +++++++++++----- pkg/services/blkio/blkio.go | 51 +++++++-------- pkg/services/cachelimit/cachelimit.go | 66 ++++++++++---------- pkg/services/registry.go | 2 +- pkg/services/servicemanager.go | 90 +++++++++++++++++++++++++++ 9 files changed, 216 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index 5312ed6..522d1d6 100644 --- a/Makefile +++ b/Makefile @@ -22,13 +22,15 @@ BUILD_TIME := $(shell date "+%Y-%m-%d") GIT_COMMIT := $(if $(shell git rev-parse --short HEAD),$(shell git rev-parse --short HEAD),$(shell cat ./git-commit | head -c 7)) DEBUG_FLAGS := -gcflags="all=-N -l" +EXTRALDFLAGS := -extldflags=-ftrapv \ + -extldflags=-Wl,-z,relro,-z,now -linkmode=external -extldflags=-static + LD_FLAGS := -ldflags '-buildid=none -tmpdir=$(TMP_DIR) \ -X isula.org/rubik/pkg/version.GitCommit=$(GIT_COMMIT) \ -X isula.org/rubik/pkg/version.BuildTime=$(BUILD_TIME) \ -X isula.org/rubik/pkg/version.Version=$(VERSION) \ -X isula.org/rubik/pkg/version.Release=$(RELEASE) \ - -extldflags=-ftrapv \ - -extldflags=-Wl,-z,relro,-z,now -linkmode=external -extldflags=-static' + $(EXTRALDFLAGS)' GO_BUILD=CGO_ENABLED=1 \ CGO_CFLAGS="-fstack-protector-strong -fPIE" \ @@ -51,13 +53,21 @@ help: @echo "make cover # generate coverage report" @echo "make install # install files to /var/lib/rubik" -release: +prepare: mkdir -p $(TMP_DIR) $(BUILD_DIR) - rm -rf $(TMP_DIR) && mkdir -p $(ORG_PATH) $(TMP_DIR) + rm -rf $(TMP_DIR) && mkdir -p $(TMP_DIR) + +release: prepare $(GO_BUILD) -o $(BUILD_DIR)/rubik $(LD_FLAGS) rubik.go sed 's/__RUBIK_IMAGE__/rubik:$(VERSION)-$(RELEASE)/g' hack/rubik-daemonset.yaml > $(BUILD_DIR)/rubik-daemonset.yaml cp hack/rubik.service $(BUILD_DIR) +debug: prepare + EXTRALDFLAGS="" + go build $(LD_FLAGS) $(DEBUG_FLAGS) -o $(BUILD_DIR)/rubik rubik.go + sed 's/__RUBIK_IMAGE__/rubik:$(VERSION)-$(RELEASE)/g' hack/rubik-daemonset.yaml > $(BUILD_DIR)/rubik-daemonset.yaml + cp hack/rubik.service $(BUILD_DIR) + image: release docker build -f Dockerfile -t rubik:$(VERSION)-$(RELEASE) . diff --git a/pkg/api/api.go b/pkg/api/api.go index 7b4575f..bbc4ece 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -16,7 +16,6 @@ package api import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "isula.org/rubik/pkg/core/typedef" ) @@ -30,17 +29,28 @@ type Registry interface { ListServices() ([]*Service, error) } +type ServiceChecker interface { + Validate() bool +} + +type EventFunc interface { + AddFunc(podInfo *typedef.PodInfo) error + UpdateFunc(old, new *typedef.PodInfo) error + DeleteFunc(podInfo *typedef.PodInfo) error +} + // Service contains progress that all services need to have type Service interface { - Init() error + EventFunc + ServiceChecker + ID() string Setup() error - Run() error TearDown() error - PodEventHandler() error } type PersistentService interface { - Init() error + ServiceChecker + Setup(viewer Viewer) error Start(stopCh <-chan struct{}) } @@ -49,14 +59,6 @@ type ConfigParser interface { UnmarshalSubConfig(data interface{}, v interface{}) error } -// PodEventSubscriber control pod activities -type PodEventSubscriber interface { - AddPod(pod *corev1.Pod) - UpdatePod(pod *corev1.Pod) - DeletePod(podID types.UID) - ID() string -} - // Viewer collect on/offline pods info type Viewer interface { ListOnlinePods() ([]*corev1.Pod, error) diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index ba777f8..6c8dde8 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -142,7 +142,7 @@ func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { if manager.pods.podExist(podInfo.UID) { oldPod := manager.pods.getPod(podInfo.UID) manager.pods.updatePod(podInfo) - manager.Publish(typedef.INFO_UPDATE, []interface{}{oldPod, podInfo.Clone()}) + manager.Publish(typedef.INFO_UPDATE, []*typedef.PodInfo{oldPod, podInfo.Clone()}) } } diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index 0eb7826..4d970c3 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -3,5 +3,6 @@ package rubik import ( // import packages to auto register service _ "isula.org/rubik/pkg/services/blkio" + _ "isula.org/rubik/pkg/services/cachelimit" _ "isula.org/rubik/pkg/version" ) diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 30cedcc..b742c68 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -2,33 +2,51 @@ package rubik import ( "fmt" + "os" + "os/signal" + "syscall" - "isula.org/rubik/pkg/registry" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/config" + "isula.org/rubik/pkg/services" ) func Run() int { fmt.Println("rubik running") + stopChan := make(chan struct{}) + go signalHandler(stopChan) - // 0. services automatic registration - // done in import.go - - // 1. enable autoconfig(informer) - // podinformer.Init() + c := config.NewConfig(config.JSON) + if err := c.LoadConfig(constant.ConfigFile); err != nil { + log.Errorf("load config failed: %v\n", err) + return -1 + } - // 2. enable checkpoint - services, err := registry.DefaultRegister.ListServices() - if err != nil { + if err := c.PrepareServices(); err != nil { + log.Errorf("prepare services failed: %v\n", err) return -1 } - for _, s := range services { - if err := s.PodEventHandler(); err != nil { - continue + + sm := services.GetServiceManager() + sm.Setup() + sm.Run(stopChan) + select { + case <-stopChan: + for _, s := range sm.RunningServices { + s.TearDown() } - s.Init() - s.Setup() - s.Run() - defer s.TearDown() + return 0 } +} - return 0 +func signalHandler(stopChan chan struct{}) { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) + for sig := range signalChan { + if sig == syscall.SIGTERM || sig == syscall.SIGINT { + log.Infof("signal %v received and starting exit...", sig) + close(stopChan) + } + } } diff --git a/pkg/services/blkio/blkio.go b/pkg/services/blkio/blkio.go index 7a87241..f2e0cf5 100644 --- a/pkg/services/blkio/blkio.go +++ b/pkg/services/blkio/blkio.go @@ -3,14 +3,26 @@ package blkio import ( "fmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - + "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" ) +// DeviceConfig defines blkio device configurations +type DeviceConfig struct { + DeviceName string `json:"device,omitempty"` + DeviceValue string `json:"value,omitempty"` +} + +type BlkioConfig struct { + DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` + DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` + DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` + DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` +} + type Blkio struct { - Name string + Name string `json:"-"` + Config BlkioConfig } func init() { @@ -23,43 +35,32 @@ func NewBlkio() *Blkio { return &Blkio{Name: "blkio"} } -func (b *Blkio) Init() error { - fmt.Println("blkio Init()") - return nil -} - func (b *Blkio) Setup() error { fmt.Println("blkio Setup()") return nil } -func (b *Blkio) Run() error { - fmt.Println("blkio Run()") - return nil -} - func (b *Blkio) TearDown() error { fmt.Println("blkio TearDown()") return nil } -func (b *Blkio) AddPod(pod *corev1.Pod) { - fmt.Println("blkio AddPod()") +func (b *Blkio) ID() string { + return b.Name } -func (b *Blkio) UpdatePod(pod *corev1.Pod) { - fmt.Println("blkio UpdatePod()") +func (b *Blkio) AddFunc(podInfo *typedef.PodInfo) error { + return nil } -func (b *Blkio) DeletePod(podID types.UID) { - fmt.Println("blkio DeletePod()") +func (b *Blkio) UpdateFunc(old, new *typedef.PodInfo) error { + return nil } -func (b *Blkio) ID() string { - return b.Name +func (b *Blkio) DeleteFunc(podInfo *typedef.PodInfo) error { + return nil } -func (b *Blkio) PodEventHandler() error { - fmt.Println("blkio PodEventHandler") - return nil +func (b *Blkio) Validate() bool { + return true } diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go index 97bb9b6..615df08 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -3,9 +3,7 @@ package cachelimit import ( "fmt" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - + "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" ) @@ -28,8 +26,7 @@ type MultiLvlPercent struct { High int `json:"high,omitempty"` } -type CacheLimit struct { - Name string `json:"-"` +type CacheLimitConfig struct { DefaultLimitMode string `json:"defaultLimitMode,omitempty"` DefaultResctrlDir string `json:"-"` AdjustInterval int `json:"adjustInterval,omitempty"` @@ -38,6 +35,11 @@ type CacheLimit struct { MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` } +type CacheLimit struct { + Name string `json:"-"` + Config CacheLimitConfig +} + func init() { services.Register("cacheLimit", func() interface{} { return NewCacheLimit() @@ -46,28 +48,26 @@ func init() { func NewCacheLimit() *CacheLimit { return &CacheLimit{ - DefaultLimitMode: "static", - DefaultResctrlDir: "/sys/fs/resctrl", - AdjustInterval: defaultAdInt, - PerfDuration: defaultPerfDur, - L3Percent: MultiLvlPercent{ - Low: defaultLowL3, - Mid: defaultMidL3, - High: defaultHighL3, - }, - MemBandPercent: MultiLvlPercent{ - Low: defaultLowMB, - Mid: defaultMidMB, - High: defaultHighMB, + Name: "cacheLimit", + Config: CacheLimitConfig{ + DefaultLimitMode: "static", + DefaultResctrlDir: "/sys/fs/resctrl", + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{ + Low: defaultLowL3, + Mid: defaultMidL3, + High: defaultHighL3, + }, + MemBandPercent: MultiLvlPercent{ + Low: defaultLowMB, + Mid: defaultMidMB, + High: defaultHighMB, + }, }, } } -func (c *CacheLimit) Init() error { - fmt.Println("cache limit Init()") - return nil -} - func (c *CacheLimit) Setup() error { fmt.Println("cache limit Setup()") return nil @@ -83,22 +83,22 @@ func (c *CacheLimit) TearDown() error { return nil } -func (c *CacheLimit) AddPod(pod *corev1.Pod) { - fmt.Println("cache limit AddPod()") +func (c *CacheLimit) ID() string { + return c.Name } -func (c *CacheLimit) UpdatePod(pod *corev1.Pod) { - fmt.Println("cache limit UpdatePod()") +func (c *CacheLimit) AddFunc(podInfo *typedef.PodInfo) error { + return nil } -func (c *CacheLimit) DeletePod(podID types.UID) { - fmt.Println("cache limit DeletePod()") +func (c *CacheLimit) UpdateFunc(old, new *typedef.PodInfo) error { + return nil } -func (c *CacheLimit) ID() string { - return c.Name +func (c *CacheLimit) DeleteFunc(podInfo *typedef.PodInfo) error { + return nil } -func (c *CacheLimit) PodEventHandler() error { - return nil +func (c *CacheLimit) Validate() bool { + return true } diff --git a/pkg/services/registry.go b/pkg/services/registry.go index 001dda8..986c461 100644 --- a/pkg/services/registry.go +++ b/pkg/services/registry.go @@ -27,7 +27,7 @@ func Register(name string, creator Creator) { servicesRegistry.Lock() servicesRegistry.services[name] = creator servicesRegistry.Unlock() - log.Debugf("func register (%s)\n", name) + log.Debugf("func register (%s)", name) } // GetServiceCreator returns the service creator based on the incoming service name diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 8772806..174e5e9 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -32,6 +32,7 @@ func AddRunningService(name string, service interface{}) { type ServiceManager struct { api.Subscriber + api.Viewer sync.RWMutex RunningServices map[string]api.Service RunningPersistentServices map[string]api.PersistentService @@ -56,8 +57,11 @@ func GetServiceManager() *ServiceManager { func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { switch eventType { case typedef.INFO_ADD: + manager.AddFunc(event) case typedef.INFO_UPDATE: + manager.UpdateFunc(event) case typedef.INFO_DELETE: + manager.DeleteFunc(event) default: log.Infof("service manager fail to process %s type", eventType.String()) } @@ -90,3 +94,89 @@ func (manager *ServiceManager) tryAddPersistentService(name string, service inte } return ok } + +func (manager *ServiceManager) Setup() error { + for _, s := range manager.RunningServices { + if err := s.Setup(); err != nil { + s.TearDown() + return err + } + } + for _, s := range manager.RunningPersistentServices { + if err := s.Setup(manager.Viewer); err != nil { + return err + } + } + return nil +} + +func (manager *ServiceManager) Run(stopChan chan struct{}) error { + for _, s := range manager.RunningPersistentServices { + s.Start(stopChan) + } + return nil +} + +func (manager *ServiceManager) AddFunc(event typedef.Event) { + podInfo, ok := event.(*typedef.PodInfo) + if !ok { + log.Warnf("receive invalid event: %T", event) + return + } + + runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + wg.Add(1) + s.AddFunc(podInfo) + wg.Done() + } + manager.RLock() + var wg sync.WaitGroup + for _, s := range manager.RunningServices { + go runOnce(s, podInfo.Clone(), &wg) + } + manager.RUnlock() +} + +func (manager *ServiceManager) UpdateFunc(event typedef.Event) { + podInfos, ok := event.([]*typedef.PodInfo) + if !ok { + log.Warnf("receive invalid event: %T", event) + return + } + const podInfosLen = 2 + if len(podInfos) != podInfosLen { + log.Warnf("pod infos contains more than two pods") + return + } + runOnce := func(s api.Service, old, new *typedef.PodInfo, wg *sync.WaitGroup) { + wg.Add(1) + s.UpdateFunc(old, new) + wg.Done() + } + manager.RLock() + var wg sync.WaitGroup + for _, s := range manager.RunningServices { + go runOnce(s, podInfos[0], podInfos[1], &wg) + } + manager.RUnlock() +} + +func (manager *ServiceManager) DeleteFunc(event typedef.Event) { + podInfo, ok := event.(*typedef.PodInfo) + if !ok { + log.Warnf("receive invalid event: %T", event) + return + } + + runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + wg.Add(1) + s.DeleteFunc(podInfo) + wg.Done() + } + manager.RLock() + var wg sync.WaitGroup + for _, s := range manager.RunningServices { + go runOnce(s, podInfo.Clone(), &wg) + } + manager.RUnlock() +} -- Gitee From 3f070eb07ee3f761aa4553d833dea08f909e448c Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 28 Jan 2023 17:39:18 +0800 Subject: [PATCH 12/73] refactor rubik overall logics 1. add rubik agent to control life cycle of all components 2. optimize service 3. use factory to support informer diversification 4. rename kubeinformer to APIServerInformer 5. Adapt existing files Signed-off-by: wujing --- pkg/api/api.go | 22 ++-- pkg/common/util/disposablechannel.go | 57 +++++++++ pkg/config/config.go | 3 + pkg/core/publisher/publisherFactory.go | 8 +- .../{kubeinformer.go => apiserverinformer.go} | 40 +++--- pkg/informer/informerfactory.go | 56 +++++++++ pkg/podmanager/podmanager.go | 8 ++ pkg/rubik/rubik.go | 119 ++++++++++++++---- pkg/services/blkio/blkio.go | 9 +- pkg/services/cachelimit/cachelimit.go | 11 +- pkg/services/servicemanager.go | 89 +++++++++++-- rubik.go | 2 +- 12 files changed, 346 insertions(+), 78 deletions(-) create mode 100644 pkg/common/util/disposablechannel.go rename pkg/informer/{kubeinformer.go => apiserverinformer.go} (71%) create mode 100644 pkg/informer/informerfactory.go diff --git a/pkg/api/api.go b/pkg/api/api.go index bbc4ece..1edb8d4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -15,8 +15,6 @@ package api import ( - corev1 "k8s.io/api/core/v1" - "isula.org/rubik/pkg/core/typedef" ) @@ -33,6 +31,10 @@ type ServiceChecker interface { Validate() bool } +type ServiceDescriber interface { + ID() string +} + type EventFunc interface { AddFunc(podInfo *typedef.PodInfo) error UpdateFunc(old, new *typedef.PodInfo) error @@ -41,17 +43,17 @@ type EventFunc interface { // Service contains progress that all services need to have type Service interface { - EventFunc + ServiceDescriber ServiceChecker - ID() string - Setup() error - TearDown() error + EventFunc + PreStart() error } type PersistentService interface { + ServiceDescriber ServiceChecker - Setup(viewer Viewer) error - Start(stopCh <-chan struct{}) + PreStart(viewer Viewer) error + Start(stopCh <-chan struct{}) error } type ConfigParser interface { @@ -61,8 +63,8 @@ type ConfigParser interface { // Viewer collect on/offline pods info type Viewer interface { - ListOnlinePods() ([]*corev1.Pod, error) - ListOfflinePods() ([]*corev1.Pod, error) + ListOnlinePods() ([]*typedef.PodInfo, error) + ListOfflinePods() ([]*typedef.PodInfo, error) } // Publisher is a generic interface for Observables diff --git a/pkg/common/util/disposablechannel.go b/pkg/common/util/disposablechannel.go new file mode 100644 index 0000000..5727bd0 --- /dev/null +++ b/pkg/common/util/disposablechannel.go @@ -0,0 +1,57 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines DisposableChannel and relative methods + +// Package util implementes public methods +package util + +import ( + "sync/atomic" +) + +// DisposableChannel only accepts data once and actively closes the channel, only allowed to close once +type DisposableChannel struct { + closeCount uint32 + ch chan struct{} +} + +// NewDisposableChannel creates and returns a DisposableChannel pointer variable +func NewDisposableChannel() *DisposableChannel { + return &DisposableChannel{ + closeCount: 0, + ch: make(chan struct{}, 1), + } +} + +// Close only allows closing the channel once if there are multiple calls +func (ch *DisposableChannel) Close() { + if atomic.AddUint32(&ch.closeCount, 1) == 1 { + close(ch.ch) + } +} + +// Wait waits for messages until the channel is closed or a message is received +func (ch *DisposableChannel) Wait() { + if atomic.LoadUint32(&ch.closeCount) > 0 { + return + } + <-ch.ch + ch.Close() +} + +// Channel returns the channel entity if the channel is not closed +func (ch *DisposableChannel) Channel() chan struct{} { + if atomic.LoadUint32(&ch.closeCount) == 0 { + return ch.ch + } + return nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 02945c0..565dc65 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -80,6 +80,9 @@ func (c *Config) LoadConfig(path string) error { } c.Fields = fields c.parseAgentConfig() + if err := c.PrepareServices(); err != nil { + return fmt.Errorf("error preparing services: %s", err) + } return nil } diff --git a/pkg/core/publisher/publisherFactory.go b/pkg/core/publisher/publisherFactory.go index c582956..51f51eb 100644 --- a/pkg/core/publisher/publisherFactory.go +++ b/pkg/core/publisher/publisherFactory.go @@ -27,9 +27,11 @@ const ( type PublisherFactory struct { } -// NewPublisherFactory creates a publisher factory class -func NewPublisherFactory() *PublisherFactory { - return &PublisherFactory{} +var publisherFactory = &PublisherFactory{} + +// NewPublisherFactory creates a publisher factory instance +func GetPublisherFactory() *PublisherFactory { + return publisherFactory } // GetPublisher returns the publisher entity according to the publisher type diff --git a/pkg/informer/kubeinformer.go b/pkg/informer/apiserverinformer.go similarity index 71% rename from pkg/informer/kubeinformer.go rename to pkg/informer/apiserverinformer.go index ec3bc83..ed9b829 100644 --- a/pkg/informer/kubeinformer.go +++ b/pkg/informer/apiserverinformer.go @@ -11,7 +11,7 @@ // Create: 2023-01-05 // Description: This file defines apiinformer which interact with kubernetes apiserver -// Package typedef implement informer interface +// Package informer implements informer interface package informer import ( @@ -31,20 +31,20 @@ import ( "isula.org/rubik/pkg/core/typedef" ) -// KubeInformer interacts with k8s api server and forward data to the internal -type kubeInformer struct { +// APIServerInformer interacts with k8s api server and forward data to the internal +type APIServerInformer struct { api.Publisher client *kubernetes.Clientset nodeName string } -// NewKubeInformer creates an KubeInformer instance -func NewKubeInformer(publisher api.Publisher) (*kubeInformer, error) { - informer := &kubeInformer{ +// NewAPIServerInformer creates an PIServerInformer instance +func NewAPIServerInformer(publisher api.Publisher) (api.Informer, error) { + informer := &APIServerInformer{ Publisher: publisher, } - // interact with apiserver + // create apiserver client client, err := initKubeClient() if err != nil { return nil, fmt.Errorf("fail to init kubenetes client: %v", err) @@ -76,34 +76,34 @@ func initKubeClient() (*kubernetes.Clientset, error) { return kubeClient, nil } -// Start starts and enables KubeInfomer -func (ki *kubeInformer) Start(stopCh <-chan struct{}) { +// Start starts and enables PIServerInformer +func (ai *APIServerInformer) Start(stopCh <-chan struct{}) { const ( reSyncTime = 30 specNodeNameField = "spec.nodeName" ) - kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(ki.client, + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(ai.client, time.Duration(reSyncTime)*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { // set Options to return only pods on the current node. - options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, ki.nodeName).String() + options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, ai.nodeName).String() })) kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: ki.addFunc, - UpdateFunc: ki.updateFunc, - DeleteFunc: ki.deleteFunc, + AddFunc: ai.addFunc, + UpdateFunc: ai.updateFunc, + DeleteFunc: ai.deleteFunc, }) kubeInformerFactory.Start(stopCh) } -func (ki *kubeInformer) addFunc(obj interface{}) { - ki.Publish(typedef.RAW_POD_ADD, obj) +func (ai *APIServerInformer) addFunc(obj interface{}) { + ai.Publish(typedef.RAW_POD_ADD, obj) } -func (ki *kubeInformer) updateFunc(oldObj, newObj interface{}) { - ki.Publish(typedef.RAW_POD_UPDATE, newObj) +func (ai *APIServerInformer) updateFunc(oldObj, newObj interface{}) { + ai.Publish(typedef.RAW_POD_UPDATE, newObj) } -func (ki *kubeInformer) deleteFunc(obj interface{}) { - ki.Publish(typedef.RAW_POD_DELETE, obj) +func (ai *APIServerInformer) deleteFunc(obj interface{}) { + ai.Publish(typedef.RAW_POD_DELETE, obj) } diff --git a/pkg/informer/informerfactory.go b/pkg/informer/informerfactory.go new file mode 100644 index 0000000..f46a492 --- /dev/null +++ b/pkg/informer/informerfactory.go @@ -0,0 +1,56 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines informerFactory which return the informer creator + +// Package informer implements informer interface +package informer + +import ( + "fmt" + + "isula.org/rubik/pkg/api" +) + +type ( + // the definition of informer type + informerType int8 + // informer's factory class + informerFactory struct{} +) + +const ( + // APISERVER instructs the informer to interact with the api server of kubernetes to obtain data + APISERVER informerType = iota +) + +// defaultInformerFactory is globally unique informer factory +var defaultInformerFactory *informerFactory + +// GetInformerCreator returns the constructor of the informer of the specified type +func (factory *informerFactory) GetInformerCreator(iType informerType) func(publisher api.Publisher) (api.Informer, error) { + switch iType { + case APISERVER: + return NewAPIServerInformer + default: + return func(publisher api.Publisher) (api.Informer, error) { + return nil, fmt.Errorf("infomer not implemented") + } + } +} + +// GetInfomerFactory returns the Informer factory class entity +func GetInfomerFactory() *informerFactory { + if defaultInformerFactory == nil { + defaultInformerFactory = &informerFactory{} + } + return defaultInformerFactory +} diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index 6c8dde8..7d8faf0 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -155,3 +155,11 @@ func (manager *PodManager) tryDelete(id string) { manager.Publish(typedef.INFO_DELETE, oldPod) } } + +func (manager *PodManager) ListOfflinePods() ([]*typedef.PodInfo, error) { + return nil, nil +} + +func (manager *PodManager) ListOnlinePods() ([]*typedef.PodInfo, error) { + return nil, nil +} diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index b742c68..6d367e5 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines rubik agent to control the life cycle of each component + +// // Package rubik defines the overall logic package rubik import ( @@ -8,45 +22,104 @@ import ( "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/config" + "isula.org/rubik/pkg/core/publisher" + "isula.org/rubik/pkg/informer" + "isula.org/rubik/pkg/podmanager" "isula.org/rubik/pkg/services" ) -func Run() int { - fmt.Println("rubik running") - stopChan := make(chan struct{}) - go signalHandler(stopChan) +// Agent runs a series of rubik services and manages data +type Agent struct { + Config *config.Config + PodManager *podmanager.PodManager + stopCh *util.DisposableChannel +} - c := config.NewConfig(config.JSON) - if err := c.LoadConfig(constant.ConfigFile); err != nil { - log.Errorf("load config failed: %v\n", err) - return -1 +// NewAgent returns an agent for given configuration +func NewAgent(cfg *config.Config) *Agent { + publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) + a := &Agent{ + Config: cfg, + stopCh: util.NewDisposableChannel(), + PodManager: podmanager.NewPodManager(publisher), } + return a +} - if err := c.PrepareServices(); err != nil { - log.Errorf("prepare services failed: %v\n", err) - return -1 +// Run starts and runs the agent until receiving stop signal +func (a *Agent) Run() error { + go a.handleSignals() + if err := a.startServiceHandler(); err != nil { + return err } - - sm := services.GetServiceManager() - sm.Setup() - sm.Run(stopChan) - select { - case <-stopChan: - for _, s := range sm.RunningServices { - s.TearDown() - } - return 0 + if err := a.startInformer(); err != nil { + return err } + a.stopCh.Wait() + if err := a.stopServiceHandler(); err != nil { + return err + } + return nil } -func signalHandler(stopChan chan struct{}) { +// handleSignals handles external signal input +func (a *Agent) handleSignals() { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) for sig := range signalChan { if sig == syscall.SIGTERM || sig == syscall.SIGINT { log.Infof("signal %v received and starting exit...", sig) - close(stopChan) + a.stopCh.Close() } } } + +// startInformer starts informer to obtain external data +func (a *Agent) startInformer() error { + publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) + informer, err := informer.GetInfomerFactory().GetInformerCreator(informer.APISERVER)(publisher) + if err != nil { + return fmt.Errorf("fail to set informer: %v", err) + } + informer.Start(a.stopCh.Channel()) + return nil +} + +// startServiceHandler starts and runs the service +func (a *Agent) startServiceHandler() error { + serviceManager := services.GetServiceManager() + if err := serviceManager.Setup(a.PodManager); err != nil { + return fmt.Errorf("error setting service handler: %v", err) + } + if err := serviceManager.Start(a.stopCh.Channel()); err != nil { + return fmt.Errorf("error setting service handler: %v", err) + } + a.PodManager.Subscribe(serviceManager) + return nil +} + +// stopServiceHandler stops the service +func (a *Agent) stopServiceHandler() error { + serviceManager := services.GetServiceManager() + if err := serviceManager.Stop(); err != nil { + return fmt.Errorf("error stop service handler: %v", err) + } + return nil +} + +// RunAgent creates and runs rubik's agent +func RunAgent() int { + c := config.NewConfig(config.JSON) + if err := c.LoadConfig(constant.ConfigFile); err != nil { + log.Errorf("load config failed: %v\n", err) + return -1 + } + agent := NewAgent(c) + if err := agent.Run(); err != nil { + log.Errorf("error running agent: %v", err) + return -1 + } + return 0 +} diff --git a/pkg/services/blkio/blkio.go b/pkg/services/blkio/blkio.go index f2e0cf5..c5701f0 100644 --- a/pkg/services/blkio/blkio.go +++ b/pkg/services/blkio/blkio.go @@ -21,8 +21,7 @@ type BlkioConfig struct { } type Blkio struct { - Name string `json:"-"` - Config BlkioConfig + Name string `json:"-"` } func init() { @@ -35,13 +34,13 @@ func NewBlkio() *Blkio { return &Blkio{Name: "blkio"} } -func (b *Blkio) Setup() error { +func (b *Blkio) PreStart() error { fmt.Println("blkio Setup()") return nil } -func (b *Blkio) TearDown() error { - fmt.Println("blkio TearDown()") +func (b *Blkio) Terminate() error { + fmt.Println("blkio Terminate") return nil } diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go index 615df08..3bfdcd0 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -68,18 +68,13 @@ func NewCacheLimit() *CacheLimit { } } -func (c *CacheLimit) Setup() error { +func (c *CacheLimit) PreStart() error { fmt.Println("cache limit Setup()") return nil } -func (c *CacheLimit) Run() error { - fmt.Println("cache limit Run()") - return nil -} - -func (c *CacheLimit) TearDown() error { - fmt.Println("cache limit TearDown()") +func (c *CacheLimit) Terminate() error { + fmt.Println("cache limit Terminate()") return nil } diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 174e5e9..4d58819 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -1,6 +1,21 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines ServiceManager to manage the lifecycle of services + +// Package services implements service registration, discovery and management functions package services import ( + "fmt" "sync" "isula.org/rubik/pkg/api" @@ -30,14 +45,17 @@ func AddRunningService(name string, service interface{}) { log.Debugf("pre-start service %s", name) } +// ServiceManager is used to manage the lifecycle of services type ServiceManager struct { api.Subscriber api.Viewer sync.RWMutex RunningServices map[string]api.Service RunningPersistentServices map[string]api.PersistentService + exitFuncs []func() error } +// serviceManager is the only global service manager var serviceManager = newServiceManager() func newServiceManager() *ServiceManager { @@ -54,6 +72,7 @@ func GetServiceManager() *ServiceManager { return serviceManager } +// HandleEvent is used to handle PodInfo events pushed by the publisher func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { switch eventType { case typedef.INFO_ADD: @@ -67,7 +86,8 @@ func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event ty } } -func (cmanager *ServiceManager) EventTypes() []typedef.EventType { +// EventTypes returns the type of event the serviceManager is interested in +func (manager *ServiceManager) EventTypes() []typedef.EventType { return []typedef.EventType{typedef.INFO_ADD, typedef.INFO_UPDATE, typedef.INFO_DELETE} } @@ -95,28 +115,72 @@ func (manager *ServiceManager) tryAddPersistentService(name string, service inte return ok } -func (manager *ServiceManager) Setup() error { - for _, s := range manager.RunningServices { - if err := s.Setup(); err != nil { - s.TearDown() +// Setup pre-starts services, such as preparing the environment, etc. +func (manager *ServiceManager) Setup(v api.Viewer) error { + var ( + exitFuncs []func() error + // process when fail to pre-start services + errorHandler = func(err error) error { + for _, exitFunc := range exitFuncs { + exitFunc() + } return err } + ) + // pre-start services + for _, s := range manager.RunningServices { + if err := s.PreStart(); err != nil { + return errorHandler(fmt.Errorf("error running services %s: %v", s.ID(), err)) + } + if t, ok := s.(Terminator); ok { + exitFuncs = append(exitFuncs, t.Terminate) + } } + // pre-start persistent services only when viewer is prepared + if v == nil { + return nil + } + manager.Viewer = v for _, s := range manager.RunningPersistentServices { - if err := s.Setup(manager.Viewer); err != nil { - return err + if err := s.PreStart(manager.Viewer); err != nil { + return errorHandler(fmt.Errorf("error running persistent services %s: %v", s.ID(), err)) + } + if t, ok := s.(Terminator); ok { + exitFuncs = append(exitFuncs, t.Terminate) } } + manager.exitFuncs = exitFuncs return nil } -func (manager *ServiceManager) Run(stopChan chan struct{}) error { +// Start starts and runs the service +func (manager *ServiceManager) Start(stopChan <-chan struct{}) error { for _, s := range manager.RunningPersistentServices { s.Start(stopChan) } return nil } +// Start starts and runs the service +func (manager *ServiceManager) Stop() error { + var ( + allErr error + errorOccur bool = false + ) + for _, exitFunc := range manager.exitFuncs { + if err := exitFunc(); err != nil { + log.Errorf("error stopping services: %v", err) + allErr = fmt.Errorf("error stopping services") + errorOccur = true + } + } + if errorOccur { + return allErr + } + return nil +} + +// AddFunc handles pod addition events func (manager *ServiceManager) AddFunc(event typedef.Event) { podInfo, ok := event.(*typedef.PodInfo) if !ok { @@ -134,9 +198,11 @@ func (manager *ServiceManager) AddFunc(event typedef.Event) { for _, s := range manager.RunningServices { go runOnce(s, podInfo.Clone(), &wg) } + wg.Wait() manager.RUnlock() } +// UpdateFunc() handles pod update events func (manager *ServiceManager) UpdateFunc(event typedef.Event) { podInfos, ok := event.([]*typedef.PodInfo) if !ok { @@ -158,9 +224,11 @@ func (manager *ServiceManager) UpdateFunc(event typedef.Event) { for _, s := range manager.RunningServices { go runOnce(s, podInfos[0], podInfos[1], &wg) } + wg.Wait() manager.RUnlock() } +// DeleteFunc handles pod deletion events func (manager *ServiceManager) DeleteFunc(event typedef.Event) { podInfo, ok := event.(*typedef.PodInfo) if !ok { @@ -178,5 +246,10 @@ func (manager *ServiceManager) DeleteFunc(event typedef.Event) { for _, s := range manager.RunningServices { go runOnce(s, podInfo.Clone(), &wg) } + wg.Wait() manager.RUnlock() } + +type Terminator interface { + Terminate() error +} diff --git a/rubik.go b/rubik.go index 3c7676b..a3056ea 100644 --- a/rubik.go +++ b/rubik.go @@ -7,5 +7,5 @@ import ( ) func main() { - os.Exit(rubik.Run()) + os.Exit(rubik.RunAgent()) } -- Gitee From 2504c88ae9ac91cd33cc75df02f3c9c1e811a56a Mon Sep 17 00:00:00 2001 From: vegbir Date: Sun, 29 Jan 2023 21:25:50 +0800 Subject: [PATCH 13/73] refactor logics 1. Use context instead of using one-time channels 2. Optimize the service startup process, enhance exception handling, and improve reliability 3. Optimization: renaming, adding comments, etc. --- pkg/api/api.go | 18 ++- pkg/common/util/disposablechannel.go | 57 ---------- pkg/config/config.go | 11 ++ pkg/core/publisher/genericpublisher.go | 6 +- pkg/informer/apiserverinformer.go | 30 ++--- pkg/rubik/rubik.go | 96 +++++++++------- pkg/services/blkio/blkio.go | 13 +-- pkg/services/cachelimit/cachelimit.go | 24 ++-- pkg/services/servicemanager.go | 149 +++++++++++++++---------- rubik.go | 2 +- 10 files changed, 200 insertions(+), 206 deletions(-) delete mode 100644 pkg/common/util/disposablechannel.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 1edb8d4..1971ef1 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -15,6 +15,8 @@ package api import ( + "context" + "isula.org/rubik/pkg/core/typedef" ) @@ -27,10 +29,6 @@ type Registry interface { ListServices() ([]*Service, error) } -type ServiceChecker interface { - Validate() bool -} - type ServiceDescriber interface { ID() string } @@ -44,16 +42,14 @@ type EventFunc interface { // Service contains progress that all services need to have type Service interface { ServiceDescriber - ServiceChecker EventFunc - PreStart() error } +// PersistentService is an abstract persistent running service type PersistentService interface { ServiceDescriber - ServiceChecker - PreStart(viewer Viewer) error - Start(stopCh <-chan struct{}) error + // Run is a service processing logic, which is blocking (implemented in an infinite loop, etc.) + Run(ctx context.Context) } type ConfigParser interface { @@ -70,7 +66,7 @@ type Viewer interface { // Publisher is a generic interface for Observables type Publisher interface { Subscribe(s Subscriber) error - Unsubscribe(s Subscriber) error + Unsubscribe(s Subscriber) Publish(topic typedef.EventType, event typedef.Event) } @@ -90,5 +86,5 @@ type EventHandler interface { // Informer is an interface for external pod data sources to interact with rubik type Informer interface { Publisher - Start(stopCh <-chan struct{}) + Start(ctx context.Context) } diff --git a/pkg/common/util/disposablechannel.go b/pkg/common/util/disposablechannel.go deleted file mode 100644 index 5727bd0..0000000 --- a/pkg/common/util/disposablechannel.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2023-01-28 -// Description: This file defines DisposableChannel and relative methods - -// Package util implementes public methods -package util - -import ( - "sync/atomic" -) - -// DisposableChannel only accepts data once and actively closes the channel, only allowed to close once -type DisposableChannel struct { - closeCount uint32 - ch chan struct{} -} - -// NewDisposableChannel creates and returns a DisposableChannel pointer variable -func NewDisposableChannel() *DisposableChannel { - return &DisposableChannel{ - closeCount: 0, - ch: make(chan struct{}, 1), - } -} - -// Close only allows closing the channel once if there are multiple calls -func (ch *DisposableChannel) Close() { - if atomic.AddUint32(&ch.closeCount, 1) == 1 { - close(ch.ch) - } -} - -// Wait waits for messages until the channel is closed or a message is received -func (ch *DisposableChannel) Wait() { - if atomic.LoadUint32(&ch.closeCount) > 0 { - return - } - <-ch.ch - ch.Close() -} - -// Channel returns the channel entity if the channel is not closed -func (ch *DisposableChannel) Channel() chan struct{} { - if atomic.LoadUint32(&ch.closeCount) == 0 { - return ch.ch - } - return nil -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 565dc65..455a486 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -108,7 +108,18 @@ func (c *Config) PrepareServices() error { if err := c.UnmarshalSubConfig(config, service); err != nil { return fmt.Errorf("error unmarshaling %s config: %v", name, err) } + + // try to verify configuration + if validator, ok := service.(Validator); ok { + if err := validator.Validate(); err != nil { + return fmt.Errorf("error configuring service %s: %v", name, err) + } + } services.AddRunningService(name, service) } return nil } + +type Validator interface { + Validate() error +} diff --git a/pkg/core/publisher/genericpublisher.go b/pkg/core/publisher/genericpublisher.go index caff433..bc6bd1a 100644 --- a/pkg/core/publisher/genericpublisher.go +++ b/pkg/core/publisher/genericpublisher.go @@ -87,10 +87,11 @@ func (pub *genericPublisher) Subscribe(s api.Subscriber) error { } // Unsubscribe unsubscribes the indicated subscriber -func (pub *genericPublisher) Unsubscribe(s api.Subscriber) error { +func (pub *genericPublisher) Unsubscribe(s api.Subscriber) { id := s.ID() if !pub.subscriberExisted(id) { - return fmt.Errorf("subscriber %v has not registered", id) + log.Warnf("subscriber %v has not registered", id) + return } pub.Lock() for _, topic := range s.TopicsFunc() { @@ -99,7 +100,6 @@ func (pub *genericPublisher) Unsubscribe(s api.Subscriber) error { } delete(pub.subscribers, id) pub.Unlock() - return nil } // Publish publishes Event to subscribers interested in specified topic diff --git a/pkg/informer/apiserverinformer.go b/pkg/informer/apiserverinformer.go index ed9b829..3e07f78 100644 --- a/pkg/informer/apiserverinformer.go +++ b/pkg/informer/apiserverinformer.go @@ -15,6 +15,7 @@ package informer import ( + "context" "fmt" "os" "time" @@ -77,33 +78,36 @@ func initKubeClient() (*kubernetes.Clientset, error) { } // Start starts and enables PIServerInformer -func (ai *APIServerInformer) Start(stopCh <-chan struct{}) { +func (informer *APIServerInformer) Start(ctx context.Context) { const ( reSyncTime = 30 specNodeNameField = "spec.nodeName" ) - kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(ai.client, + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(informer.client, time.Duration(reSyncTime)*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { // set Options to return only pods on the current node. - options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, ai.nodeName).String() + options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, informer.nodeName).String() })) kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: ai.addFunc, - UpdateFunc: ai.updateFunc, - DeleteFunc: ai.deleteFunc, + AddFunc: informer.AddFunc, + UpdateFunc: informer.UpdateFunc, + DeleteFunc: informer.DeleteFunc, }) - kubeInformerFactory.Start(stopCh) + kubeInformerFactory.Start(ctx.Done()) } -func (ai *APIServerInformer) addFunc(obj interface{}) { - ai.Publish(typedef.RAW_POD_ADD, obj) +// AddFunc handles the raw pod increase event +func (informer *APIServerInformer) AddFunc(obj interface{}) { + informer.Publish(typedef.RAW_POD_ADD, obj) } -func (ai *APIServerInformer) updateFunc(oldObj, newObj interface{}) { - ai.Publish(typedef.RAW_POD_UPDATE, newObj) +// UpdateFunc handles the raw pod update event +func (informer *APIServerInformer) UpdateFunc(oldObj, newObj interface{}) { + informer.Publish(typedef.RAW_POD_UPDATE, newObj) } -func (ai *APIServerInformer) deleteFunc(obj interface{}) { - ai.Publish(typedef.RAW_POD_DELETE, obj) +// DeleteFunc handles the raw pod deletion event +func (informer *APIServerInformer) DeleteFunc(obj interface{}) { + informer.Publish(typedef.RAW_POD_DELETE, obj) } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 6d367e5..c31d862 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -15,14 +15,15 @@ package rubik import ( + "context" "fmt" "os" "os/signal" "syscall" + "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/publisher" "isula.org/rubik/pkg/informer" @@ -34,7 +35,7 @@ import ( type Agent struct { Config *config.Config PodManager *podmanager.PodManager - stopCh *util.DisposableChannel + informer api.Informer } // NewAgent returns an agent for given configuration @@ -42,83 +43,96 @@ func NewAgent(cfg *config.Config) *Agent { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) a := &Agent{ Config: cfg, - stopCh: util.NewDisposableChannel(), PodManager: podmanager.NewPodManager(publisher), } return a } // Run starts and runs the agent until receiving stop signal -func (a *Agent) Run() error { - go a.handleSignals() - if err := a.startServiceHandler(); err != nil { +func (a *Agent) Run(ctx context.Context) error { + if err := a.startServiceHandler(ctx); err != nil { return err } - if err := a.startInformer(); err != nil { - return err - } - a.stopCh.Wait() - if err := a.stopServiceHandler(); err != nil { + if err := a.startInformer(ctx); err != nil { return err } + <-ctx.Done() + a.stopInformer() + a.stopServiceHandler() return nil } -// handleSignals handles external signal input -func (a *Agent) handleSignals() { - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) - for sig := range signalChan { - if sig == syscall.SIGTERM || sig == syscall.SIGINT { - log.Infof("signal %v received and starting exit...", sig) - a.stopCh.Close() - } - } -} - // startInformer starts informer to obtain external data -func (a *Agent) startInformer() error { +func (a *Agent) startInformer(ctx context.Context) error { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) informer, err := informer.GetInfomerFactory().GetInformerCreator(informer.APISERVER)(publisher) if err != nil { return fmt.Errorf("fail to set informer: %v", err) } - informer.Start(a.stopCh.Channel()) + if err := informer.Subscribe(a.PodManager); err != nil { + return fmt.Errorf("fail to subscribe informer: %v", err) + } + a.informer = informer + informer.Start(ctx) return nil } +// stopInformer stops the infomer +func (a *Agent) stopInformer() { + a.informer.Unsubscribe(a.PodManager) +} + // startServiceHandler starts and runs the service -func (a *Agent) startServiceHandler() error { +func (a *Agent) startServiceHandler(ctx context.Context) error { serviceManager := services.GetServiceManager() if err := serviceManager.Setup(a.PodManager); err != nil { return fmt.Errorf("error setting service handler: %v", err) } - if err := serviceManager.Start(a.stopCh.Channel()); err != nil { - return fmt.Errorf("error setting service handler: %v", err) - } + serviceManager.Start(ctx) a.PodManager.Subscribe(serviceManager) return nil } -// stopServiceHandler stops the service -func (a *Agent) stopServiceHandler() error { +// stopServiceHandler stops sending data to the ServiceManager +func (a *Agent) stopServiceHandler() { serviceManager := services.GetServiceManager() - if err := serviceManager.Stop(); err != nil { - return fmt.Errorf("error stop service handler: %v", err) - } - return nil + a.PodManager.Unsubscribe(serviceManager) + serviceManager.Stop() } -// RunAgent creates and runs rubik's agent -func RunAgent() int { +// runAgent creates and runs rubik's agent +func runAgent(ctx context.Context) error { c := config.NewConfig(config.JSON) if err := c.LoadConfig(constant.ConfigFile); err != nil { - log.Errorf("load config failed: %v\n", err) - return -1 + return fmt.Errorf("error loading config: %v", err) + } + if err := log.InitConfig(c.Agent.LogDriver, c.Agent.LogDir, c.Agent.LogLevel, c.Agent.LogSize); err != nil { + return fmt.Errorf("error initializing log: %v", err) } agent := NewAgent(c) - if err := agent.Run(); err != nil { - log.Errorf("error running agent: %v", err) + if err := agent.Run(ctx); err != nil { + return fmt.Errorf("error running agent: %v", err) + } + return nil +} + +// Run runs agent and process signal +func Run() int { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) + for sig := range signalChan { + if sig == syscall.SIGTERM || sig == syscall.SIGINT { + log.Infof("signal %v received and starting exit...", sig) + cancel() + } + } + }() + + if err := runAgent(ctx); err != nil { + log.Errorf("error running rubik agent: %v", err) return -1 } return 0 diff --git a/pkg/services/blkio/blkio.go b/pkg/services/blkio/blkio.go index c5701f0..bc02fbd 100644 --- a/pkg/services/blkio/blkio.go +++ b/pkg/services/blkio/blkio.go @@ -3,6 +3,7 @@ package blkio import ( "fmt" + "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" ) @@ -34,13 +35,13 @@ func NewBlkio() *Blkio { return &Blkio{Name: "blkio"} } -func (b *Blkio) PreStart() error { - fmt.Println("blkio Setup()") +func (b *Blkio) PreStart(viewer api.Viewer) error { + fmt.Println("blkio prestart") return nil } -func (b *Blkio) Terminate() error { - fmt.Println("blkio Terminate") +func (b *Blkio) Terminate(viewer api.Viewer) error { + fmt.Println("blkio Terminates") return nil } @@ -59,7 +60,3 @@ func (b *Blkio) UpdateFunc(old, new *typedef.PodInfo) error { func (b *Blkio) DeleteFunc(podInfo *typedef.PodInfo) error { return nil } - -func (b *Blkio) Validate() bool { - return true -} diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go index 3bfdcd0..acf5f35 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -1,9 +1,10 @@ package cachelimit import ( + "context" "fmt" - "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/services" ) @@ -68,12 +69,12 @@ func NewCacheLimit() *CacheLimit { } } -func (c *CacheLimit) PreStart() error { - fmt.Println("cache limit Setup()") +func (c *CacheLimit) PreStart(viewer api.Viewer) error { + fmt.Println("cache limit Prestart()") return nil } -func (c *CacheLimit) Terminate() error { +func (c *CacheLimit) Terminate(viewer api.Viewer) error { fmt.Println("cache limit Terminate()") return nil } @@ -82,18 +83,11 @@ func (c *CacheLimit) ID() string { return c.Name } -func (c *CacheLimit) AddFunc(podInfo *typedef.PodInfo) error { - return nil -} - -func (c *CacheLimit) UpdateFunc(old, new *typedef.PodInfo) error { - return nil +func (c *CacheLimit) Run(ctx context.Context) { + fmt.Println("cacheLimit Run") } -func (c *CacheLimit) DeleteFunc(podInfo *typedef.PodInfo) error { +func (b *CacheLimit) Validate() error { + fmt.Println("cachelimit Validate()") return nil } - -func (c *CacheLimit) Validate() bool { - return true -} diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 4d58819..2a2a02d 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -15,13 +15,16 @@ package services import ( + "context" "fmt" "sync" + "time" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/subscriber" "isula.org/rubik/pkg/core/typedef" + "k8s.io/apimachinery/pkg/util/wait" ) // serviceManagerName is the unique ID of the service manager @@ -52,7 +55,7 @@ type ServiceManager struct { sync.RWMutex RunningServices map[string]api.Service RunningPersistentServices map[string]api.PersistentService - exitFuncs []func() error + TerminateFuncs map[string]Terminator } // serviceManager is the only global service manager @@ -74,13 +77,18 @@ func GetServiceManager() *ServiceManager { // HandleEvent is used to handle PodInfo events pushed by the publisher func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { + defer func() { + if err := recover(); err != nil { + log.Errorf("panic occurr: %v", err) + } + }() switch eventType { case typedef.INFO_ADD: - manager.AddFunc(event) + manager.addFunc(event) case typedef.INFO_UPDATE: - manager.UpdateFunc(event) + manager.updateFunc(event) case typedef.INFO_DELETE: - manager.DeleteFunc(event) + manager.deleteFunc(event) default: log.Infof("service manager fail to process %s type", eventType.String()) } @@ -115,73 +123,97 @@ func (manager *ServiceManager) tryAddPersistentService(name string, service inte return ok } -// Setup pre-starts services, such as preparing the environment, etc. -func (manager *ServiceManager) Setup(v api.Viewer) error { - var ( - exitFuncs []func() error - // process when fail to pre-start services - errorHandler = func(err error) error { - for _, exitFunc := range exitFuncs { - exitFunc() - } - return err - } - ) - // pre-start services - for _, s := range manager.RunningServices { - if err := s.PreStart(); err != nil { - return errorHandler(fmt.Errorf("error running services %s: %v", s.ID(), err)) - } - if t, ok := s.(Terminator); ok { - exitFuncs = append(exitFuncs, t.Terminate) +// terminatingRunningServices handles services exits during the setup and exit phases +func (manager *ServiceManager) terminatingRunningServices(err error) error { + if manager.TerminateFuncs == nil { + return nil + } + for id, t := range manager.TerminateFuncs { + if termErr := t.Terminate(manager.Viewer); termErr != nil { + log.Errorf("error terminating services %s: %v", id, termErr) } } - // pre-start persistent services only when viewer is prepared + return err +} + +// Setup pre-starts services, such as preparing the environment, etc. +func (manager *ServiceManager) Setup(v api.Viewer) error { + // only when viewer is prepared if v == nil { return nil } manager.Viewer = v - for _, s := range manager.RunningPersistentServices { - if err := s.PreStart(manager.Viewer); err != nil { - return errorHandler(fmt.Errorf("error running persistent services %s: %v", s.ID(), err)) - } + manager.TerminateFuncs = make(map[string]Terminator) + setupFunc := func(id string, s interface{}) error { + // 1. record the termination function of the service that has been setup if t, ok := s.(Terminator); ok { - exitFuncs = append(exitFuncs, t.Terminate) + manager.TerminateFuncs[id] = t + } + // 2. execute the pre-start function of the service + p, ok := s.(PreStarter) + if !ok { + return nil } + if err := p.PreStart(manager.Viewer); err != nil { + return err + } + return nil } - manager.exitFuncs = exitFuncs - return nil -} -// Start starts and runs the service -func (manager *ServiceManager) Start(stopChan <-chan struct{}) error { + // 1. pre-start services + for _, s := range manager.RunningServices { + if err := setupFunc(s.ID(), s); err != nil { + /* + handle the error and terminate all services that have been started + when any setup stage failed + */ + return manager.terminatingRunningServices(fmt.Errorf("error running services %s: %v", s.ID(), err)) + } + } + // 2. pre-start persistent services for _, s := range manager.RunningPersistentServices { - s.Start(stopChan) + if err := setupFunc(s.ID(), s); err != nil { + return manager.terminatingRunningServices(fmt.Errorf("error running services %s: %v", s.ID(), err)) + } } return nil } -// Start starts and runs the service -func (manager *ServiceManager) Stop() error { - var ( - allErr error - errorOccur bool = false - ) - for _, exitFunc := range manager.exitFuncs { - if err := exitFunc(); err != nil { - log.Errorf("error stopping services: %v", err) - allErr = fmt.Errorf("error stopping services") - errorOccur = true - } +// Start starts and runs the persistent service +func (manager *ServiceManager) Start(ctx context.Context) { + /* + The Run function of the service will be called continuously until the context is canceled. + When a service panics while running, recover will catch the violation + and briefly restart for a short period of time. + */ + const restartDuration = 2 * time.Second + runner := func(ctx context.Context, id string, runFunc func(ctx context.Context)) { + var restartCount int64 = 0 + wait.UntilWithContext(ctx, func(ctx context.Context) { + defer func() { + if err := recover(); err != nil { + log.Errorf("service %s catch a panic: %v", id, err) + } + }() + if restartCount > 0 { + log.Warnf("service %s has restart %v times", id, restartCount) + } + restartCount++ + runFunc(ctx) + }, restartDuration) } - if errorOccur { - return allErr + for id, s := range manager.RunningPersistentServices { + go runner(ctx, id, s.Run) } - return nil } -// AddFunc handles pod addition events -func (manager *ServiceManager) AddFunc(event typedef.Event) { +// Stop terminates the running service +func (manager *ServiceManager) Stop() error { + return manager.terminatingRunningServices(nil) +} + +// addFunc handles pod addition events +func (manager *ServiceManager) addFunc(event typedef.Event) { podInfo, ok := event.(*typedef.PodInfo) if !ok { log.Warnf("receive invalid event: %T", event) @@ -202,8 +234,8 @@ func (manager *ServiceManager) AddFunc(event typedef.Event) { manager.RUnlock() } -// UpdateFunc() handles pod update events -func (manager *ServiceManager) UpdateFunc(event typedef.Event) { +// updateFunc handles pod update events +func (manager *ServiceManager) updateFunc(event typedef.Event) { podInfos, ok := event.([]*typedef.PodInfo) if !ok { log.Warnf("receive invalid event: %T", event) @@ -228,8 +260,8 @@ func (manager *ServiceManager) UpdateFunc(event typedef.Event) { manager.RUnlock() } -// DeleteFunc handles pod deletion events -func (manager *ServiceManager) DeleteFunc(event typedef.Event) { +// deleteFunc handles pod deletion events +func (manager *ServiceManager) deleteFunc(event typedef.Event) { podInfo, ok := event.(*typedef.PodInfo) if !ok { log.Warnf("receive invalid event: %T", event) @@ -251,5 +283,8 @@ func (manager *ServiceManager) DeleteFunc(event typedef.Event) { } type Terminator interface { - Terminate() error + Terminate(api.Viewer) error +} +type PreStarter interface { + PreStart(api.Viewer) error } diff --git a/rubik.go b/rubik.go index a3056ea..3c7676b 100644 --- a/rubik.go +++ b/rubik.go @@ -7,5 +7,5 @@ import ( ) func main() { - os.Exit(rubik.RunAgent()) + os.Exit(rubik.Run()) } -- Gitee From 6dbc57e656054dfa4cca0473bb6cffd9c6b5361d Mon Sep 17 00:00:00 2001 From: wujing Date: Tue, 31 Jan 2023 09:18:02 +0800 Subject: [PATCH 14/73] refactor informer using list-watch Save podmanager data by first listing all pods and then monitoring pod change events Signed-off-by: wujing --- pkg/core/typedef/event.go | 19 +++++---- pkg/informer/apiserverinformer.go | 27 ++++++++++--- pkg/podmanager/podcache.go | 17 ++++++++ pkg/podmanager/podmanager.go | 67 ++++++++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 16 deletions(-) diff --git a/pkg/core/typedef/event.go b/pkg/core/typedef/event.go index b25565c..9e74a59 100644 --- a/pkg/core/typedef/event.go +++ b/pkg/core/typedef/event.go @@ -34,17 +34,20 @@ const ( INFO_UPDATE // PodManager deletes pod information event INFO_DELETE + // Full amount of kubernetes pods + RAW_POD_SYNC_ALL ) -const unknownType = "unknown" +const undefinedType = "undefined" var eventTypeToString = map[EventType]string{ - RAW_POD_ADD: "addrawpod", - RAW_POD_UPDATE: "updaterawpod", - RAW_POD_DELETE: "deleterawpod", - INFO_ADD: "addinfo", - INFO_UPDATE: "updateinfo", - INFO_DELETE: "deleteinfo", + RAW_POD_ADD: "addrawpod", + RAW_POD_UPDATE: "updaterawpod", + RAW_POD_DELETE: "deleterawpod", + INFO_ADD: "addinfo", + INFO_UPDATE: "updateinfo", + INFO_DELETE: "deleteinfo", + RAW_POD_SYNC_ALL: "syncallrawpods", } // String returns the string of the current event type @@ -52,5 +55,5 @@ func (t EventType) String() string { if str, ok := eventTypeToString[t]; ok { return str } - return unknownType + return undefinedType } diff --git a/pkg/informer/apiserverinformer.go b/pkg/informer/apiserverinformer.go index 3e07f78..4d1c9dd 100644 --- a/pkg/informer/apiserverinformer.go +++ b/pkg/informer/apiserverinformer.go @@ -29,6 +29,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" ) @@ -79,15 +80,29 @@ func initKubeClient() (*kubernetes.Clientset, error) { // Start starts and enables PIServerInformer func (informer *APIServerInformer) Start(ctx context.Context) { - const ( - reSyncTime = 30 - specNodeNameField = "spec.nodeName" - ) + const specNodeNameField = "spec.nodeName" + // set options to return only pods on the current node. + var fieldSelector = fields.OneTermEqualSelector(specNodeNameField, informer.nodeName).String() + informer.listFunc(fieldSelector) + informer.watchFunc(ctx, fieldSelector) +} + +func (informer *APIServerInformer) listFunc(fieldSelector string) { + pods, err := informer.client.CoreV1().Pods("").List(context.Background(), + metav1.ListOptions{FieldSelector: fieldSelector}) + if err != nil { + log.Errorf("error listing all pods: %v", err) + return + } + informer.Publish(typedef.RAW_POD_SYNC_ALL, pods.Items) +} + +func (informer *APIServerInformer) watchFunc(ctx context.Context, fieldSelector string) { + const reSyncTime = 30 kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(informer.client, time.Duration(reSyncTime)*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { - // set Options to return only pods on the current node. - options.FieldSelector = fields.OneTermEqualSelector(specNodeNameField, informer.nodeName).String() + options.FieldSelector = fieldSelector })) kubeInformerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: informer.AddFunc, diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go index 04a9e79..35aedda 100644 --- a/pkg/podmanager/podcache.go +++ b/pkg/podmanager/podcache.go @@ -86,3 +86,20 @@ func (cache *podCache) updatePod(pod *typedef.PodInfo) { cache.Unlock() log.Debugf("update pod %v", pod.UID) } + +// substitute replaces all the data in the cache +func (cache *podCache) substitute(pods []*typedef.PodInfo) { + cache.Lock() + defer cache.Unlock() + cache.Pods = make(map[string]*typedef.PodInfo, 0) + if pods == nil { + return + } + for _, pod := range pods { + if pod == nil || pod.UID == "" { + continue + } + cache.Pods[pod.UID] = pod + log.Debugf("substituting pod %v", pod.UID) + } +} diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index 7d8faf0..c5f0637 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -47,11 +47,24 @@ func NewPodManager(publisher api.Publisher) *PodManager { // HandleEvent handles the event from publisher func (manager *PodManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { + switch eventType { + case typedef.RAW_POD_ADD, typedef.RAW_POD_UPDATE, typedef.RAW_POD_DELETE: + manager.handleWatchEvent(eventType, event) + case typedef.RAW_POD_SYNC_ALL: + manager.handleListEvent(eventType, event) + default: + log.Infof("fail to process %s type event", eventType.String()) + } +} + +// handleWatchEvent handles the watch event +func (manager *PodManager) handleWatchEvent(eventType typedef.EventType, event typedef.Event) { pod, err := eventToRawPod(event) if err != nil { log.Warnf(err.Error()) return } + switch eventType { case typedef.RAW_POD_ADD: manager.addFunc(pod) @@ -60,13 +73,32 @@ func (manager *PodManager) HandleEvent(eventType typedef.EventType, event typede case typedef.RAW_POD_DELETE: manager.deleteFunc(pod) default: - log.Infof("fail to process %s type event", eventType.String()) + log.Errorf("code problem, should not go here...") + } +} + +// handleListEvent handles the list event +func (manager *PodManager) handleListEvent(eventType typedef.EventType, event typedef.Event) { + pods, err := eventToRawPods(event) + if err != nil { + log.Errorf(err.Error()) + return + } + switch eventType { + case typedef.RAW_POD_SYNC_ALL: + manager.sync(pods) + default: + log.Errorf("code problem, should not go here...") } } // EventTypes returns the intersted event types func (manager *PodManager) EventTypes() []typedef.EventType { - return []typedef.EventType{typedef.RAW_POD_ADD, typedef.RAW_POD_UPDATE, typedef.RAW_POD_DELETE} + return []typedef.EventType{typedef.RAW_POD_ADD, + typedef.RAW_POD_UPDATE, + typedef.RAW_POD_DELETE, + typedef.RAW_POD_SYNC_ALL, + } } // eventToRawPod converts the event interface to RawPod pointer @@ -79,6 +111,23 @@ func eventToRawPod(e typedef.Event) (*typedef.RawPod, error) { return &rawPod, nil } +// eventToRawPods converts the event interface to RawPod pointer slice +func eventToRawPods(e typedef.Event) ([]*typedef.RawPod, error) { + pods, ok := e.([]corev1.Pod) + if !ok { + return nil, fmt.Errorf("fail to get *typedef.RawPod which type is %T", e) + } + toRawPodPointer := func(pod corev1.Pod) *typedef.RawPod { + tmp := typedef.RawPod(pod) + return &tmp + } + var pointerPods []*typedef.RawPod + for _, pod := range pods { + pointerPods = append(pointerPods, toRawPodPointer(pod)) + } + return pointerPods, nil +} + // addFunc handles the pod add event func (manager *PodManager) addFunc(pod *typedef.RawPod) { // condition 1: only add running pod @@ -156,10 +205,24 @@ func (manager *PodManager) tryDelete(id string) { } } +// sync replaces all Pod information sent over +func (manager *PodManager) sync(pods []*typedef.RawPod) { + var newPods []*typedef.PodInfo + for _, pod := range pods { + if pod == nil || !pod.Running() { + continue + } + newPods = append(newPods, pod.StripInfo()) + } + manager.pods.substitute(newPods) +} + +// ListOfflinePods returns offline pods func (manager *PodManager) ListOfflinePods() ([]*typedef.PodInfo, error) { return nil, nil } +// ListOnlinePods returns online pods func (manager *PodManager) ListOnlinePods() ([]*typedef.PodInfo, error) { return nil, nil } -- Gitee From 6ff010806b6f8855f53417d5cef0ad8da78a267b Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 31 Jan 2023 15:02:01 +0800 Subject: [PATCH 15/73] optimize podInfo & containerInfo 1. use map[string]string to store annotations 2. eliminate parameter CgroupRoot 3. optimize the logic of extracting Pod information and container information --- pkg/common/util/cgroup.go | 20 ++++ pkg/core/typedef/containerinfo.go | 101 ++++++++++++---- pkg/core/typedef/podinfo.go | 193 +++++------------------------- pkg/core/typedef/rawpod.go | 89 ++++++++++++-- pkg/podmanager/podcache.go | 2 +- pkg/podmanager/podmanager.go | 10 +- pkg/rubik/rubik.go | 3 + pkg/services/servicemanager.go | 4 +- 8 files changed, 225 insertions(+), 197 deletions(-) create mode 100644 pkg/common/util/cgroup.go diff --git a/pkg/common/util/cgroup.go b/pkg/common/util/cgroup.go new file mode 100644 index 0000000..7cb475b --- /dev/null +++ b/pkg/common/util/cgroup.go @@ -0,0 +1,20 @@ +package util + +import ( + "path/filepath" + + "isula.org/rubik/pkg/common/constant" +) + +var ( + // CgroupRoot is the unique cgroup mount point globally + CgroupRoot = constant.DefaultCgroupRoot +) + +// AbsoluteCgroupPath returns absolute cgroup path of specified subsystem of a relative path +func AbsoluteCgroupPath(subsys string, relativePath string) string { + if subsys == "" || relativePath == "" { + return "" + } + return filepath.Join(CgroupRoot, subsys, relativePath) +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index dc94a75..2f72f8a 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -14,40 +14,101 @@ // Package typedef defines core struct and methods for rubik package typedef -import "path/filepath" +import ( + "path/filepath" + "strings" + "sync" + + "isula.org/rubik/pkg/common/log" +) + +// ContainerEngineType indicates the type of container engine +type ContainerEngineType int8 + +const ( + UNDEFINED ContainerEngineType = iota + DOCKER + CONTAINERD +) + +var ( + supportEnginesPrefixMap = map[ContainerEngineType]string{ + DOCKER: "docker://", + CONTAINERD: "containerd://", + } + currentContainerEngines = UNDEFINED + setContainerEnginesOnce sync.Once +) + +// Support returns true when the container uses the container engine +func (engine *ContainerEngineType) Support(cont *RawContainer) bool { + if *engine == UNDEFINED { + return false + } + return strings.HasPrefix(cont.ContainerID, engine.Prefix()) +} + +// Prefix returns the ID prefix of the container engine +func (engine *ContainerEngineType) Prefix() string { + prefix, ok := supportEnginesPrefixMap[*engine] + if !ok { + return "" + } + return prefix +} // ContainerInfo contains the interested information of container type ContainerInfo struct { - // Basic Information Name string `json:"name"` ID string `json:"id"` - PodID string `json:"podID"` - CgroupRoot string `json:"cgroupRoot"` - CgroupAddr string `json:"cgroupAddr"` + CgroupPath string `json:"cgroupPath"` } // NewContainerInfo creates a ContainerInfo instance -func NewContainerInfo(container RawContainer, podID, conID, cgroupRoot, podCgroupPath string) *ContainerInfo { - c := ContainerInfo{ - Name: container.Name, - ID: conID, - PodID: podID, - CgroupRoot: cgroupRoot, - CgroupAddr: filepath.Join(podCgroupPath, conID), +func NewContainerInfo(name, id, podCgroupPath string) *ContainerInfo { + return &ContainerInfo{ + Name: name, + ID: id, + CgroupPath: filepath.Join(podCgroupPath, id), + } +} + +func fixContainerEngine(containerID string) { + for engine, prefix := range supportEnginesPrefixMap { + if strings.HasPrefix(containerID, prefix) { + currentContainerEngines = engine + return + } } - return &c + currentContainerEngines = UNDEFINED } -// CgroupPath returns cgroup path of specified subsystem of a container -func (ci *ContainerInfo) CgroupPath(subsys string) string { - if ci == nil || ci.Name == "" { +// getRealContainerID parses the containerID of k8s +func (cont *RawContainer) getRealContainerID() string { + /* + Note: + An UNDEFINED container engine was used when the function was executed for the first time + it seems unlikely to support different container engines at runtime, + So we don't consider the case of midway container engine changes + `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time + */ + setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.ContainerID) }) + + if !currentContainerEngines.Support(cont) { + log.Errorf("fatal error : unsupported container engine") + return "" + } + + cid := cont.ContainerID[len(currentContainerEngines.Prefix()):] + // the container may be in the creation or deletion phase. + if len(cid) == 0 { return "" } - return filepath.Join(ci.CgroupRoot, subsys, ci.CgroupAddr) + return cid } -// Clone returns deepcopy object. -func (ci *ContainerInfo) Clone() *ContainerInfo { - copy := *ci +// DeepCopy returns deepcopy object. +func (info *ContainerInfo) DeepCopy() *ContainerInfo { + copy := *info return © } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index b9d50e8..9aa7be8 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -14,179 +14,48 @@ // Package typedef defines core struct and methods for rubik package typedef -import ( - "path/filepath" - "strconv" - "strings" - - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/common/constant" -) - // PodInfo represents pod type PodInfo struct { - // Basic Information - Containers map[string]*ContainerInfo `json:"containers,omitempty"` - Name string `json:"name"` - UID string `json:"uid"` - CgroupPath string `json:"cgroupPath"` - Namespace string `json:"namespace"` - // TODO: elimate cgroupRoot - CgroupRoot string `json:"cgroupRoot"` - - // TODO: use map[string]interface to replace annotations - // Service Information - Offline bool `json:"offline"` - CacheLimitLevel string `json:"cacheLimitLevel,omitempty"` - - // value of quota burst - QuotaBurst int64 `json:"quotaBurst"` + IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` + Name string `json:"name"` + UID string `json:"uid"` + CgroupPath string `json:"cgroupPath"` + Namespace string `json:"namespace"` + Annotations map[string]string `json:"annotations,omitempty"` } // NewPodInfo creates the PodInfo instance -func NewPodInfo(pod *RawPod, cgroupRoot string) *PodInfo { - pi := &PodInfo{ - Name: pod.Name, - UID: string(pod.UID), - Containers: make(map[string]*ContainerInfo, 0), - CgroupPath: GetPodCgroupPath(pod), - Namespace: pod.Namespace, - CgroupRoot: cgroupRoot, +func NewPodInfo(pod *RawPod) *PodInfo { + return &PodInfo{ + Name: pod.Name, + Namespace: pod.Namespace, + UID: pod.ID(), + CgroupPath: pod.CgroupPath(), + IDContainersMap: pod.ExtractContainerInfos(), + Annotations: pod.DeepCopy().Annotations, } - updatePodInfoNoLock(pi, pod) - return pi } -// updatePodInfoNoLock updates PodInfo from the pod of Kubernetes. -// UpdatePodInfoNoLock does not lock pods during the modification. -// Therefore, ensure that the pod is being used only by this function. -// Currently, the checkpoint manager variable is locked when this function is invoked. -func updatePodInfoNoLock(pi *PodInfo, pod *RawPod) { - const ( - dockerPrefix = "docker://" - containerdPrefix = "containerd://" - ) - pi.Name = pod.Name - pi.Offline = IsOffline(pod) - pi.CacheLimitLevel = GetPodCacheLimit(pod) - pi.QuotaBurst = GetQuotaBurst(pod) - - nameID := make(map[string]string, len(pod.Status.ContainerStatuses)) - for _, c := range pod.Status.ContainerStatuses { - // rubik is compatible with dockerd and containerd container engines. - cid := strings.TrimPrefix(c.ContainerID, dockerPrefix) - cid = strings.TrimPrefix(cid, containerdPrefix) - - // the container may be in the creation or deletion phase. - if len(cid) == 0 { - // log.Debugf("no container id found of container %v", c.Name) - continue - } - nameID[c.Name] = cid - } - // update ContainerInfo in a PodInfo - for _, c := range pod.Spec.Containers { - ci, ok := pi.Containers[c.Name] - // add a container - if !ok { - // log.Debugf("add new container %v", c.Name) - pi.AddContainerInfo(NewContainerInfo(&c, string(pod.UID), nameID[c.Name], - pi.CgroupRoot, pi.CgroupPath)) - continue - } - // The container name remains unchanged, and other information about the container is updated. - ci.ID = nameID[c.Name] - ci.CgroupAddr = filepath.Join(pi.CgroupPath, ci.ID) - } - // delete a container that does not exist - for name := range pi.Containers { - if _, ok := nameID[name]; !ok { - // log.Debugf("delete container %v", name) - delete(pi.Containers, name) - } - } -} - -// Clone returns deepcopy object -func (pi *PodInfo) Clone() *PodInfo { +// DeepCopy returns deepcopy object +func (pi *PodInfo) DeepCopy() *PodInfo { if pi == nil { return nil } - copy := *pi // deepcopy reference object - copy.Containers = make(map[string]*ContainerInfo, len(pi.Containers)) - for _, c := range pi.Containers { - copy.Containers[c.Name] = c.Clone() - } - return © -} - -// AddContainerInfo add container info to pod -func (pi *PodInfo) AddContainerInfo(containerInfo *ContainerInfo) { - // key should not be empty - if containerInfo.Name == "" { - return - } - pi.Containers[containerInfo.Name] = containerInfo -} - -const configHashAnnotationKey = "kubernetes.io/config.hash" - -// IsOffline judges whether pod is offline pod -func IsOffline(pod *RawPod) bool { - return pod.Annotations[constant.PriorityAnnotationKey] == "true" -} - -// GetPodCacheLimit returns cachelimit annotation -func GetPodCacheLimit(pod *RawPod) string { - return pod.Annotations[constant.CacheLimitAnnotationKey] -} - -// ParseInt64 converts the string type to Int64 -func ParseInt64(str string) (int64, error) { - const ( - base = 10 - bitSize = 64 - ) - return strconv.ParseInt(str, base, bitSize) -} - -// GetQuotaBurst checks CPU quota burst annotation value. -func GetQuotaBurst(pod *RawPod) int64 { - quota := pod.Annotations[constant.QuotaBurstAnnotationKey] - if quota == "" { - return constant.InvalidBurst + idContainersMap := make(map[string]*ContainerInfo, len(pi.IDContainersMap)) + for key, value := range pi.IDContainersMap { + idContainersMap[key] = value.DeepCopy() + } + annotations := make(map[string]string) + for key, value := range pi.Annotations { + annotations[key] = value + } + return &PodInfo{ + Name: pi.Name, + UID: pi.UID, + CgroupPath: pi.CgroupPath, + Namespace: pi.Namespace, + Annotations: annotations, + IDContainersMap: idContainersMap, } - - quotaBurst, err := ParseInt64(quota) - if err != nil { - return constant.InvalidBurst - } - if quotaBurst < 0 { - return constant.InvalidBurst - } - return quotaBurst -} - -// GetPodCgroupPath returns cgroup path of pod -func GetPodCgroupPath(pod *RawPod) string { - var cgroupPath string - id := string(pod.UID) - if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { - id = configHash - } - - switch pod.Status.QOSClass { - case corev1.PodQOSGuaranteed: - cgroupPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBurstable: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), - constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBestEffort: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), - constant.PodCgroupNamePrefix+id) - } - - return cgroupPath } diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index 2d5becb..5a5d6b4 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -15,28 +15,46 @@ package typedef import ( + "path/filepath" + "strings" + + "isula.org/rubik/pkg/common/constant" corev1 "k8s.io/api/core/v1" ) -// RUNNING means the Pod is in the running phase -const RUNNING = corev1.PodRunning +const ( + configHashAnnotationKey = "kubernetes.io/config.hash" + // RUNNING means the Pod is in the running phase + RUNNING = corev1.PodRunning +) type ( // RawContainer is kubernetes contaienr structure - RawContainer *corev1.Container + RawContainer struct { + /* + The container information of kubernetes will be stored in pod.Status.ContainerStatuses + and pod.Spec.Containers respectively. + The container ID information is stored in ContainerStatuses. + Currently our use of container information is limited to ID and Name, + so we only implemented a simple RawContainer structure. + You can continue to expand RawContainer in the future, + such as saving the running state of the container. + */ + corev1.ContainerStatus + } // RawPod represents kubernetes pod structure RawPod corev1.Pod ) -// StripInfo strips podInfo from RawPod instance -func (pod *RawPod) StripInfo() *PodInfo { +// ExtractPodInfo returns podInfo from RawPod +func (pod *RawPod) ExtractPodInfo() *PodInfo { if pod == nil { return nil } - return NewPodInfo(pod, "") + return NewPodInfo(pod) } -// Running return true when pod is in the running phase +// Running returns true when pod is in the running phase func (pod *RawPod) Running() bool { if pod == nil { return false @@ -51,3 +69,60 @@ func (pod *RawPod) ID() string { } return string(pod.UID) } + +// CgroupPath returns cgroup path of raw pod +func (pod *RawPod) CgroupPath() string { + id := string(pod.UID) + if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { + id = configHash + } + + switch pod.Status.QOSClass { + case corev1.PodQOSGuaranteed: + return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBurstable: + return filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), + constant.PodCgroupNamePrefix+id) + case corev1.PodQOSBestEffort: + return filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), + constant.PodCgroupNamePrefix+id) + default: + return "" + } +} + +// ListRawContainers returns all RawContainers in the RawPod +func (pod *RawPod) ListRawContainers() map[string]*RawContainer { + if pod == nil { + return nil + } + var nameRawContainersMap = make(map[string]*RawContainer) + for _, containerStatus := range pod.Status.ContainerStatuses { + // Since corev1.Container only exists the container name, use Name as the unique key + nameRawContainersMap[containerStatus.Name] = &RawContainer{ + containerStatus, + } + } + return nameRawContainersMap +} + +// ExtractContainerInfos returns container information from Pod +func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { + var idContainersMap = make(map[string]*ContainerInfo, 0) + // 1. get list of raw containers + nameRawContainersMap := pod.ListRawContainers() + if len(nameRawContainersMap) == 0 { + return idContainersMap + } + + // 2. generate ID-Container mapping + podCgroupPath := pod.CgroupPath() + for _, rawContainer := range nameRawContainersMap { + id := rawContainer.getRealContainerID() + if id == "" { + continue + } + idContainersMap[id] = NewContainerInfo(rawContainer.Name, id, podCgroupPath) + } + return idContainersMap +} diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go index 35aedda..c4f8168 100644 --- a/pkg/podmanager/podcache.go +++ b/pkg/podmanager/podcache.go @@ -38,7 +38,7 @@ func NewPodCache() *podCache { func (cache *podCache) getPod(podID string) *typedef.PodInfo { cache.RLock() defer cache.RUnlock() - return cache.Pods[podID].Clone() + return cache.Pods[podID].DeepCopy() } // podExist returns true if there is a pod whose key is podID in the pods diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index c5f0637..b79be68 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -141,7 +141,7 @@ func (manager *PodManager) addFunc(pod *typedef.RawPod) { return } // step1: get pod information - podInfo := pod.StripInfo() + podInfo := pod.ExtractPodInfo() if podInfo == nil { log.Errorf("fail to strip info from raw pod") return @@ -159,7 +159,7 @@ func (manager *PodManager) updateFunc(pod *typedef.RawPod) { } // add or update information for running pod - podInfo := pod.StripInfo() + podInfo := pod.ExtractPodInfo() if podInfo == nil { log.Errorf("fail to strip info from raw pod") return @@ -181,7 +181,7 @@ func (manager *PodManager) tryAdd(podInfo *typedef.PodInfo) { // only add when pod is not existed if !manager.pods.podExist(podInfo.UID) { manager.pods.addPod(podInfo) - manager.Publish(typedef.INFO_ADD, podInfo) + manager.Publish(typedef.INFO_ADD, podInfo.DeepCopy()) } } @@ -191,7 +191,7 @@ func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { if manager.pods.podExist(podInfo.UID) { oldPod := manager.pods.getPod(podInfo.UID) manager.pods.updatePod(podInfo) - manager.Publish(typedef.INFO_UPDATE, []*typedef.PodInfo{oldPod, podInfo.Clone()}) + manager.Publish(typedef.INFO_UPDATE, []*typedef.PodInfo{oldPod, podInfo.DeepCopy()}) } } @@ -212,7 +212,7 @@ func (manager *PodManager) sync(pods []*typedef.RawPod) { if pod == nil || !pod.Running() { continue } - newPods = append(newPods, pod.StripInfo()) + newPods = append(newPods, pod.ExtractPodInfo()) } manager.pods.substitute(newPods) } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index c31d862..81a16fd 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -24,6 +24,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/publisher" "isula.org/rubik/pkg/informer" @@ -106,9 +107,11 @@ func runAgent(ctx context.Context) error { if err := c.LoadConfig(constant.ConfigFile); err != nil { return fmt.Errorf("error loading config: %v", err) } + // Agent parameter enable if err := log.InitConfig(c.Agent.LogDriver, c.Agent.LogDir, c.Agent.LogLevel, c.Agent.LogSize); err != nil { return fmt.Errorf("error initializing log: %v", err) } + util.CgroupRoot = c.Agent.CgroupRoot agent := NewAgent(c) if err := agent.Run(ctx); err != nil { return fmt.Errorf("error running agent: %v", err) diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 2a2a02d..a732fc7 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -228,7 +228,7 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { manager.RLock() var wg sync.WaitGroup for _, s := range manager.RunningServices { - go runOnce(s, podInfo.Clone(), &wg) + go runOnce(s, podInfo.DeepCopy(), &wg) } wg.Wait() manager.RUnlock() @@ -276,7 +276,7 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { manager.RLock() var wg sync.WaitGroup for _, s := range manager.RunningServices { - go runOnce(s, podInfo.Clone(), &wg) + go runOnce(s, podInfo.DeepCopy(), &wg) } wg.Wait() manager.RUnlock() -- Gitee From ea2d170e555a2a24467dce674e57a4a658559464 Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 31 Jan 2023 19:13:31 +0800 Subject: [PATCH 16/73] clean up code 1. Delete redundant useless code 2. Constant classification Signed-off-by: vegbir --- pkg/common/constant/constant.go | 109 ++++------ pkg/common/tinylog/tinylog.go | 306 ----------------------------- pkg/common/tinylog/tinylog_test.go | 273 ------------------------- pkg/common/util/file.go | 5 +- pkg/common/util/pod.go | 77 -------- pkg/common/util/pod_test.go | 145 -------------- pkg/registry/registry.go | 56 ------ pkg/rubik/rubik.go | 44 +++-- 8 files changed, 69 insertions(+), 946 deletions(-) delete mode 100644 pkg/common/tinylog/tinylog.go delete mode 100644 pkg/common/tinylog/tinylog_test.go delete mode 100644 pkg/common/util/pod.go delete mode 100644 pkg/common/util/pod_test.go delete mode 100644 pkg/registry/registry.go diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index f016532..a87dbbe 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -15,54 +15,43 @@ package constant import ( - "errors" "os" - "time" ) +// the files and directories used by the system by default const ( - // RubikSock is path for rubik socket file - RubikSock = "/run/rubik/rubik.sock" // ConfigFile is rubik config file ConfigFile = "/var/lib/rubik/config.json" // LockFile is rubik lock file LockFile = "/run/rubik/rubik.lock" - // ReadTimeout is timeout for http read - ReadTimeout = 60 * time.Second - // WriteTimeout is timeout for http write - WriteTimeout = 60 * time.Second - // DefaultSucceedCode is succeed code - DefaultSucceedCode = 0 // DefaultCgroupRoot is mount point DefaultCgroupRoot = "/sys/fs/cgroup" - // CPUCgroupFileName is name of cgroup file used for cpu qos level setting - CPUCgroupFileName = "cpu.qos_level" - // MemoryCgroupFileName is name of cgroup file used for memory qos level setting - MemoryCgroupFileName = "memory.qos_level" - // DefaultFileMode is file mode for cgroup files - DefaultFileMode os.FileMode = 0600 - // DefaultDirMode is dir default mode - DefaultDirMode os.FileMode = 0700 - // DefaultUmask is default umask - DefaultUmask = 0077 - // MaxCgroupPathLen is max cgroup path length for pod - MaxCgroupPathLen = 4096 - // MaxPodIDLen is max pod id length - MaxPodIDLen = 256 - // MaxPodsPerRequest is max pods number per http request - MaxPodsPerRequest = 100 // TmpTestDir is tmp directory for test TmpTestDir = "/tmp/rubik-test" - // TaskChanCapacity is capacity for task chan - TaskChanCapacity = 1024 - // WorkerNum is number of workers - WorkerNum = 1 +) + +// kubernetes related configuration +const ( // KubepodsCgroup is kubepods root cgroup KubepodsCgroup = "kubepods" // PodCgroupNamePrefix is pod cgroup name prefix PodCgroupNamePrefix = "pod" // NodeNameEnvKey is node name environment variable key NodeNameEnvKey = "RUBIK_NODE_NAME" +) + +// File permission +const ( + // DefaultUmask is default umask + DefaultUmask = 0077 + // DefaultFileMode is file mode for cgroup files + DefaultFileMode os.FileMode = 0600 + // DefaultDirMode is dir default mode + DefaultDirMode os.FileMode = 0700 +) + +// Pod Annotation +const ( // PriorityAnnotationKey is annotation key to mark offline pod PriorityAnnotationKey = "volcano.sh/preemptable" // CacheLimitAnnotationKey is annotation key to set L3/Mb resctrl group @@ -71,58 +60,30 @@ const ( QuotaBurstAnnotationKey = "volcano.sh/quota-burst-time" // BlkioKey is annotation key to set blkio limit BlkioKey = "volcano.sh/blkio-limit" - // DefaultMemCheckInterval indicates the default memory check interval 5s. - DefaultMemCheckInterval = 5 - // DefaultMaxMemCheckInterval indicates the default max memory check interval 30s. - DefaultMaxMemCheckInterval = 30 - // DefaultMemStrategy indicates the default memory strategy. - DefaultMemStrategy = "none" ) // log config const ( - LogDriverStdio = "stdio" - LogDriverFile = "file" - - LogLevelDebug = "debug" - LogLevelInfo = "info" - LogLevelWarn = "warn" - LogLevelError = "error" - LogLevelStack = "stack" - - // DefaultLogDir is default log dir + LogDriverStdio = "stdio" + LogDriverFile = "file" + LogLevelDebug = "debug" + LogLevelInfo = "info" + LogLevelWarn = "warn" + LogLevelError = "error" + LogLevelStack = "stack" DefaultLogDir = "/var/log/rubik" DefaultLogLevel = LogLevelInfo DefaultLogSize = 1024 ) -// LevelType is type definition of qos level -type LevelType int32 - -const ( - // MinLevel is min level for qos level - MinLevel LevelType = -1 - // MaxLevel is max level for qos level - MaxLevel LevelType = 0 -) - -// Int is type casting for type LevelType -func (l LevelType) Int() int { - return int(l) -} - -const ( - // ErrCodeFailed for normal failed - ErrCodeFailed = 1 -) - -// error define ref from src/internal/oserror/errors.go -var ( - // ErrFileTooBig file too big - ErrFileTooBig = errors.New("file too big") -) - +// exit code const ( - // InvalidBurst for invalid quota burst - InvalidBurst = -1 + // NORMALEXIT for the normal exit code + NormalExitCode int = iota + // ArgumentErrorExitCode for normal failed + ArgumentErrorExitCode + //RepeatRunExitCode for repeat run exit + RepeatRunExitCode + // ErrorExitCode failed during run + ErrorExitCode ) diff --git a/pkg/common/tinylog/tinylog.go b/pkg/common/tinylog/tinylog.go deleted file mode 100644 index 57de2da..0000000 --- a/pkg/common/tinylog/tinylog.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Haomin Tsai -// Create: 2021-09-28 -// Description: This file is used for rubik log - -// Package tinylog is for rubik log -package tinylog - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" - - "isula.org/rubik/pkg/common/constant" -) - -// CtxKey used for UUID -type CtxKey string - -const ( - // UUID is log uuid - UUID = "uuid" - - logStdio = 0 - logFile = 1 - logDriverStdio = "stdio" - logDriverFile = "file" - - logDebug = 0 - logInfo = 1 - logError = 2 - logStack = 20 - logStackFrom = 2 - logLevelInfo = "info" - logLevelStack = "stack" - - logFileNum = 10 - logSizeMin int64 = 10 // 10MB - logSizeMax int64 = 1024 * 1024 // 1TB - unitMB int64 = 1024 * 1024 -) - -var ( - logDriver = logStdio - logFname = filepath.Join(constant.DefaultLogDir, "rubik.log") - logLevel = 0 - logSize int64 = 1024 - logFileMaxSize int64 - logFileSize int64 - - lock = sync.Mutex{} -) - -func makeLogDir(logDir string) error { - if !filepath.IsAbs(logDir) { - return fmt.Errorf("log-dir %v must be an absolute path", logDir) - } - - if err := os.MkdirAll(logDir, constant.DefaultDirMode); err != nil { - return fmt.Errorf("create log directory %v failed", logDir) - } - - return nil -} - -// InitConfig init log config -func InitConfig(driver, logdir, level string, size int64) error { - if driver == "" { - driver = logDriverStdio - } - if driver != logDriverStdio && driver != logDriverFile { - return fmt.Errorf("invalid log driver %s", driver) - } - logDriver = logStdio - if driver == logDriverFile { - logDriver = logFile - } - - if level == "" { - level = logLevelInfo - } - levelstr, err := logLevelFromString(level) - if err != nil { - return err - } - logLevel = levelstr - - if size < logSizeMin || size > logSizeMax { - return fmt.Errorf("invalid log size %d", size) - } - logSize = size - logFileMaxSize = logSize / logFileNum - - if driver == "file" { - if err := makeLogDir(logdir); err != nil { - return err - } - logFname = filepath.Join(logdir, "rubik.log") - if f, err := os.Stat(logFname); err == nil { - atomic.StoreInt64(&logFileSize, f.Size()) - } - } - - return nil -} - -// DropError drop unused error -func DropError(args ...interface{}) { - argn := len(args) - if argn == 0 { - return - } - arg := args[argn-1] - if arg != nil { - fmt.Printf("drop error: %v\n", arg) - } -} - -func logLevelToString(level int) string { - switch level { - case logDebug: - return "debug" - case logInfo: - return "info" - case logError: - return "error" - case logStack: - return logLevelStack - default: - return "" - } -} - -func logLevelFromString(level string) (int, error) { - switch level { - case "debug": - return logDebug, nil - case "info", "": - return logInfo, nil - case "error": - return logError, nil - default: - return logInfo, fmt.Errorf("invalid log level %s", level) - } -} - -func logRename() { - for i := logFileNum - 1; i > 1; i-- { - old := logFname + fmt.Sprintf(".%d", i-1) - new := logFname + fmt.Sprintf(".%d", i) - if _, err := os.Stat(old); err == nil { - DropError(os.Rename(old, new)) - } - } - DropError(os.Rename(logFname, logFname+".1")) -} - -func logRotate(line int64) string { - if atomic.AddInt64(&logFileSize, line) > logFileMaxSize*unitMB { - logRename() - atomic.StoreInt64(&logFileSize, line) - } - - return logFname -} - -func writeLine(line string) { - if logDriver == logStdio { - fmt.Printf("%s", line) - return - } - - lock.Lock() - defer lock.Unlock() - - f, err := os.OpenFile(logRotate(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) - if err != nil { - return - } - - DropError(f.WriteString(line)) - DropError(f.Close()) -} - -func logf(level string, format string, args ...interface{}) { - tag := fmt.Sprintf("%s [rubik] level=%s ", time.Now().Format("2006-01-02 15:04:05.000"), level) - raw := fmt.Sprintf(format, args...) + "\n" - - depth := 1 - if level == logLevelStack { - depth = logStack - } - - for i := logStackFrom; i < logStackFrom+depth; i++ { - line := tag + raw - pc, file, linum, ok := runtime.Caller(i) - if ok { - fs := strings.Split(runtime.FuncForPC(pc).Name(), "/") - fs = strings.Split("."+fs[len(fs)-1], ".") - fn := fs[len(fs)-1] - line = tag + fmt.Sprintf("%s:%d:%s() ", file, linum, fn) + raw - } else if level == logLevelStack { - break - } - writeLine(line) - } -} - -// Logf log info level -func Logf(format string, args ...interface{}) { - if logInfo >= logLevel { - logf(logLevelToString(logInfo), format, args...) - } -} - -// Infof log info level -func Infof(format string, args ...interface{}) { - if logInfo >= logLevel { - logf(logLevelToString(logInfo), format, args...) - } -} - -// Debugf log debug level -func Debugf(format string, args ...interface{}) { - if logDebug >= logLevel { - logf(logLevelToString(logDebug), format, args...) - } -} - -// Errorf log error level -func Errorf(format string, args ...interface{}) { - if logError >= logLevel { - logf(logLevelToString(logError), format, args...) - } -} - -// Stackf log stack dump -func Stackf(format string, args ...interface{}) { - logf("stack", format, args...) -} - -// Entry is log entry -type Entry struct { - Ctx context.Context -} - -// WithCtx create entry with ctx -func WithCtx(ctx context.Context) *Entry { - return &Entry{ - Ctx: ctx, - } -} - -func (e *Entry) level(l int) string { - uuid, ok := e.Ctx.Value(CtxKey(UUID)).(string) - if ok { - return logLevelToString(l) + " UUID=" + uuid - } - return logLevelToString(l) -} - -// Logf write logs -func (e *Entry) Logf(f string, args ...interface{}) { - if logInfo < logLevel { - return - } - logf(e.level(logInfo), f, args...) -} - -// Infof write logs -func (e *Entry) Infof(f string, args ...interface{}) { - if logInfo < logLevel { - return - } - logf(e.level(logInfo), f, args...) -} - -// Debugf write verbose logs -func (e *Entry) Debugf(f string, args ...interface{}) { - if logDebug < logLevel { - return - } - logf(e.level(logDebug), f, args...) -} - -// Errorf write error logs -func (e *Entry) Errorf(f string, args ...interface{}) { - if logError < logLevel { - return - } - logf(e.level(logError), f, args...) -} diff --git a/pkg/common/tinylog/tinylog_test.go b/pkg/common/tinylog/tinylog_test.go deleted file mode 100644 index f5c3719..0000000 --- a/pkg/common/tinylog/tinylog_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2021-05-24 -// Description: This file is used for testing tinylog - -package tinylog - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/try" -) - -// test_rubik_set_logdriver_0001 -func TestInitConfigLogDriver(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - // case: rubik.log already exist. - try.WriteFile(logFilePath, []byte(""), constant.DefaultFileMode) - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - - err = os.RemoveAll(logDir) - assert.NoError(t, err) - - // logDriver is file - err = InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logFile, logDriver) - logString := "Test InitConfig with logDriver file" - Logf(logString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, true, strings.Contains(string(b), logString)) - - // logDriver is stdio - os.Remove(logFilePath) - err = InitConfig("stdio", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logStdio, logDriver) - logString = "Test InitConfig with logDriver stdio" - Logf(logString) - b, err = ioutil.ReadFile(logFilePath) - assert.Equal(t, true, err != nil) - - // logDriver invalid - err = InitConfig("std", logDir, "", logSize) - assert.Equal(t, true, err != nil) - - // logDriver is null - err = InitConfig("", logDir, "", logSize) - assert.NoError(t, err) - assert.Equal(t, logStdio, logDriver) -} - -// test_rubik_set_logdir_0001 -func TestInitConfigLogDir(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - // LogDir valid - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - logString := "Test InitConfig with logDir valid" - Logf(logString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, true, strings.Contains(string(b), logString)) - - // logDir invalid - err = InitConfig("file", "invalid/log", "", logSize) - assert.Equal(t, true, err != nil) -} - -type logTC struct { - name, logLevel string - wantErr, debug, info, error bool -} - -func createLogTC() []logTC { - return []logTC{ - { - name: "TC1-logLevel debug", - logLevel: "debug", - wantErr: false, - debug: true, - info: true, - error: true, - }, - { - name: "TC2-logLevel info", - logLevel: "info", - wantErr: false, - debug: false, - info: true, - error: true, - }, - { - name: "TC3-logLevel error", - logLevel: "error", - wantErr: false, - debug: false, - info: false, - error: true, - }, - { - name: "TC4-logLevel null", - logLevel: "", - wantErr: false, - debug: false, - info: true, - error: true, - }, - { - name: "TC5-logLevel invalid", - logLevel: "inf", - wantErr: true, - }, - } -} - -// test_rubik_set_loglevel_0001 -func TestInitConfigLogLevel(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - debugLogSting, infoLogSting, errorLogSting, logLogString := "Test InitConfig debug log", - "Test InitConfig info log", "Test InitConfig error log", "Test InitConfig log log" - for _, tt := range createLogTC() { - t.Run(tt.name, func(t *testing.T) { - err := InitConfig("file", logDir, tt.logLevel, logSize) - if (err != nil) != tt.wantErr { - t.Errorf("InitConfig() = %v, want %v", err, tt.wantErr) - } else if tt.wantErr == false { - Debugf(debugLogSting) - Infof(infoLogSting) - Errorf(errorLogSting) - Logf(logLogString) - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) - assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) - os.Remove(logFilePath) - - ctx := context.WithValue(context.Background(), CtxKey(UUID), "abc123") - WithCtx(ctx).Debugf(debugLogSting) - WithCtx(ctx).Infof(infoLogSting) - WithCtx(ctx).Errorf(errorLogSting) - WithCtx(ctx).Logf(logLogString) - b, err = ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), infoLogSting)) - assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) - assert.Equal(t, tt.info, strings.Contains(string(b), logLogString)) - assert.Equal(t, true, strings.Contains(string(b), "abc123")) - err = os.RemoveAll(logDir) - assert.NoError(t, err) - } - }) - } -} - -// test_rubik_set_logsize_0001 -func TestInitConfigLogSize(t *testing.T) { - logDir := try.GenTestDir().String() - // LogSize invalid - err := InitConfig("file", logDir, "", logSizeMin-1) - assert.Equal(t, true, err != nil) - err = InitConfig("file", logDir, "", logSizeMax+1) - assert.Equal(t, true, err != nil) - - // logSize valid - testSize, printLine, repeat := 100, 50000, 100 - err = InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - for i := 0; i < printLine; i++ { - Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) - } - err = InitConfig("file", logDir, "", int64(testSize)) - assert.NoError(t, err) - for i := 0; i < printLine; i++ { - Logf(strings.Repeat("TestInitConfigLogSize log", repeat)) - } - var size int64 - err = filepath.Walk(logDir, func(_ string, f os.FileInfo, _ error) error { - size += f.Size() - return nil - }) - assert.NoError(t, err) - assert.Equal(t, true, size < int64(testSize)*unitMB) - err = os.RemoveAll(constant.TmpTestDir) - assert.NoError(t, err) -} - -// TestLogStack is Stackf function test -func TestLogStack(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - Stackf("test stack log") - b, err := ioutil.ReadFile(logFilePath) - assert.NoError(t, err) - fmt.Println(string(b)) - assert.Equal(t, true, strings.Contains(string(b), t.Name())) - line := strings.Split(string(b), "\n") - maxLineNum := 5 - assert.Equal(t, true, len(line) < maxLineNum) -} - -// TestDropError is DropError function test -func TestDropError(t *testing.T) { - logDir := try.GenTestDir().String() - logFilePath := filepath.Join(logDir, "rubik.log") - - err := InitConfig("file", logDir, "", logSize) - assert.NoError(t, err) - DropError() - dropError := "test drop error" - DropError(dropError) - DropError(nil) - _, err = ioutil.ReadFile(logFilePath) - assert.Equal(t, true, err != nil) -} - -// TestLogOthers is log other tests -func TestLogOthers(t *testing.T) { - logDir := filepath.Join(try.GenTestDir().String(), "regular-file") - try.WriteFile(logDir, []byte{}, constant.DefaultFileMode) - - err := makeLogDir(logDir) - assert.Equal(t, true, err != nil) - - level1 := 3 - s := logLevelToString(level1) - assert.Equal(t, "", s) - level2 := 20 - s = logLevelToString(level2) - assert.Equal(t, "stack", s) - - logDriver = 1 - logFname = filepath.Join(constant.TmpTestDir, "log-not-exist") - os.MkdirAll(logFname, constant.DefaultDirMode) - writeLine("abc") - - s = WithCtx(context.Background()).level(1) - assert.Equal(t, "info", s) - - logLevel = logError + 1 - WithCtx(context.Background()).Errorf("abc") -} diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go index 15de5e6..9e44df8 100644 --- a/pkg/common/util/file.go +++ b/pkg/common/util/file.go @@ -14,13 +14,14 @@ package util import ( + "fmt" "io/ioutil" "os" "path/filepath" "syscall" "isula.org/rubik/pkg/common/constant" - log "isula.org/rubik/pkg/common/tinylog" + log "isula.org/rubik/pkg/common/log" ) const ( @@ -58,7 +59,7 @@ func ReadSmallFile(path string) ([]byte, error) { return nil, err } if st.Size() > fileMaxSize { - return nil, constant.ErrFileTooBig + return nil, fmt.Errorf("file too big") } return ioutil.ReadFile(path) // nolint: gosec } diff --git a/pkg/common/util/pod.go b/pkg/common/util/pod.go deleted file mode 100644 index a0e061e..0000000 --- a/pkg/common/util/pod.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Danni Xia -// Create: 2022-05-25 -// Description: Pod related common functions - -package util - -import ( - "path/filepath" - "strings" - - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/common/constant" - log "isula.org/rubik/pkg/common/tinylog" - "isula.org/rubik/pkg/common/typedef" -) - -const configHashAnnotationKey = "kubernetes.io/config.hash" - -// IsOffline judges whether pod is offline pod -func IsOffline(pod *corev1.Pod) bool { - return pod.Annotations[constant.PriorityAnnotationKey] == "true" -} - -func GetPodCacheLimit(pod *corev1.Pod) string { - return pod.Annotations[constant.CacheLimitAnnotationKey] -} - -// GetQuotaBurst checks CPU quota burst annotation value. -func GetQuotaBurst(pod *corev1.Pod) int64 { - quota := pod.Annotations[constant.QuotaBurstAnnotationKey] - if quota == "" { - return constant.InvalidBurst - } - - quotaBurst, err := typedef.ParseInt64(quota) - if err != nil { - log.Errorf("pod %s burst quota annotation value %v is invalid, expect integer", pod.Name, quotaBurst) - return constant.InvalidBurst - } - if quotaBurst < 0 { - log.Errorf("pod %s burst quota annotation value %v is invalid, expect positive", pod.Name, quotaBurst) - return constant.InvalidBurst - } - return quotaBurst -} - -// GetPodCgroupPath returns cgroup path of pod -func GetPodCgroupPath(pod *corev1.Pod) string { - var cgroupPath string - id := string(pod.UID) - if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { - id = configHash - } - - switch pod.Status.QOSClass { - case corev1.PodQOSGuaranteed: - cgroupPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBurstable: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), - constant.PodCgroupNamePrefix+id) - case corev1.PodQOSBestEffort: - cgroupPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), - constant.PodCgroupNamePrefix+id) - } - - return cgroupPath -} diff --git a/pkg/common/util/pod_test.go b/pkg/common/util/pod_test.go deleted file mode 100644 index bc66202..0000000 --- a/pkg/common/util/pod_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jingxiao Lu -// Create: 2022-05-25 -// Description: tests for pod.go - -package util - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - - "isula.org/rubik/pkg/common/constant" -) - -const ( - trueStr = "true" -) - -func TestIsOffline(t *testing.T) { - var pod = &corev1.Pod{} - pod.Annotations = make(map[string]string) - pod.Annotations[constant.PriorityAnnotationKey] = trueStr - if !IsOffline(pod) { - t.Fatalf("%s failed for Annotations is %s", t.Name(), trueStr) - } - - delete(pod.Annotations, constant.PriorityAnnotationKey) - if IsOffline(pod) { - t.Fatalf("%s failed for Annotations no such key", t.Name()) - } -} - -// TestGetQuotaBurst is testcase for GetQuotaBurst -func TestGetQuotaBurst(t *testing.T) { - pod := &corev1.Pod{} - pod.Annotations = make(map[string]string) - maxInt64PlusOne := "9223372036854775808" - tests := []struct { - name string - quotaBurst string - want int64 - }{ - { - name: "TC1-valid quota burst", - quotaBurst: "1", - want: 1, - }, - { - name: "TC2-empty quota burst", - quotaBurst: "", - want: -1, - }, - { - name: "TC3-zero quota burst", - quotaBurst: "0", - want: 0, - }, - { - name: "TC4-negative quota burst", - quotaBurst: "-100", - want: -1, - }, - { - name: "TC5-float quota burst", - quotaBurst: "100.34", - want: -1, - }, - { - name: "TC6-nonnumerical quota burst", - quotaBurst: "nonnumerical", - want: -1, - }, - { - name: "TC7-exceed max int64", - quotaBurst: maxInt64PlusOne, - want: -1, - }, - } - for _, tt := range tests { - pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.quotaBurst - assert.Equal(t, GetQuotaBurst(pod), tt.want) - } -} - -func TestGetPodCgroupPath(t *testing.T) { - var pod = &corev1.Pod{} - pod.UID = "AAA" - var guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+string(pod.UID)) - var burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+string(pod.UID)) - var besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+string(pod.UID)) - pod.Annotations = make(map[string]string) - - // no pod.Annotations[configHashAnnotationKey] - pod.Status.QOSClass = corev1.PodQOSGuaranteed - if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { - t.Fatalf("%s failed for PodQOSGuaranteed without configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBurstable - if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { - t.Fatalf("%s failed for PodQOSBurstable without configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBestEffort - if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { - t.Fatalf("%s failed for PodQOSBestEffort without configHash", t.Name()) - } - pod.Status.QOSClass = "" - if !assert.Equal(t, GetPodCgroupPath(pod), "") { - t.Fatalf("%s failed for not setting QOSClass without configHash", t.Name()) - } - - // has pod.Annotations[configHashAnnotationKey] - pod.Annotations[configHashAnnotationKey] = "BBB" - var id = pod.Annotations[configHashAnnotationKey] - guaranteedPath = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - burstablePath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), constant.PodCgroupNamePrefix+id) - besteffortPath = filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), constant.PodCgroupNamePrefix+id) - pod.Status.QOSClass = corev1.PodQOSGuaranteed - if !assert.Equal(t, GetPodCgroupPath(pod), guaranteedPath) { - t.Fatalf("%s failed for PodQOSGuaranteed with configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBurstable - if !assert.Equal(t, GetPodCgroupPath(pod), burstablePath) { - t.Fatalf("%s failed for PodQOSBurstable with configHash", t.Name()) - } - pod.Status.QOSClass = corev1.PodQOSBestEffort - if !assert.Equal(t, GetPodCgroupPath(pod), besteffortPath) { - t.Fatalf("%s failed for PodQOSBestEffort with configHash", t.Name()) - } - pod.Status.QOSClass = "" - if !assert.Equal(t, GetPodCgroupPath(pod), "") { - t.Fatalf("%s failed for not setting QOSClass with configHash", t.Name()) - } -} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go deleted file mode 100644 index fcc5195..0000000 --- a/pkg/registry/registry.go +++ /dev/null @@ -1,56 +0,0 @@ -package registry - -import ( - "fmt" - - "isula.org/rubik/pkg/api" -) - -type RubikRegistry struct { - services map[string]api.Service -} - -var DefaultRegister = NewRegistry() - -func NewRegistry() RubikRegistry { - return RubikRegistry{ - services: make(map[string]api.Service), - } -} - -func (r *RubikRegistry) Init() error { - fmt.Println("rubik registry Init()") - return nil -} - -func (r *RubikRegistry) Register(s api.Service, name string) error { - fmt.Printf("rubik registry Register(%s)\n", name) - if _, ok := r.services[name]; !ok { - r.services[name] = s - } - return nil -} - -func (r *RubikRegistry) Deregister(s api.Service, name string) error { - fmt.Printf("rubik register Deregister(%s)\n", name) - delete(r.services, name) - return nil -} - -func (r *RubikRegistry) GetService(name string) (api.Service, error) { - fmt.Printf("rubik register GetService(%s)\n", name) - if s, ok := r.services[name]; ok { - return s, nil - } else { - return nil, fmt.Errorf("service %s did not registered", name) - } -} - -func (r *RubikRegistry) ListServices() ([]api.Service, error) { - fmt.Println("rubik register ListServices()") - var services []api.Service - for _, s := range r.services { - services = append(services, s) - } - return services, nil -} diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 81a16fd..04e1e4a 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -21,6 +21,7 @@ import ( "os/signal" "syscall" + "golang.org/x/sys/unix" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" @@ -121,22 +122,39 @@ func runAgent(ctx context.Context) error { // Run runs agent and process signal func Run() int { - ctx, cancel := context.WithCancel(context.Background()) + // 0. file mask permission setting and parameter checking + unix.Umask(constant.DefaultUmask) + if len(os.Args) > 1 { + fmt.Println("args not allowed") + return constant.ArgumentErrorExitCode + } + // 1. apply file locks, only one rubik process can run at the same time + lock, err := util.CreateLockFile(constant.LockFile) + if err != nil { + fmt.Printf("set rubik lock failed: %v, check if there is another rubik running\n", err) + return constant.RepeatRunExitCode + } + defer util.RemoveLockFile(lock, constant.LockFile) - go func() { - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) - for sig := range signalChan { - if sig == syscall.SIGTERM || sig == syscall.SIGINT { - log.Infof("signal %v received and starting exit...", sig) - cancel() - } - } - }() + ctx, cancel := context.WithCancel(context.Background()) + // 2. handle external signals + go handelSignals(cancel) + // 3. run rubik-agent if err := runAgent(ctx); err != nil { log.Errorf("error running rubik agent: %v", err) - return -1 + return constant.ErrorExitCode + } + return constant.NormalExitCode +} + +func handelSignals(cancel context.CancelFunc) { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) + for sig := range signalChan { + if sig == syscall.SIGTERM || sig == syscall.SIGINT { + log.Infof("signal %v received and starting exit...", sig) + cancel() + } } - return 0 } -- Gitee From d8d5f4fd19a15995eb04dad3e12aa5b3560f2322 Mon Sep 17 00:00:00 2001 From: vegbir Date: Wed, 1 Feb 2023 15:51:11 +0800 Subject: [PATCH 17/73] Decoupling log and service 1. independent Log service enhances service scalability 2. log.go optimization 3. add the logic of adding a separate log for the service and adapt it 4. clean up useless packages Signed-off-by: vegbir --- pkg/api/api.go | 12 ++ pkg/common/constant/constant.go | 2 + pkg/common/log/log.go | 27 ++--- pkg/common/log/log_test.go | 6 +- pkg/common/typedef/convert.go | 60 ---------- pkg/common/typedef/convert_test.go | 156 ------------------------- pkg/common/typedef/types.go | 98 ---------------- pkg/common/typedef/types_test.go | 142 ---------------------- pkg/config/config.go | 6 + pkg/core/publisher/genericpublisher.go | 4 +- pkg/services/blkio/blkio.go | 7 +- pkg/services/cachelimit/cachelimit.go | 12 +- pkg/services/servicemanager.go | 12 ++ 13 files changed, 57 insertions(+), 487 deletions(-) delete mode 100644 pkg/common/typedef/convert.go delete mode 100644 pkg/common/typedef/convert_test.go delete mode 100644 pkg/common/typedef/types.go delete mode 100644 pkg/common/typedef/types_test.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 1971ef1..f971ec6 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -88,3 +88,15 @@ type Informer interface { Publisher Start(ctx context.Context) } + +// Logger is the handler to print the log +type Logger interface { + // Errorf logs bugs that affect normal functionality + Errorf(f string, args ...interface{}) + // Warnf logs produce unexpected results + Warnf(f string, args ...interface{}) + // Infof logs normal messages + Infof(f string, args ...interface{}) + // Debugf logs verbose messages + Debugf(f string, args ...interface{}) +} diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index a87dbbe..970ca21 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -74,6 +74,8 @@ const ( DefaultLogDir = "/var/log/rubik" DefaultLogLevel = LogLevelInfo DefaultLogSize = 1024 + // LogEntryKey is the key representing EntryName in the context + LogEntryKey = "module" ) // exit code diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index f5f8fb9..9751b21 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -32,16 +32,12 @@ import ( type CtxKey string const ( - // UUID is log uuid - UUID = "uuid" - - logStack = 20 - logStackFrom = 2 - - logFileNum = 10 - logSizeMin int64 = 10 // 10MB - logSizeMax int64 = 1024 * 1024 // 1TB - unitMB int64 = 1024 * 1024 + logStack = 20 + logStackFrom = 2 + logFileNum = 10 + logSizeMin int64 = 10 // 10MB + logSizeMax int64 = 1024 * 1024 // 1TB + unitMB int64 = 1024 * 1024 ) const ( @@ -63,8 +59,7 @@ var ( logSize int64 = 1024 logFileMaxSize int64 logFileSize int64 - - lock = sync.Mutex{} + lock = sync.Mutex{} ) func makeLogDir(logDir string) error { @@ -272,15 +267,15 @@ func WithCtx(ctx context.Context) *Entry { } func (e *Entry) level(l int) string { - uuid, ok := e.Ctx.Value(CtxKey(UUID)).(string) + id, ok := e.Ctx.Value(CtxKey(constant.LogEntryKey)).(string) if ok { - return logLevelToString(l) + " UUID=" + uuid + return logLevelToString(l) + " " + constant.LogEntryKey + "=" + id } return logLevelToString(l) } -// Logf write logs -func (e *Entry) Logf(f string, args ...interface{}) { +// Warnf write logs +func (e *Entry) Warnf(f string, args ...interface{}) { if logInfo < logLevel { return } diff --git a/pkg/common/log/log_test.go b/pkg/common/log/log_test.go index 40be768..e4d651f 100644 --- a/pkg/common/log/log_test.go +++ b/pkg/common/log/log_test.go @@ -58,7 +58,7 @@ func TestInitConfigLogDriver(t *testing.T) { assert.Equal(t, stdio, logDriver) logString = "Test InitConfig with logDriver stdio" Infof(logString) - b, err = ioutil.ReadFile(logFilePath) + _, err = ioutil.ReadFile(logFilePath) assert.Equal(t, true, err != nil) // logDriver invalid @@ -162,11 +162,11 @@ func TestInitConfigLogLevel(t *testing.T) { assert.Equal(t, tt.error, strings.Contains(string(b), errorLogSting)) os.Remove(logFilePath) - ctx := context.WithValue(context.Background(), CtxKey(UUID), "abc123") + ctx := context.WithValue(context.Background(), CtxKey(constant.LogEntryKey), "abc123") WithCtx(ctx).Debugf(debugLogSting) WithCtx(ctx).Infof(infoLogSting) WithCtx(ctx).Errorf(errorLogSting) - WithCtx(ctx).Logf(logLogString) + WithCtx(ctx).Warnf(logLogString) b, err = ioutil.ReadFile(logFilePath) assert.NoError(t, err) assert.Equal(t, tt.debug, strings.Contains(string(b), debugLogSting)) diff --git a/pkg/common/typedef/convert.go b/pkg/common/typedef/convert.go deleted file mode 100644 index 77af888..0000000 --- a/pkg/common/typedef/convert.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-03-25 -// Description: This package stores basic type conversion functions. - -package typedef - -import ( - "crypto/rand" - "math/big" - "strconv" -) - -// FormatInt64 convert the int 64 type to a string -func FormatInt64(n int64) string { - const base = 10 - return strconv.FormatInt(n, base) -} - -// ParseInt64 convert the string type to Int64 -func ParseInt64(str string) (int64, error) { - const ( - base = 10 - bitSize = 64 - ) - return strconv.ParseInt(str, base, bitSize) -} - -// ParseFloat64 convert the string type to Float64 -func ParseFloat64(str string) (float64, error) { - const bitSize = 64 - return strconv.ParseFloat(str, bitSize) -} - -// FormatFloat64 convert the Float64 type to string -func FormatFloat64(f float64) string { - const ( - precision = -1 - bitSize = 64 - format = 'f' - ) - return strconv.FormatFloat(f, format, precision, bitSize) -} - -// RandInt provide safe rand int in range [0, max) -func RandInt(max int) int { - n, err := rand.Int(rand.Reader, big.NewInt(int64(max))) - if err != nil { - return 0 - } - return int(n.Int64()) -} diff --git a/pkg/common/typedef/convert_test.go b/pkg/common/typedef/convert_test.go deleted file mode 100644 index 35cc50c..0000000 --- a/pkg/common/typedef/convert_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2022-07-10 -// Description: This package stores basic type conversion functions. - -// Package typedef provides a common type conversion method. -package typedef - -import ( - "math" - "testing" -) - -// TestFormatInt64 is testcase for FormatInt64 -func TestFormatInt64(t *testing.T) { - type args struct { - n int64 - } - validNum := 100 - tests := []struct { - name string - args args - want string - }{ - { - name: "TC-convert the int 64 to string", - args: args{n: int64(validNum)}, - want: "100", - }, - { - name: "TC-convert the big int", - args: args{math.MaxInt64}, - want: "9223372036854775807", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := FormatInt64(tt.args.n); got != tt.want { - t.Errorf("FormatInt64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestParseInt64 is testcase for ParseInt64 -func TestParseInt64(t *testing.T) { - type args struct { - str string - } - validNum := 100 - tests := []struct { - name string - args args - want int64 - wantErr bool - }{ - { - name: "TC-convert the int 64 to string", - args: args{str: "100"}, - want: int64(validNum), - }, - { - name: "TC-convert the big int", - args: args{str: "9223372036854775807"}, - want: math.MaxInt64, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseInt64(tt.args.str) - if (err != nil) != tt.wantErr { - t.Errorf("ParseInt64() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("ParseInt64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestFormatFloat64 is testcase for FormatFloat64 -func TestFormatFloat64(t *testing.T) { - type args struct { - f float64 - } - validNum := 100.0 - tests := []struct { - name string - args args - want string - }{ - { - name: "TC-convert the float64 to string", - args: args{f: validNum}, - want: "100", - }, - { - name: "TC-convert the big float", - args: args{math.MaxFloat64}, - want: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := FormatFloat64(tt.args.f); got != tt.want { - t.Errorf("FormatFloat64() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestParseFloat64 is testcase for ParseFloat64 -func TestParseFloat64(t *testing.T) { - type args struct { - str string - } - validNum := 100.0 - tests := []struct { - name string - args args - want float64 - wantErr bool - }{ - { - name: "TC-convert the string to float64", - args: args{str: "100"}, - want: validNum, - }, - { - name: "TC-convert the big float", - args: args{str: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - want: math.MaxFloat64, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseFloat64(tt.args.str) - if (err != nil) != tt.wantErr { - t.Errorf("ParseFloat64() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("ParseFloat64() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/common/typedef/types.go b/pkg/common/typedef/types.go deleted file mode 100644 index a9a1e28..0000000 --- a/pkg/common/typedef/types.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jing Rui -// Create: 2021-04-27 -// Description: This file contains default constants used in the project - -// Package typedef is general used types. -package typedef - -import ( - "path/filepath" - - corev1 "k8s.io/api/core/v1" -) - -// ContainerInfo represent container -type ContainerInfo struct { - // Basic Information - Name string `json:"name"` - ID string `json:"id"` - PodID string `json:"podID"` - CgroupRoot string `json:"cgroupRoot"` - CgroupAddr string `json:"cgroupAddr"` -} - -// NewContainerInfo create container info -func NewContainerInfo(container corev1.Container, podID, conID, cgroupRoot, podCgroupPath string) *ContainerInfo { - c := ContainerInfo{ - Name: container.Name, - ID: conID, - PodID: podID, - CgroupRoot: cgroupRoot, - CgroupAddr: filepath.Join(podCgroupPath, conID), - } - return &c -} - -// CgroupPath return full cgroup path -func (ci *ContainerInfo) CgroupPath(subsys string) string { - if ci == nil || ci.Name == "" { - return "" - } - return filepath.Join(ci.CgroupRoot, subsys, ci.CgroupAddr) -} - -// Clone return deepcopy object. -func (ci *ContainerInfo) Clone() *ContainerInfo { - copy := *ci - return © -} - -// PodInfo represent pod -type PodInfo struct { - // Basic Information - Containers map[string]*ContainerInfo `json:"containers,omitempty"` - Name string `json:"name"` - UID string `json:"uid"` - CgroupPath string `json:"cgroupPath"` - Namespace string `json:"namespace"` - CgroupRoot string `json:"cgroupRoot"` - - // Service Information - Offline bool `json:"offline"` - CacheLimitLevel string `json:"cacheLimitLevel,omitempty"` - - // value of quota burst - QuotaBurst int64 `json:"quotaBurst"` -} - -// Clone return deepcopy object -func (pi *PodInfo) Clone() *PodInfo { - if pi == nil { - return nil - } - copy := *pi - // deepcopy reference object - copy.Containers = make(map[string]*ContainerInfo, len(pi.Containers)) - for _, c := range pi.Containers { - copy.Containers[c.Name] = c.Clone() - } - return © -} - -// AddContainerInfo store container info to checkpoint -func (pi *PodInfo) AddContainerInfo(containerInfo *ContainerInfo) { - // key should not be empty - if containerInfo.Name == "" { - return - } - pi.Containers[containerInfo.Name] = containerInfo -} diff --git a/pkg/common/typedef/types_test.go b/pkg/common/typedef/types_test.go deleted file mode 100644 index ca5f28d..0000000 --- a/pkg/common/typedef/types_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jing Rui -// Create: 2022-07-10 -// Description: This file contains default constants used in the project - -// Package typedef is general used types. -package typedef - -import ( - "io/ioutil" - "log" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/common/constant" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -func init() { - err := os.MkdirAll(constant.TmpTestDir, constant.DefaultDirMode) - if err != nil { - log.Fatalf("Failed to create tmp test dir for testing!") - } -} - -func genContainer() corev1.Container { - c := corev1.Container{} - c.Name = "testContainer" - c.Resources.Requests = make(corev1.ResourceList) - c.Resources.Limits = make(corev1.ResourceList) - c.Resources.Requests["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - c.Resources.Limits["cpu"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - c.Resources.Limits["memory"] = *resource.NewMilliQuantity(10000, resource.DecimalSI) - - return c -} - -// TestNewContainerInfo is testcase for NewContainerInfo -func TestNewContainerInfo(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - - c := genContainer() - type args struct { - container corev1.Container - podID string - conID string - cgroupRoot string - podCgroupPath string - } - tests := []struct { - want *ContainerInfo - name string - args args - }{ - { - name: "TC", - args: args{container: c, podID: "podID", cgroupRoot: cgRoot, conID: "cID", podCgroupPath: podCGPath}, - want: &ContainerInfo{ - Name: "testContainer", - ID: "cID", - PodID: "podID", - CgroupRoot: cgRoot, - CgroupAddr: filepath.Join(podCGPath, "cID"), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := NewContainerInfo(tt.args.container, tt.args.podID, tt.args.conID, tt.args.cgroupRoot, tt.args.podCgroupPath); !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewContainerInfo() = %v, want %v", got, tt.want) - } - }) - } -} - -// TestContainerInfo_CgroupPath is testcase for ContainerInfo.CgroupPath -func TestContainerInfo_CgroupPath(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(podCGPath) - - emptyCi := &ContainerInfo{} - assert.Equal(t, "", emptyCi.CgroupPath("cpu")) - - ci := emptyCi.Clone() - - ci.Name = "testContainer" - ci.ID = "cID" - ci.PodID = "podID" - ci.CgroupRoot = cgRoot - ci.CgroupAddr = filepath.Join(podCGPath, "cID") - assert.Equal(t, ci.CgroupPath("cpu"), - filepath.Join(cgRoot, "cpu", filepath.Join(podCGPath, "cID"))) -} - -// TestPodInfo_Clone is testcase for PodInfo.Clone -func TestPodInfo_Clone(t *testing.T) { - cgRoot, err := ioutil.TempDir(constant.TmpTestDir, "cgRoot") - assert.NoError(t, err) - defer os.RemoveAll(cgRoot) - podCGPath, err := ioutil.TempDir(constant.TmpTestDir, "pod") - assert.NoError(t, err) - defer os.RemoveAll(podCGPath) - emptyPI := &PodInfo{} - pi := emptyPI.Clone() - pi.Containers = make(map[string]*ContainerInfo) - pi.Name = "testPod" - pi.UID = "abcd" - pi.CgroupPath = cgRoot - - containerWithOutName := genContainer() - containerWithOutName.Name = "" - - emptyNameCI := NewContainerInfo(containerWithOutName, "testPod", "cID", cgRoot, podCGPath) - pi.AddContainerInfo(emptyNameCI) - assert.Equal(t, len(pi.Containers), 0) - - ci := NewContainerInfo(genContainer(), "testPod", "cID", cgRoot, podCGPath) - pi.AddContainerInfo(ci) - newPi := pi.Clone() - assert.Equal(t, len(newPi.Containers), 1) -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 455a486..8e4af89 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,11 +1,13 @@ package config import ( + "context" "fmt" "io/ioutil" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/services" ) @@ -109,6 +111,10 @@ func (c *Config) PrepareServices() error { return fmt.Errorf("error unmarshaling %s config: %v", name, err) } + if services.SetLoggerOnService(service, log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { + log.Debugf("set logger for service: %s", name) + } + // try to verify configuration if validator, ok := service.(Validator); ok { if err := validator.Validate(); err != nil { diff --git a/pkg/core/publisher/genericpublisher.go b/pkg/core/publisher/genericpublisher.go index bc6bd1a..adad52b 100644 --- a/pkg/core/publisher/genericpublisher.go +++ b/pkg/core/publisher/genericpublisher.go @@ -79,7 +79,7 @@ func (pub *genericPublisher) Subscribe(s api.Subscriber) error { pub.topicSubscribersMap[topic] = make(subscriberIDs, 0) } pub.topicSubscribersMap[topic][id] = struct{}{} - log.Debugf("%s subscribes topic %s\n", id, topic) + log.Debugf("%s subscribes topic %s", id, topic) } pub.subscribers[id] = s.NotifyFunc pub.Unlock() @@ -96,7 +96,7 @@ func (pub *genericPublisher) Unsubscribe(s api.Subscriber) { pub.Lock() for _, topic := range s.TopicsFunc() { delete(pub.topicSubscribersMap[topic], id) - log.Debugf("%s unsubscribes topic %s\n", id, topic) + log.Debugf("%s unsubscribes topic %s", id, topic) } delete(pub.subscribers, id) pub.Unlock() diff --git a/pkg/services/blkio/blkio.go b/pkg/services/blkio/blkio.go index bc02fbd..522284a 100644 --- a/pkg/services/blkio/blkio.go +++ b/pkg/services/blkio/blkio.go @@ -1,8 +1,6 @@ package blkio import ( - "fmt" - "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" @@ -23,6 +21,7 @@ type BlkioConfig struct { type Blkio struct { Name string `json:"-"` + Log api.Logger } func init() { @@ -36,12 +35,12 @@ func NewBlkio() *Blkio { } func (b *Blkio) PreStart(viewer api.Viewer) error { - fmt.Println("blkio prestart") + b.Log.Debugf("blkio prestart") return nil } func (b *Blkio) Terminate(viewer api.Viewer) error { - fmt.Println("blkio Terminates") + b.Log.Infof("blkio Terminate") return nil } diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go index acf5f35..4d04104 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -2,7 +2,6 @@ package cachelimit import ( "context" - "fmt" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/services" @@ -38,6 +37,7 @@ type CacheLimitConfig struct { type CacheLimit struct { Name string `json:"-"` + Log api.Logger Config CacheLimitConfig } @@ -70,12 +70,12 @@ func NewCacheLimit() *CacheLimit { } func (c *CacheLimit) PreStart(viewer api.Viewer) error { - fmt.Println("cache limit Prestart()") + c.Log.Infof("cache limit Prestart()") return nil } func (c *CacheLimit) Terminate(viewer api.Viewer) error { - fmt.Println("cache limit Terminate()") + c.Log.Infof("cache limit Terminate()") return nil } @@ -84,10 +84,10 @@ func (c *CacheLimit) ID() string { } func (c *CacheLimit) Run(ctx context.Context) { - fmt.Println("cacheLimit Run") + c.Log.Infof("cacheLimit Run") } -func (b *CacheLimit) Validate() error { - fmt.Println("cachelimit Validate()") +func (c *CacheLimit) Validate() error { + c.Log.Infof("cachelimit Validate()") return nil } diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index a732fc7..f5a973d 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -17,6 +17,7 @@ package services import ( "context" "fmt" + "reflect" "sync" "time" @@ -136,6 +137,17 @@ func (manager *ServiceManager) terminatingRunningServices(err error) error { return err } +// SetLoggerOnService assigns a value to the variable Log member if there is a Log field +func SetLoggerOnService(value interface{}, logger api.Logger) bool { + // look for a member variable named Log + field := reflect.ValueOf(value).Elem().FieldByName("Log") + if !field.IsValid() || !field.CanSet() || field.Type().String() != "api.Logger" { + return false + } + field.Set(reflect.ValueOf(logger)) + return true +} + // Setup pre-starts services, such as preparing the environment, etc. func (manager *ServiceManager) Setup(v api.Viewer) error { // only when viewer is prepared -- Gitee From 868f7a27da72501c0ca855aeed084f1ed4e4fb40 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 2 Feb 2023 10:06:47 +0800 Subject: [PATCH 18/73] clean code 1. add comments 2. rename variables Signed-off-by: vegbir --- pkg/api/api.go | 3 ++ pkg/common/constant/constant.go | 2 +- pkg/common/log/log.go | 46 +++++++++---------- pkg/common/log/log_test.go | 4 +- pkg/common/util/cgroup.go | 14 ++++++ pkg/common/util/file.go | 5 +- pkg/config/config.go | 18 +++++++- pkg/config/config_test.go | 14 ++++++ pkg/config/jsonparser.go | 16 ++++++- pkg/config/parserfactory.go | 14 ++++++ ...ublisherFactory.go => publisherfactory.go} | 2 +- pkg/core/subscriber/genericsubscriber.go | 14 ++++++ pkg/core/typedef/containerinfo.go | 13 ++++-- pkg/core/typedef/event.go | 18 ++++---- pkg/core/typedef/rawpod.go | 9 ++-- pkg/informer/informerfactory.go | 3 +- pkg/podmanager/podcache.go | 2 +- pkg/rubik/import.go | 16 ++++++- pkg/rubik/rubik.go | 3 +- pkg/services/registry.go | 14 ++++++ pkg/services/servicemanager.go | 6 ++- 21 files changed, 181 insertions(+), 55 deletions(-) rename pkg/core/publisher/{publisherFactory.go => publisherfactory.go} (95%) diff --git a/pkg/api/api.go b/pkg/api/api.go index f971ec6..9ebd28a 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -29,10 +29,12 @@ type Registry interface { ListServices() ([]*Service, error) } +// ServiceDescriber describes services type ServiceDescriber interface { ID() string } +// EventFunc is the event handler for Service type EventFunc interface { AddFunc(podInfo *typedef.PodInfo) error UpdateFunc(old, new *typedef.PodInfo) error @@ -52,6 +54,7 @@ type PersistentService interface { Run(ctx context.Context) } +// ConfigParser is a configuration parser for different languages type ConfigParser interface { ParseConfig(data []byte) (map[string]interface{}, error) UnmarshalSubConfig(data interface{}, v interface{}) error diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index 970ca21..c817f0f 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -84,7 +84,7 @@ const ( NormalExitCode int = iota // ArgumentErrorExitCode for normal failed ArgumentErrorExitCode - //RepeatRunExitCode for repeat run exit + // RepeatRunExitCode for repeat run exit RepeatRunExitCode // ErrorExitCode failed during run ErrorExitCode diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index 9751b21..85e9e61 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -90,7 +90,7 @@ func InitConfig(driver, logdir, level string, size int64) error { if level == "" { level = constant.LogLevelInfo } - levelstr, err := logLevelFromString(level) + levelstr, err := levelFromString(level) if err != nil { return err } @@ -127,7 +127,7 @@ func DropError(args ...interface{}) { } } -func logLevelToString(level int) string { +func levelToString(level int) string { switch level { case logDebug: return constant.LogLevelDebug @@ -144,7 +144,7 @@ func logLevelToString(level int) string { } } -func logLevelFromString(level string) (int, error) { +func levelFromString(level string) (int, error) { switch level { case constant.LogLevelDebug: return logDebug, nil @@ -159,20 +159,20 @@ func logLevelFromString(level string) (int, error) { } } -func logRename() { +func renameLogFile() { for i := logFileNum - 1; i > 1; i-- { - old := logFname + fmt.Sprintf(".%d", i-1) - new := logFname + fmt.Sprintf(".%d", i) - if _, err := os.Stat(old); err == nil { - DropError(os.Rename(old, new)) + oldFile := logFname + fmt.Sprintf(".%d", i-1) + newFile := logFname + fmt.Sprintf(".%d", i) + if _, err := os.Stat(oldFile); err == nil { + DropError(os.Rename(oldFile, newFile)) } } DropError(os.Rename(logFname, logFname+".1")) } -func logRotate(line int64) string { +func rotateLog(line int64) string { if atomic.AddInt64(&logFileSize, line) > logFileMaxSize*unitMB { - logRename() + renameLogFile() atomic.StoreInt64(&logFileSize, line) } @@ -188,7 +188,7 @@ func writeLine(line string) { lock.Lock() defer lock.Unlock() - f, err := os.OpenFile(logRotate(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) + f, err := os.OpenFile(rotateLog(int64(len(line))), os.O_CREATE|os.O_APPEND|os.O_WRONLY, constant.DefaultFileMode) if err != nil { return } @@ -197,7 +197,7 @@ func writeLine(line string) { DropError(f.Close()) } -func logf(level string, format string, args ...interface{}) { +func output(level string, format string, args ...interface{}) { tag := fmt.Sprintf("%s [rubik] level=%s ", time.Now().Format("2006-01-02 15:04:05.000"), level) raw := fmt.Sprintf(format, args...) + "\n" @@ -224,34 +224,34 @@ func logf(level string, format string, args ...interface{}) { // Warnf log warn level func Warnf(format string, args ...interface{}) { if logWarn >= logLevel { - logf(logLevelToString(logInfo), format, args...) + output(levelToString(logInfo), format, args...) } } // Infof log info level func Infof(format string, args ...interface{}) { if logInfo >= logLevel { - logf(logLevelToString(logInfo), format, args...) + output(levelToString(logInfo), format, args...) } } // Debugf log debug level func Debugf(format string, args ...interface{}) { if logDebug >= logLevel { - logf(logLevelToString(logDebug), format, args...) + output(levelToString(logDebug), format, args...) } } // Errorf log error level func Errorf(format string, args ...interface{}) { if logError >= logLevel { - logf(logLevelToString(logError), format, args...) + output(levelToString(logError), format, args...) } } // Stackf log stack dump func Stackf(format string, args ...interface{}) { - logf("stack", format, args...) + output("stack", format, args...) } // Entry is log entry @@ -269,9 +269,9 @@ func WithCtx(ctx context.Context) *Entry { func (e *Entry) level(l int) string { id, ok := e.Ctx.Value(CtxKey(constant.LogEntryKey)).(string) if ok { - return logLevelToString(l) + " " + constant.LogEntryKey + "=" + id + return levelToString(l) + " " + constant.LogEntryKey + "=" + id } - return logLevelToString(l) + return levelToString(l) } // Warnf write logs @@ -279,7 +279,7 @@ func (e *Entry) Warnf(f string, args ...interface{}) { if logInfo < logLevel { return } - logf(e.level(logInfo), f, args...) + output(e.level(logInfo), f, args...) } // Infof write logs @@ -287,7 +287,7 @@ func (e *Entry) Infof(f string, args ...interface{}) { if logInfo < logLevel { return } - logf(e.level(logInfo), f, args...) + output(e.level(logInfo), f, args...) } // Debugf write verbose logs @@ -295,7 +295,7 @@ func (e *Entry) Debugf(f string, args ...interface{}) { if logDebug < logLevel { return } - logf(e.level(logDebug), f, args...) + output(e.level(logDebug), f, args...) } // Errorf write error logs @@ -303,5 +303,5 @@ func (e *Entry) Errorf(f string, args ...interface{}) { if logError < logLevel { return } - logf(e.level(logError), f, args...) + output(e.level(logError), f, args...) } diff --git a/pkg/common/log/log_test.go b/pkg/common/log/log_test.go index e4d651f..2b2e3be 100644 --- a/pkg/common/log/log_test.go +++ b/pkg/common/log/log_test.go @@ -254,10 +254,10 @@ func TestLogOthers(t *testing.T) { assert.Equal(t, true, err != nil) const outOfRangeLogLevel = 100 - s := logLevelToString(outOfRangeLogLevel) + s := levelToString(outOfRangeLogLevel) assert.Equal(t, "", s) const stackLoglevel = 20 - s = logLevelToString(stackLoglevel) + s = levelToString(stackLoglevel) assert.Equal(t, "stack", s) logDriver = 1 diff --git a/pkg/common/util/cgroup.go b/pkg/common/util/cgroup.go index 7cb475b..26553e8 100644 --- a/pkg/common/util/cgroup.go +++ b/pkg/common/util/cgroup.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-02-01 +// Description: This file contains commom cgroup operation + +// Package util is common utilitization package util import ( diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go index 9e44df8..68bb996 100644 --- a/pkg/common/util/file.go +++ b/pkg/common/util/file.go @@ -21,7 +21,7 @@ import ( "syscall" "isula.org/rubik/pkg/common/constant" - log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/log" ) const ( @@ -33,8 +33,7 @@ func CreateFile(path string) error { if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { return err } - - f, err := os.Create(path) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DefaultFileMode) if err != nil { return err } diff --git a/pkg/config/config.go b/pkg/config/config.go index 8e4af89..bfb03c7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-02-01 +// Description: This file contains configuration content and provides external interaction functions + +// Package config is used to manage the configuration of rubik package config import ( @@ -111,7 +125,8 @@ func (c *Config) PrepareServices() error { return fmt.Errorf("error unmarshaling %s config: %v", name, err) } - if services.SetLoggerOnService(service, log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { + if services.SetLoggerOnService(service, + log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { log.Debugf("set logger for service: %s", name) } @@ -126,6 +141,7 @@ func (c *Config) PrepareServices() error { return nil } +// Validator is a function interface to verify whether the configuration is correct or not type Validator interface { Validate() error } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 94a0dc8..02631d2 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-02-01 +// Description: This file is used to test the functions of config.go + +// Package config is used to manage the configuration of rubik package config import ( diff --git a/pkg/config/jsonparser.go b/pkg/config/jsonparser.go index 7c37a11..17fa586 100644 --- a/pkg/config/jsonparser.go +++ b/pkg/config/jsonparser.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-02-01 +// Description: This file contains parsing functions for the json language + +// Package config is used to manage the configuration of rubik package config import ( @@ -11,7 +25,7 @@ var defaultJsonParser *jsonParser // jsonParser is used to parse json type jsonParser struct{} -//getJsonParser gets the globally unique json parser +// getJsonParser gets the globally unique json parser func getJsonParser() *jsonParser { if defaultJsonParser == nil { defaultJsonParser = &jsonParser{} diff --git a/pkg/config/parserfactory.go b/pkg/config/parserfactory.go index 9f3d22b..921eeba 100644 --- a/pkg/config/parserfactory.go +++ b/pkg/config/parserfactory.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-02-01 +// Description: This file contains factory classes for configuring parsers for different languages + +// Package config is used to manage the configuration of rubik package config import "isula.org/rubik/pkg/api" diff --git a/pkg/core/publisher/publisherFactory.go b/pkg/core/publisher/publisherfactory.go similarity index 95% rename from pkg/core/publisher/publisherFactory.go rename to pkg/core/publisher/publisherfactory.go index 51f51eb..f5a8038 100644 --- a/pkg/core/publisher/publisherFactory.go +++ b/pkg/core/publisher/publisherfactory.go @@ -29,7 +29,7 @@ type PublisherFactory struct { var publisherFactory = &PublisherFactory{} -// NewPublisherFactory creates a publisher factory instance +// GetPublisherFactory creates a publisher factory instance func GetPublisherFactory() *PublisherFactory { return publisherFactory } diff --git a/pkg/core/subscriber/genericsubscriber.go b/pkg/core/subscriber/genericsubscriber.go index 901cba0..5df89d2 100644 --- a/pkg/core/subscriber/genericsubscriber.go +++ b/pkg/core/subscriber/genericsubscriber.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file implements the generic subscriber functionality + +// Package subscriber implements generic subscriber interface package subscriber import ( diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 2f72f8a..6dc746e 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -26,8 +26,11 @@ import ( type ContainerEngineType int8 const ( + // UNDEFINED means undefined container engine UNDEFINED ContainerEngineType = iota + // DOCKER means docker container engine DOCKER + // CONTAINERD means containerd container engine CONTAINERD ) @@ -45,7 +48,7 @@ func (engine *ContainerEngineType) Support(cont *RawContainer) bool { if *engine == UNDEFINED { return false } - return strings.HasPrefix(cont.ContainerID, engine.Prefix()) + return strings.HasPrefix(cont.status.ContainerID, engine.Prefix()) } // Prefix returns the ID prefix of the container engine @@ -92,14 +95,14 @@ func (cont *RawContainer) getRealContainerID() string { So we don't consider the case of midway container engine changes `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time */ - setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.ContainerID) }) + setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.status.ContainerID) }) if !currentContainerEngines.Support(cont) { log.Errorf("fatal error : unsupported container engine") return "" } - cid := cont.ContainerID[len(currentContainerEngines.Prefix()):] + cid := cont.status.ContainerID[len(currentContainerEngines.Prefix()):] // the container may be in the creation or deletion phase. if len(cid) == 0 { return "" @@ -109,6 +112,6 @@ func (cont *RawContainer) getRealContainerID() string { // DeepCopy returns deepcopy object. func (info *ContainerInfo) DeepCopy() *ContainerInfo { - copy := *info - return © + copyObject := *info + return ©Object } diff --git a/pkg/core/typedef/event.go b/pkg/core/typedef/event.go index 9e74a59..65b7d84 100644 --- a/pkg/core/typedef/event.go +++ b/pkg/core/typedef/event.go @@ -15,26 +15,26 @@ package typedef type ( - // the type of event published by generic publisher + // EventType is the type of event published by generic publisher EventType int8 - // the event published by generic publisher + // Event is the event published by generic publisher Event interface{} ) const ( - // Kubernetes starts a new Pod event + // RAW_POD_ADD means Kubernetes starts a new Pod event RAW_POD_ADD EventType = iota - // Kubernetes updates Pod event + // RAW_POD_UPDATE means Kubernetes updates Pod event RAW_POD_UPDATE - // Kubernetes deletes Pod event + // RAW_POD_DELETE means Kubernetes deletes Pod event RAW_POD_DELETE - // PodManager adds pod information event + // INFO_ADD means PodManager adds pod information event INFO_ADD - // PodManager updates pod information event + // INFO_UPDATE means PodManager updates pod information event INFO_UPDATE - // PodManager deletes pod information event + // INFO_DELETE means PodManager deletes pod information event INFO_DELETE - // Full amount of kubernetes pods + // RAW_POD_SYNC_ALL means Full amount of kubernetes pods RAW_POD_SYNC_ALL ) diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index 5a5d6b4..96c45af 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -18,8 +18,9 @@ import ( "path/filepath" "strings" - "isula.org/rubik/pkg/common/constant" corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/common/constant" ) const ( @@ -40,7 +41,7 @@ type ( You can continue to expand RawContainer in the future, such as saving the running state of the container. */ - corev1.ContainerStatus + status corev1.ContainerStatus } // RawPod represents kubernetes pod structure RawPod corev1.Pod @@ -100,7 +101,7 @@ func (pod *RawPod) ListRawContainers() map[string]*RawContainer { for _, containerStatus := range pod.Status.ContainerStatuses { // Since corev1.Container only exists the container name, use Name as the unique key nameRawContainersMap[containerStatus.Name] = &RawContainer{ - containerStatus, + status: containerStatus, } } return nameRawContainersMap @@ -122,7 +123,7 @@ func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { if id == "" { continue } - idContainersMap[id] = NewContainerInfo(rawContainer.Name, id, podCgroupPath) + idContainersMap[id] = NewContainerInfo(rawContainer.status.Name, id, podCgroupPath) } return idContainersMap } diff --git a/pkg/informer/informerfactory.go b/pkg/informer/informerfactory.go index f46a492..8a3df5e 100644 --- a/pkg/informer/informerfactory.go +++ b/pkg/informer/informerfactory.go @@ -25,6 +25,7 @@ type ( informerType int8 // informer's factory class informerFactory struct{} + informerCreator func(publisher api.Publisher) (api.Informer, error) ) const ( @@ -36,7 +37,7 @@ const ( var defaultInformerFactory *informerFactory // GetInformerCreator returns the constructor of the informer of the specified type -func (factory *informerFactory) GetInformerCreator(iType informerType) func(publisher api.Publisher) (api.Informer, error) { +func (factory *informerFactory) GetInformerCreator(iType informerType) informerCreator { switch iType { case APISERVER: return NewAPIServerInformer diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go index c4f8168..7179f92 100644 --- a/pkg/podmanager/podcache.go +++ b/pkg/podmanager/podcache.go @@ -92,7 +92,7 @@ func (cache *podCache) substitute(pods []*typedef.PodInfo) { cache.Lock() defer cache.Unlock() cache.Pods = make(map[string]*typedef.PodInfo, 0) - if pods == nil { + if len(pods) == 0 { return } for _, pod := range pods { diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index 4d970c3..aad0c19 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -1,7 +1,21 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines the services supported by rubik + +// Package rubik defines the overall logic package rubik import ( - // import packages to auto register service + // introduce packages to auto register service _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/services/cachelimit" _ "isula.org/rubik/pkg/version" diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 04e1e4a..e5da815 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -11,7 +11,7 @@ // Create: 2023-01-28 // Description: This file defines rubik agent to control the life cycle of each component -// // Package rubik defines the overall logic +// Package rubik defines the overall logic package rubik import ( @@ -22,6 +22,7 @@ import ( "syscall" "golang.org/x/sys/unix" + "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" diff --git a/pkg/services/registry.go b/pkg/services/registry.go index 986c461..8588865 100644 --- a/pkg/services/registry.go +++ b/pkg/services/registry.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-28 +// Description: This file defines registry for service registration + +// Package services implements service registration, discovery and management functions package services import ( diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index f5a973d..4903e76 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -21,11 +21,12 @@ import ( "sync" "time" + "k8s.io/apimachinery/pkg/util/wait" + "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/subscriber" "isula.org/rubik/pkg/core/typedef" - "k8s.io/apimachinery/pkg/util/wait" ) // serviceManagerName is the unique ID of the service manager @@ -294,9 +295,12 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { manager.RUnlock() } +// Terminator is an interface that calls a collection of methods when the service terminates type Terminator interface { Terminate(api.Viewer) error } + +// PreStarter is an interface for calling a collection of methods when the service is pre-started type PreStarter interface { PreStart(api.Viewer) error } -- Gitee From 1579ec58d5713ebfd447ab799437eada3905d890 Mon Sep 17 00:00:00 2001 From: vegbir Date: Wed, 8 Feb 2023 22:09:53 +0800 Subject: [PATCH 19/73] add pod information and common functions (such as reading and writing cgroups) 1. add more useful common functions 2. containerInfo supports resource fields 3. decoupling util and log --- .gitignore | 1 + pkg/common/util/cgroup.go | 19 ++++++ pkg/common/util/conversion.go | 74 ++++++++++++++++++++ pkg/common/util/file.go | 109 ++++++++++++++++++++++-------- pkg/common/util/file_test.go | 19 +++--- pkg/common/util/math.go | 63 +++++++++++++++++ pkg/core/typedef/containerinfo.go | 78 +++++++++++++-------- pkg/core/typedef/podinfo.go | 45 +++++++----- pkg/core/typedef/rawpod.go | 83 ++++++++++++++++++++--- pkg/rubik/rubik.go | 9 ++- 10 files changed, 409 insertions(+), 91 deletions(-) create mode 100644 pkg/common/util/conversion.go create mode 100644 pkg/common/util/math.go diff --git a/.gitignore b/.gitignore index d9c35d3..e056f68 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .out .vscode .fortest +/unit_test.log diff --git a/pkg/common/util/cgroup.go b/pkg/common/util/cgroup.go index 26553e8..d65aae6 100644 --- a/pkg/common/util/cgroup.go +++ b/pkg/common/util/cgroup.go @@ -15,6 +15,7 @@ package util import ( + "fmt" "path/filepath" "isula.org/rubik/pkg/common/constant" @@ -32,3 +33,21 @@ func AbsoluteCgroupPath(subsys string, relativePath string) string { } return filepath.Join(CgroupRoot, subsys, relativePath) } + +// ReadCgroupFile reads data from cgroup files +func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) { + cgfile := filepath.Join(CgroupRoot, subsys, cgroupParent, cgroupFileName) + if !PathExist(cgfile) { + return nil, fmt.Errorf("%v: no such file or diretory", cgfile) + } + return ReadFile(cgfile) +} + +// WriteCgroupFile writes data to cgroup file +func WriteCgroupFile(subsys, cgroupParent, cgroupFileName string, content string) error { + cgfile := filepath.Join(CgroupRoot, subsys, cgroupParent, cgroupFileName) + if !PathExist(cgfile) { + return fmt.Errorf("%v: no such file or diretory", cgfile) + } + return WriteFile(cgfile, content) +} diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go new file mode 100644 index 0000000..f6c2019 --- /dev/null +++ b/pkg/common/util/conversion.go @@ -0,0 +1,74 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-08 +// Description: This package stores basic type conversion functions. + +// Package util is common utilitization +package util + +import ( + "strconv" +) + +// FormatInt64 convert the int 64 type to a string +func FormatInt64(n int64) string { + const base = 10 + return strconv.FormatInt(n, base) +} + +// ParseInt64 convert the string type to Int64 +func ParseInt64(str string) (int64, error) { + const ( + base = 10 + bitSize = 64 + ) + return strconv.ParseInt(str, base, bitSize) +} + +// ParseFloat64 convert the string type to Float64 +func ParseFloat64(str string) (float64, error) { + const bitSize = 64 + return strconv.ParseFloat(str, bitSize) +} + +// FormatFloat64 convert the Float64 type to string +func FormatFloat64(f float64) string { + const ( + precision = -1 + bitSize = 64 + format = 'f' + ) + return strconv.FormatFloat(f, format, precision, bitSize) +} + +// PercentageToDecimal converts percentages to decimals +func PercentageToDecimal(num float64) float64 { + const percentageofOne float64 = 100 + return Div(num, percentageofOne) +} + +// DeepCopy deep copy slice or map type data +func DeepCopy(value interface{}) interface{} { + if valueMap, ok := value.(map[string]interface{}); ok { + newMap := make(map[string]interface{}) + for k, v := range valueMap { + newMap[k] = DeepCopy(v) + } + return newMap + } else if valueSlice, ok := value.([]interface{}); ok { + newSlice := make([]interface{}, len(valueSlice)) + for k, v := range valueSlice { + newSlice[k] = DeepCopy(v) + } + return newSlice + } + return value +} diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go index 68bb996..22b4150 100644 --- a/pkg/common/util/file.go +++ b/pkg/common/util/file.go @@ -21,7 +21,6 @@ import ( "syscall" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/log" ) const ( @@ -29,20 +28,21 @@ const ( ) // CreateFile create full path including dir and file. -func CreateFile(path string) error { +func CreateFile(path string) (*os.File, error) { + path = filepath.Clean(path) if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { - return err + return nil, err } f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DefaultFileMode) if err != nil { - return err + return nil, err } - return f.Close() + return f, nil } -// IsDirectory returns true if the file exists and it is a dir -func IsDirectory(path string) bool { +// IsDir returns true if the file exists and it is a dir +func IsDir(path string) bool { fi, err := os.Lstat(path) if err != nil { return false @@ -53,14 +53,27 @@ func IsDirectory(path string) bool { // ReadSmallFile read small file less than 10MB func ReadSmallFile(path string) ([]byte, error) { - st, err := os.Lstat(path) + size, err := FileSize(path) if err != nil { return nil, err } - if st.Size() > fileMaxSize { + + if size > fileMaxSize { return nil, fmt.Errorf("file too big") } - return ioutil.ReadFile(path) // nolint: gosec + return ReadFile(path) +} + +// FileSize returns the size of file +func FileSize(path string) (int64, error) { + if !PathExist(path) { + return 0, fmt.Errorf("%v: No such file or directory", path) + } + st, err := os.Lstat(path) + if err != nil { + return 0, err + } + return st.Size(), nil } // PathExist returns true if the path exists @@ -72,30 +85,72 @@ func PathExist(path string) bool { return true } -// CreateLockFile creates a lock file -func CreateLockFile(p string) (*os.File, error) { - path := filepath.Clean(p) - if err := os.MkdirAll(filepath.Dir(path), constant.DefaultDirMode); err != nil { +// LockFile locks a file, creating a file if it does not exist +func LockFile(path string) (*os.File, error) { + lock, err := CreateFile(path) + if err != nil { return nil, err } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return lock, err + } + return lock, nil +} - lock, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DefaultFileMode) - if err != nil { - return nil, err +// UnlockFile unlock file - this function used cleanup resource, +func UnlockFile(lock *os.File) error { + // errors will ignored to make sure more source is cleaned. + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); err != nil { + return err } + return nil +} - if err = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - log.DropError(lock.Close()) - return nil, err +// ReadFile reads a file +func ReadFile(path string) ([]byte, error) { + if IsDir(path) { + return nil, fmt.Errorf("%v is not a file", path) } + if !PathExist(path) { + return nil, fmt.Errorf("%v: No such file or directory", path) + } + return ioutil.ReadFile(path) +} - return lock, nil +// WriteFile writes a file, if the file does not exist, create the file (including the upper directory) +func WriteFile(path, content string) error { + if IsDir(path) { + return fmt.Errorf("%v is not a file", path) + } + // try to create Pparent directory + dirPath := filepath.Dir(path) + if !PathExist(dirPath) { + if err := os.MkdirAll(dirPath, constant.DefaultDirMode); err != nil { + return fmt.Errorf("error create dir %v: %v", dirPath, err) + } + } + return ioutil.WriteFile(path, []byte(content), constant.DefaultFileMode) } -// RemoveLockFile removes lock file - this function used cleanup resource, -// errors will ignored to make sure more source is cleaned. -func RemoveLockFile(lock *os.File, path string) { - log.DropError(syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)) - log.DropError(lock.Close()) - log.DropError(os.Remove(path)) +// AppendFile appends content to the file +func AppendFile(path, content string) error { + if IsDir(path) { + return fmt.Errorf("%v is not a file", path) + } + if !PathExist(path) { + return fmt.Errorf("%v: No such file or directory", path) + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, constant.DefaultFileMode) + defer func() { + if err != f.Close() { + return + } + }() + if err != nil { + return fmt.Errorf("error open file: %v", err) + } + if _, err := f.WriteString(content); err != nil { + return fmt.Errorf("error write file: %v", err) + } + return nil } diff --git a/pkg/common/util/file_test.go b/pkg/common/util/file_test.go index 63fc637..62a7de9 100644 --- a/pkg/common/util/file_test.go +++ b/pkg/common/util/file_test.go @@ -60,7 +60,7 @@ func TestIsDirectory(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := IsDirectory(tt.args.path); got != tt.want { + if got := IsDir(tt.args.path); got != tt.want { t.Errorf("IsDirectory() = %v, want %v", got, tt.want) } }) @@ -117,7 +117,7 @@ func TestReadSmallFile(t *testing.T) { // case2: too big size := 20000000 - big := make([]byte, size, size) + big := make([]byte, size) err = ioutil.WriteFile(filepath.Join(filePath, "big"), big, constant.DefaultFileMode) assert.NoError(t, err) _, err = ReadSmallFile(filepath.Join(filePath, "big")) @@ -134,9 +134,12 @@ func TestCreateLockFile(t *testing.T) { err := os.RemoveAll(lockFile) assert.NoError(t, err) - lock, err := CreateLockFile(lockFile) + lock, err := LockFile(lockFile) assert.NoError(t, err) - RemoveLockFile(lock, lockFile) + UnlockFile(lock) + assert.NoError(t, lock.Close()) + assert.NoError(t, os.Remove(lockFile)) + } // TestLockFail is CreateLockFile fail test @@ -148,21 +151,21 @@ func TestLockFail(t *testing.T) { _, err = os.Create(filepath.Join(constant.TmpTestDir, "rubik-lock")) assert.NoError(t, err) - _, err = CreateLockFile(filepath.Join(constant.TmpTestDir, "rubik-lock", "rubik.lock")) + _, err = LockFile(filepath.Join(constant.TmpTestDir, "rubik-lock", "rubik.lock")) assert.Equal(t, true, err != nil) err = os.RemoveAll(filepath.Join(constant.TmpTestDir, "rubik-lock")) assert.NoError(t, err) err = os.MkdirAll(lockFile, constant.DefaultDirMode) assert.NoError(t, err) - _, err = CreateLockFile(lockFile) + _, err = LockFile(lockFile) assert.Equal(t, true, err != nil) err = os.RemoveAll(lockFile) assert.NoError(t, err) - _, err = CreateLockFile(lockFile) + _, err = LockFile(lockFile) assert.NoError(t, err) - _, err = CreateLockFile(lockFile) + _, err = LockFile(lockFile) assert.Equal(t, true, err != nil) err = os.RemoveAll(lockFile) assert.NoError(t, err) diff --git a/pkg/common/util/math.go b/pkg/common/util/math.go new file mode 100644 index 0000000..854a3ec --- /dev/null +++ b/pkg/common/util/math.go @@ -0,0 +1,63 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-08 +// Description: This file is used for math + +package util + +import ( + "fmt" + "math" +) + +// Div calculates the quotient of the divisor and the dividend, and it takes +// parameters (dividend, divisor, maximum out of range, precision, and format) +// format indicates the output format, for example, "%.2f" with two decimal places. +func Div(dividend, divisor float64, args ...interface{}) float64 { + var ( + format = "" + accuracy float64 = 0 + maxValue = math.MaxFloat64 + ) + const ( + maxValueIndex int = iota + accuracyIndex + formatIndex + ) + if len(args) > maxValueIndex { + if value, ok := args[maxValueIndex].(float64); ok { + maxValue = value + } + } + if len(args) > accuracyIndex { + if value, ok := args[accuracyIndex].(float64); ok { + accuracy = value + } + } + if len(args) > formatIndex { + if value, ok := args[formatIndex].(string); ok { + format = value + } + } + if divisor == 0 { + return maxValue + } + if math.Abs(divisor) <= accuracy { + return maxValue + } + ans := dividend / divisor + if len(format) == 0 { + if value, err := ParseFloat64(fmt.Sprintf(format, ans)); err == nil { + ans = value + } + } + return ans +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 6dc746e..839378e 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -15,15 +15,19 @@ package typedef import ( + "fmt" "path/filepath" "strings" "sync" - "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" ) // ContainerEngineType indicates the type of container engine type ContainerEngineType int8 +type CgroupKey struct { + subsys, filename string +} const ( // UNDEFINED means undefined container engine @@ -62,17 +66,22 @@ func (engine *ContainerEngineType) Prefix() string { // ContainerInfo contains the interested information of container type ContainerInfo struct { - Name string `json:"name"` - ID string `json:"id"` - CgroupPath string `json:"cgroupPath"` + Name string `json:"name"` + ID string `json:"id"` + CgroupPath string `json:"cgroupPath"` + RequestResources ResourceMap `json:"requests,omitempty"` + LimitResources ResourceMap `json:"limits,omitempty"` } // NewContainerInfo creates a ContainerInfo instance -func NewContainerInfo(name, id, podCgroupPath string) *ContainerInfo { +func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo { + requests, limits := rawContainer.GetResourceMaps() return &ContainerInfo{ - Name: name, - ID: id, - CgroupPath: filepath.Join(podCgroupPath, id), + Name: rawContainer.status.Name, + ID: id, + CgroupPath: filepath.Join(podCgroupPath, id), + RequestResources: requests, + LimitResources: limits, } } @@ -86,32 +95,41 @@ func fixContainerEngine(containerID string) { currentContainerEngines = UNDEFINED } -// getRealContainerID parses the containerID of k8s -func (cont *RawContainer) getRealContainerID() string { - /* - Note: - An UNDEFINED container engine was used when the function was executed for the first time - it seems unlikely to support different container engines at runtime, - So we don't consider the case of midway container engine changes - `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time - */ - setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.status.ContainerID) }) +// DeepCopy returns deepcopy object. +func (cont *ContainerInfo) DeepCopy() *ContainerInfo { + copyObject := *cont + copyObject.LimitResources = util.DeepCopy(cont.LimitResources).(ResourceMap) + copyObject.RequestResources = util.DeepCopy(cont.RequestResources).(ResourceMap) + return ©Object +} - if !currentContainerEngines.Support(cont) { - log.Errorf("fatal error : unsupported container engine") - return "" +// SetCgroupAttr sets the container cgroup file +func (cont *ContainerInfo) SetCgroupAttr(key *CgroupKey, value string) error { + if err := validateCgroupKey(key); err != nil { + return err } + return util.WriteCgroupFile(key.subsys, cont.CgroupPath, key.filename, value) +} - cid := cont.status.ContainerID[len(currentContainerEngines.Prefix()):] - // the container may be in the creation or deletion phase. - if len(cid) == 0 { - return "" +// GetCgroupAttr gets container cgroup file content +func (cont *ContainerInfo) GetCgroupAttr(key *CgroupKey) (string, error) { + if err := validateCgroupKey(key); err != nil { + return "", err + } + data, err := util.ReadCgroupFile(key.subsys, cont.CgroupPath, key.filename) + if err != nil { + return "", err } - return cid + return string(data), nil } -// DeepCopy returns deepcopy object. -func (info *ContainerInfo) DeepCopy() *ContainerInfo { - copyObject := *info - return ©Object +// validateCgroupKey is used to verify the validity of the cgroup key +func validateCgroupKey(key *CgroupKey) error { + if key == nil { + return fmt.Errorf("key cannot be empty") + } + if len(key.subsys) == 0 || len(key.filename) == 0 { + return fmt.Errorf("invalid key") + } + return nil } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index 9aa7be8..6ad00ad 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -14,6 +14,8 @@ // Package typedef defines core struct and methods for rubik package typedef +import "isula.org/rubik/pkg/common/util" + // PodInfo represents pod type PodInfo struct { IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` @@ -37,25 +39,36 @@ func NewPodInfo(pod *RawPod) *PodInfo { } // DeepCopy returns deepcopy object -func (pi *PodInfo) DeepCopy() *PodInfo { - if pi == nil { +func (pod *PodInfo) DeepCopy() *PodInfo { + if pod == nil { return nil } - // deepcopy reference object - idContainersMap := make(map[string]*ContainerInfo, len(pi.IDContainersMap)) - for key, value := range pi.IDContainersMap { - idContainersMap[key] = value.DeepCopy() + return &PodInfo{ + Name: pod.Name, + UID: pod.UID, + CgroupPath: pod.CgroupPath, + Namespace: pod.Namespace, + Annotations: util.DeepCopy(pod.Annotations).(map[string]string), + IDContainersMap: util.DeepCopy(pod.IDContainersMap).(map[string]*ContainerInfo), + } +} + +// SetCgroupAttr sets the container cgroup file +func (pod *PodInfo) SetCgroupAttr(key *CgroupKey, value string) error { + if err := validateCgroupKey(key); err != nil { + return err } - annotations := make(map[string]string) - for key, value := range pi.Annotations { - annotations[key] = value + return util.WriteCgroupFile(key.subsys, pod.CgroupPath, key.filename, value) +} + +// GetCgroupAttr gets container cgroup file content +func (pod *PodInfo) GetCgroupAttr(key *CgroupKey) (string, error) { + if err := validateCgroupKey(key); err != nil { + return "", err } - return &PodInfo{ - Name: pi.Name, - UID: pi.UID, - CgroupPath: pi.CgroupPath, - Namespace: pi.Namespace, - Annotations: annotations, - IDContainersMap: idContainersMap, + data, err := util.ReadCgroupFile(key.subsys, pod.CgroupPath, key.filename) + if err != nil { + return "", err } + return string(data), nil } diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index 96c45af..a5ac2a7 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -19,8 +19,10 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" ) const ( @@ -42,9 +44,17 @@ type ( such as saving the running state of the container. */ status corev1.ContainerStatus + spec corev1.Container } // RawPod represents kubernetes pod structure - RawPod corev1.Pod + RawPod corev1.Pod + ResourceType uint8 + ResourceMap map[ResourceType]float64 +) + +const ( + ResourceCPU ResourceType = iota + ResourceMem ) // ExtractPodInfo returns podInfo from RawPod @@ -78,18 +88,26 @@ func (pod *RawPod) CgroupPath() string { id = configHash } + qosClassPath := "" switch pod.Status.QOSClass { case corev1.PodQOSGuaranteed: - return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) case corev1.PodQOSBurstable: - return filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), - constant.PodCgroupNamePrefix+id) + qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable)) case corev1.PodQOSBestEffort: - return filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), - constant.PodCgroupNamePrefix+id) + qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort)) default: return "" } + /* + example: + 1. Burstable: pod requests are less than the value of limits and not 0; + kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 + 2. BestEffort: pod requests and limits are both 0; + kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 + 3. Guaranteed: pod requests are equal to the value set by limits; + kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 + */ + return filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id) } // ListRawContainers returns all RawContainers in the RawPod @@ -104,6 +122,13 @@ func (pod *RawPod) ListRawContainers() map[string]*RawContainer { status: containerStatus, } } + for _, container := range pod.Spec.Containers { + cont, ok := nameRawContainersMap[container.Name] + if !ok { + continue + } + cont.spec = container + } return nameRawContainersMap } @@ -119,11 +144,53 @@ func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { // 2. generate ID-Container mapping podCgroupPath := pod.CgroupPath() for _, rawContainer := range nameRawContainersMap { - id := rawContainer.getRealContainerID() + id := rawContainer.GetRealContainerID() if id == "" { continue } - idContainersMap[id] = NewContainerInfo(rawContainer.status.Name, id, podCgroupPath) + idContainersMap[id] = NewContainerInfo(id, podCgroupPath, rawContainer) } return idContainersMap } + +// GetRealContainerID parses the containerID of k8s +func (cont *RawContainer) GetRealContainerID() string { + /* + Note: + An UNDEFINED container engine was used when the function was executed for the first time + it seems unlikely to support different container engines at runtime, + So we don't consider the case of midway container engine changes + `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time + */ + setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.status.ContainerID) }) + + if !currentContainerEngines.Support(cont) { + log.Errorf("fatal error : unsupported container engine") + return "" + } + + cid := cont.status.ContainerID[len(currentContainerEngines.Prefix()):] + // the container may be in the creation or deletion phase. + if len(cid) == 0 { + return "" + } + return cid +} + +// GetResourceMaps returns the number of requests and limits of CPU and memory resources +func (cont *RawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { + const milli float64 = 1000 + var ( + // high precision + converter = func(value *resource.Quantity) float64 { + return float64(value.MilliValue()) / milli + } + iterator = func(resourceItems *corev1.ResourceList) ResourceMap { + results := make(ResourceMap) + results[ResourceCPU] = converter(resourceItems.Cpu()) + results[ResourceMem] = converter(resourceItems.Memory()) + return results + } + ) + return iterator(&cont.spec.Resources.Requests), iterator(&cont.spec.Resources.Limits) +} diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index e5da815..3adbfa0 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -113,6 +113,7 @@ func runAgent(ctx context.Context) error { if err := log.InitConfig(c.Agent.LogDriver, c.Agent.LogDir, c.Agent.LogLevel, c.Agent.LogSize); err != nil { return fmt.Errorf("error initializing log: %v", err) } + util.CgroupRoot = c.Agent.CgroupRoot agent := NewAgent(c) if err := agent.Run(ctx); err != nil { @@ -130,12 +131,16 @@ func Run() int { return constant.ArgumentErrorExitCode } // 1. apply file locks, only one rubik process can run at the same time - lock, err := util.CreateLockFile(constant.LockFile) + lock, err := util.LockFile(constant.LockFile) + defer func() { + lock.Close() + os.Remove(constant.LockFile) + }() if err != nil { fmt.Printf("set rubik lock failed: %v, check if there is another rubik running\n", err) return constant.RepeatRunExitCode } - defer util.RemoveLockFile(lock, constant.LockFile) + defer util.UnlockFile(lock) ctx, cancel := context.WithCancel(context.Background()) // 2. handle external signals -- Gitee From 10b269bf9fe4bd680351a10ed43c8fed44dc08c9 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 9 Feb 2023 21:20:49 +0800 Subject: [PATCH 20/73] Decoupling config and service manager Originally, we designed to generate the corresponding service and assign it to the serviceManager singleton directly when initializing the configuration. This produces: 1. config parsing and service generation are put together 2. The config package depends on the function of the service, and the modification is complicated The current approach: while creating the serviceManager, it is obviously more reasonable to obtain the service configuration from cfg and generate services --- pkg/config/config.go | 45 +++--------------- pkg/config/config_test.go | 17 +++---- pkg/rubik/rubik.go | 41 +++++++++------- pkg/services/servicemanager.go | 86 ++++++++++++++++++++++------------ 4 files changed, 97 insertions(+), 92 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index bfb03c7..8b3bb80 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,14 +15,12 @@ package config import ( - "context" "fmt" "io/ioutil" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/common/util" ) const agentKey = "agent" @@ -96,9 +94,6 @@ func (c *Config) LoadConfig(path string) error { } c.Fields = fields c.parseAgentConfig() - if err := c.PrepareServices(); err != nil { - return fmt.Errorf("error preparing services: %s", err) - } return nil } @@ -109,39 +104,13 @@ func (c *Config) filterNonServiceKeys(name string) bool { return ok } -// PrepareServices parses the to-be-run services config and loads them to the ServiceManager -func (c *Config) PrepareServices() error { - // TODO: Later, consider placing the function book in rubik.go - for name, config := range c.Fields { - if c.filterNonServiceKeys(name) { +func (c *Config) UnwarpServiceConfig() map[string]interface{} { + serviceConfig := util.DeepCopy(c.Fields).(map[string]interface{}) + for name := range c.Fields { + if !c.filterNonServiceKeys(name) { continue } - creator := services.GetServiceCreator(name) - if creator == nil { - return fmt.Errorf("no corresponding module %v, please check the module name", name) - } - service := creator() - if err := c.UnmarshalSubConfig(config, service); err != nil { - return fmt.Errorf("error unmarshaling %s config: %v", name, err) - } - - if services.SetLoggerOnService(service, - log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { - log.Debugf("set logger for service: %s", name) - } - - // try to verify configuration - if validator, ok := service.(Validator); ok { - if err := validator.Validate(); err != nil { - return fmt.Errorf("error configuring service %s: %v", name, err) - } - } - services.AddRunningService(name, service) + delete(serviceConfig, name) } - return nil -} - -// Validator is a function interface to verify whether the configuration is correct or not -type Validator interface { - Validate() error + return serviceConfig } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 02631d2..54f5953 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -44,9 +44,9 @@ var rubikConfig string = ` "adjustInterval": 1000, "perfDuration": 1000, "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 + "low": 0, + "mid": 10, + "high": 100 }, "memBandPercent": { "low": 10, @@ -57,8 +57,7 @@ var rubikConfig string = ` } ` -func TestPrepareServices(t *testing.T) { - +func TestServices(t *testing.T) { if !util.PathExist(constant.TmpTestDir) { if err := os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode); err != nil { assert.NoError(t, err) @@ -79,15 +78,17 @@ func TestPrepareServices(t *testing.T) { assert.NoError(t, err) return } - if err := c.PrepareServices(); err != nil { + serviceManager := services.NewServiceManager() + if err := serviceManager.InitServices(c.UnwarpServiceConfig(), c.ConfigParser); err != nil { assert.NoError(t, err) return } + fmt.Printf("agent: %v\n", c.Agent) - for name, service := range services.GetServiceManager().RunningServices { + for name, service := range serviceManager.RunningServices { fmt.Printf("name: %s, service: %v\n", name, service) } - for name, service := range services.GetServiceManager().RunningPersistentServices { + for name, service := range serviceManager.RunningPersistentServices { fmt.Printf("name: %s, persistent service: %v\n", name, service) } } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 3adbfa0..07295d3 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -36,19 +36,25 @@ import ( // Agent runs a series of rubik services and manages data type Agent struct { - Config *config.Config - PodManager *podmanager.PodManager - informer api.Informer + config *config.Config + podManager *podmanager.PodManager + informer api.Informer + servicesManager *services.ServiceManager } // NewAgent returns an agent for given configuration -func NewAgent(cfg *config.Config) *Agent { +func NewAgent(cfg *config.Config) (*Agent, error) { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) + serviceManager := services.NewServiceManager() + if err := serviceManager.InitServices(cfg.UnwarpServiceConfig(), cfg); err != nil { + return nil, err + } a := &Agent{ - Config: cfg, - PodManager: podmanager.NewPodManager(publisher), + config: cfg, + podManager: podmanager.NewPodManager(publisher), + servicesManager: serviceManager, } - return a + return a, nil } // Run starts and runs the agent until receiving stop signal @@ -72,7 +78,7 @@ func (a *Agent) startInformer(ctx context.Context) error { if err != nil { return fmt.Errorf("fail to set informer: %v", err) } - if err := informer.Subscribe(a.PodManager); err != nil { + if err := informer.Subscribe(a.podManager); err != nil { return fmt.Errorf("fail to subscribe informer: %v", err) } a.informer = informer @@ -82,25 +88,23 @@ func (a *Agent) startInformer(ctx context.Context) error { // stopInformer stops the infomer func (a *Agent) stopInformer() { - a.informer.Unsubscribe(a.PodManager) + a.informer.Unsubscribe(a.podManager) } // startServiceHandler starts and runs the service func (a *Agent) startServiceHandler(ctx context.Context) error { - serviceManager := services.GetServiceManager() - if err := serviceManager.Setup(a.PodManager); err != nil { + if err := a.servicesManager.Setup(a.podManager); err != nil { return fmt.Errorf("error setting service handler: %v", err) } - serviceManager.Start(ctx) - a.PodManager.Subscribe(serviceManager) + a.servicesManager.Start(ctx) + a.podManager.Subscribe(a.servicesManager) return nil } // stopServiceHandler stops sending data to the ServiceManager func (a *Agent) stopServiceHandler() { - serviceManager := services.GetServiceManager() - a.PodManager.Unsubscribe(serviceManager) - serviceManager.Stop() + a.podManager.Unsubscribe(a.servicesManager) + a.servicesManager.Stop() } // runAgent creates and runs rubik's agent @@ -115,7 +119,10 @@ func runAgent(ctx context.Context) error { } util.CgroupRoot = c.Agent.CgroupRoot - agent := NewAgent(c) + agent, err := NewAgent(c) + if err != nil { + return fmt.Errorf("error new agent: %v", err) + } if err := agent.Run(ctx); err != nil { return fmt.Errorf("error running agent: %v", err) } diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 4903e76..8daa67d 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/subscriber" "isula.org/rubik/pkg/core/typedef" @@ -32,24 +33,6 @@ import ( // serviceManagerName is the unique ID of the service manager const serviceManagerName = "serviceManager" -// AddRunningService adds a to-be-run service -func AddRunningService(name string, service interface{}) { - serviceManager.RLock() - _, ok1 := serviceManager.RunningServices[name] - _, ok2 := serviceManager.RunningPersistentServices[name] - serviceManager.RUnlock() - if ok1 || ok2 { - log.Errorf("service name conflict : \"%s\"", name) - return - } - - if !serviceManager.tryAddService(name, service) && !serviceManager.tryAddPersistentService(name, service) { - log.Errorf("invalid service : \"%s\", %T", name, service) - return - } - log.Debugf("pre-start service %s", name) -} - // ServiceManager is used to manage the lifecycle of services type ServiceManager struct { api.Subscriber @@ -60,10 +43,8 @@ type ServiceManager struct { TerminateFuncs map[string]Terminator } -// serviceManager is the only global service manager -var serviceManager = newServiceManager() - -func newServiceManager() *ServiceManager { +// NewServiceManager creates a servicemanager object +func NewServiceManager() *ServiceManager { manager := &ServiceManager{ RunningServices: make(map[string]api.Service), RunningPersistentServices: make(map[string]api.PersistentService), @@ -72,9 +53,51 @@ func newServiceManager() *ServiceManager { return manager } -// GetServiceManager returns the globally unique ServiceManager -func GetServiceManager() *ServiceManager { - return serviceManager +// InitServices parses the to-be-run services config and loads them to the ServiceManager +func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{}, parser api.ConfigParser) error { + for name, config := range serviceConfig { + creator := GetServiceCreator(name) + if creator == nil { + return fmt.Errorf("no corresponding module %v, please check the module name", name) + } + service := creator() + if err := parser.UnmarshalSubConfig(config, service); err != nil { + return fmt.Errorf("error unmarshaling %s config: %v", name, err) + } + + if SetLoggerOnService(service, + log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { + log.Debugf("set logger for service: %s", name) + } + + // try to verify configuration + if validator, ok := service.(Validator); ok { + if err := validator.Validate(); err != nil { + return fmt.Errorf("error configuring service %s: %v", name, err) + } + } + if err := manager.AddRunningService(name, service); err != nil { + return err + } + } + return nil +} + +// AddRunningService adds a to-be-run service +func (manager *ServiceManager) AddRunningService(name string, service interface{}) error { + manager.RLock() + _, existed1 := manager.RunningServices[name] + _, existed2 := manager.RunningPersistentServices[name] + manager.RUnlock() + if existed1 || existed2 { + return fmt.Errorf("service name conflict : \"%s\"", name) + } + + if !manager.tryAddService(name, service) && !manager.tryAddPersistentService(name, service) { + return fmt.Errorf("invalid service : \"%s\", %T", name, service) + } + log.Debugf("pre-start service %s", name) + return nil } // HandleEvent is used to handle PodInfo events pushed by the publisher @@ -105,9 +128,9 @@ func (manager *ServiceManager) EventTypes() []typedef.EventType { func (manager *ServiceManager) tryAddService(name string, service interface{}) bool { s, ok := service.(api.Service) if ok { - serviceManager.Lock() + manager.Lock() manager.RunningServices[name] = s - serviceManager.Unlock() + manager.Unlock() log.Debugf("service %s will run", name) } return ok @@ -117,9 +140,9 @@ func (manager *ServiceManager) tryAddService(name string, service interface{}) b func (manager *ServiceManager) tryAddPersistentService(name string, service interface{}) bool { s, ok := service.(api.PersistentService) if ok { - serviceManager.Lock() + manager.Lock() manager.RunningPersistentServices[name] = s - serviceManager.Unlock() + manager.Unlock() log.Debugf("persistent service %s will run", name) } return ok @@ -304,3 +327,8 @@ type Terminator interface { type PreStarter interface { PreStart(api.Viewer) error } + +// Validator is a function interface to verify whether the configuration is correct or not +type Validator interface { + Validate() error +} -- Gitee From 6d205d5e41273accbad86746541b11d583b786e3 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 9 Feb 2023 20:52:45 +0800 Subject: [PATCH 21/73] feat: add qos service for "xxx.qos_level" setting 1. add new service "qos" which manage cpu/memory qos_level 2. add log when service manager got error 3. add dockerfile to build container images 4. modify rubik daemonset file to adapt new service Signed-off-by: DCCooper <1866858@gmail.com> --- Dockerfile | 4 + hack/rubik-daemonset.yaml | 26 ++--- pkg/common/constant/constant.go | 15 +++ pkg/config/config_test.go | 4 + pkg/core/typedef/containerinfo.go | 10 +- pkg/core/typedef/podinfo.go | 18 ++- pkg/rubik/import.go | 1 + pkg/services/qos/qos.go | 177 ++++++++++++++++++++++++++++++ pkg/services/servicemanager.go | 15 ++- 9 files changed, 238 insertions(+), 32 deletions(-) create mode 100644 Dockerfile create mode 100644 pkg/services/qos/qos.go diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bbaf598 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM scratch +COPY ./build/rubik /rubik +ENTRYPOINT ["/rubik"] + diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml index df166d0..e6f6386 100644 --- a/hack/rubik-daemonset.yaml +++ b/hack/rubik-daemonset.yaml @@ -34,28 +34,16 @@ metadata: data: config.json: | { - "autoCheck": false, + "agent": { "logDriver": "stdio", "logDir": "/var/log/rubik", "logSize": 1024, - "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup", - "cacheConfig": { - "enable": false, - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } - } + "logLevel": "debug", + "cgroupRoot": "/sys/fs/cgroup" + }, + "qos": { + "subSys": ["cpu", "memory"] + } } --- apiVersion: apps/v1 diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index c817f0f..584ce9d 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -89,3 +89,18 @@ const ( // ErrorExitCode failed during run ErrorExitCode ) + +// qos level +const ( + Offline = -1 + Online = 0 + // TODO: add more level +) + +// cgroup file name +const ( + // CPUCgroupFileName is name of cgroup file used for cpu qos level setting + CPUCgroupFileName = "cpu.qos_level" + // MemoryCgroupFileName is name of cgroup file used for memory qos level setting + MemoryCgroupFileName = "memory.qos_level" +) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 02631d2..1a84d8c 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -28,6 +28,7 @@ import ( "isula.org/rubik/pkg/services" _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/services/cachelimit" + _ "isula.org/rubik/pkg/services/qos" ) var rubikConfig string = ` @@ -39,6 +40,9 @@ var rubikConfig string = ` "logLevel": "info" }, "blkio":{}, + "qos": { + "subSys": ["cpu", "memory"] + }, "cacheLimit": { "defaultLimitMode": "static", "adjustInterval": 1000, diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 839378e..762b652 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -26,7 +26,7 @@ import ( // ContainerEngineType indicates the type of container engine type ContainerEngineType int8 type CgroupKey struct { - subsys, filename string + SubSys, FileName string } const ( @@ -108,7 +108,7 @@ func (cont *ContainerInfo) SetCgroupAttr(key *CgroupKey, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return util.WriteCgroupFile(key.subsys, cont.CgroupPath, key.filename, value) + return util.WriteCgroupFile(key.SubSys, cont.CgroupPath, key.FileName, value) } // GetCgroupAttr gets container cgroup file content @@ -116,11 +116,11 @@ func (cont *ContainerInfo) GetCgroupAttr(key *CgroupKey) (string, error) { if err := validateCgroupKey(key); err != nil { return "", err } - data, err := util.ReadCgroupFile(key.subsys, cont.CgroupPath, key.filename) + data, err := util.ReadCgroupFile(key.SubSys, cont.CgroupPath, key.FileName) if err != nil { return "", err } - return string(data), nil + return strings.TrimSpace(string(data)), nil } // validateCgroupKey is used to verify the validity of the cgroup key @@ -128,7 +128,7 @@ func validateCgroupKey(key *CgroupKey) error { if key == nil { return fmt.Errorf("key cannot be empty") } - if len(key.subsys) == 0 || len(key.filename) == 0 { + if len(key.SubSys) == 0 || len(key.FileName) == 0 { return fmt.Errorf("invalid key") } return nil diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index 6ad00ad..aa0473b 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -14,7 +14,11 @@ // Package typedef defines core struct and methods for rubik package typedef -import "isula.org/rubik/pkg/common/util" +import ( + "strings" + + "isula.org/rubik/pkg/common/util" +) // PodInfo represents pod type PodInfo struct { @@ -58,7 +62,7 @@ func (pod *PodInfo) SetCgroupAttr(key *CgroupKey, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return util.WriteCgroupFile(key.subsys, pod.CgroupPath, key.filename, value) + return util.WriteCgroupFile(key.SubSys, pod.CgroupPath, key.FileName, value) } // GetCgroupAttr gets container cgroup file content @@ -66,9 +70,15 @@ func (pod *PodInfo) GetCgroupAttr(key *CgroupKey) (string, error) { if err := validateCgroupKey(key); err != nil { return "", err } - data, err := util.ReadCgroupFile(key.subsys, pod.CgroupPath, key.filename) + data, err := util.ReadCgroupFile(key.SubSys, pod.CgroupPath, key.FileName) if err != nil { return "", err } - return string(data), nil + return strings.TrimSpace(string(data)), nil +} + +// CgroupSetterAndGetter is used for set and get value to/from cgroup file +type CgroupSetterAndGetter interface { + SetCgroupAttr(*CgroupKey, string) error + GetCgroupAttr(*CgroupKey) (string, error) } diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index aad0c19..82e65c6 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -18,5 +18,6 @@ import ( // introduce packages to auto register service _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/services/cachelimit" + _ "isula.org/rubik/pkg/services/qos" _ "isula.org/rubik/pkg/version" ) diff --git a/pkg/services/qos/qos.go b/pkg/services/qos/qos.go new file mode 100644 index 0000000..2d5ed43 --- /dev/null +++ b/pkg/services/qos/qos.go @@ -0,0 +1,177 @@ +package qos + +import ( + "fmt" + "strconv" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/services" +) + +var supportCgroupTypes = map[string]*typedef.CgroupKey{ + "cpu": {SubSys: "cpu", FileName: constant.CPUCgroupFileName}, + "memory": {SubSys: "memory", FileName: constant.MemoryCgroupFileName}, +} + +// QoS define service which related to qos level setting +type QoS struct { + Name string `json:"-"` + Log api.Logger + Config +} + +// Config contains sub-system that need to set qos level +type Config struct { + SubSys []string `json:"subSys"` +} + +func init() { + services.Register("qos", func() interface{} { + return NewQoS() + }) +} + +// NewQoS return qos instance +func NewQoS() *QoS { + return &QoS{ + Name: "qos", + } +} + +// PreStart will do some pre-start actions +func (q *QoS) PreStart(viewer api.Viewer) error { + return nil +} + +// Terminate will do some clean-up actions +func (q *QoS) Terminate(viewer api.Viewer) error { + return nil +} + +// ID return qos service name +func (q *QoS) ID() string { + return q.Name +} + +// AddFunc implement add function when pod is added in k8s +func (q *QoS) AddFunc(pod *typedef.PodInfo) error { + if err := q.SetQoS(pod); err != nil { + return err + } + if err := q.ValidateQoS(pod); err != nil { + return err + } + return nil +} + +// UpdateFunc implement update function when pod info is changed +func (q *QoS) UpdateFunc(old, new *typedef.PodInfo) error { + oldQos, newQos := getQoSLevel(old), getQoSLevel(new) + switch { + case newQos == oldQos: + return nil + case newQos > oldQos: + return fmt.Errorf("not support change qos level from low to high") + default: + if err := q.ValidateQoS(new); err != nil { + if err := q.SetQoS(new); err != nil { + return fmt.Errorf("update qos for pod %s(%s) failed: %v", new.Name, new.UID, err) + } + } + } + return nil +} + +// DeleteFunc implement delete function when pod is deleted by k8s +func (q *QoS) DeleteFunc(pod *typedef.PodInfo) error { + return nil +} + +// ValidateQoS will validate pod's qos level between value from +// cgroup file and the one from pod info +func (q *QoS) ValidateQoS(pod *typedef.PodInfo) error { + targetLevel := getQoSLevel(pod) + var compare = func(c typedef.CgroupSetterAndGetter, subSys string) bool { + value, err := c.GetCgroupAttr(supportCgroupTypes[subSys]) + if err != nil { + q.Log.Errorf("failed to get cgroup attr: %v", err) + return false + } + level, err := strconv.Atoi(value) + if err != nil { + q.Log.Errorf("failed to convert value %s to int: %v", value, err) + return false + } + if level != targetLevel { + q.Log.Errorf("different qos value between annotation %d and cgroup file %d", targetLevel, level) + return false + } + return true + } + for _, subSys := range q.SubSys { + if !compare(pod, subSys) { + return fmt.Errorf("failed to validate pod %s", pod.Name) + } + for _, container := range pod.IDContainersMap { + if !compare(container, subSys) { + return fmt.Errorf("failed to validate container %s", container.Name) + } + } + } + return nil +} + +// SetQoS set pod and all containers' qos level within it +func (q *QoS) SetQoS(pod *typedef.PodInfo) error { + qosLevel := getQoSLevel(pod) + if qosLevel != constant.Offline { + q.Log.Debugf("pod %s already online", pod.Name) + return nil + } + + for _, sys := range q.SubSys { + if err := pod.SetCgroupAttr(supportCgroupTypes[sys], strconv.Itoa(qosLevel)); err != nil { + return err + } + for _, container := range pod.IDContainersMap { + if err := container.SetCgroupAttr(supportCgroupTypes[sys], strconv.Itoa(qosLevel)); err != nil { + return err + } + } + } + q.Log.Debugf("set pod %s(%s) qos level %d ok", pod.Name, pod.UID, qosLevel) + return nil +} + +func getQoSLevel(pod *typedef.PodInfo) int { + if pod == nil { + return 0 + } + anno, ok := pod.Annotations[constant.PriorityAnnotationKey] + if !ok { + return 0 + } + switch anno { + case "true": + return -1 + case "false": + return 0 + default: + return 0 + } +} + +// Validate will validate the qos service config +func (q *QoS) Validate() error { + if len(q.SubSys) == 0 { + return fmt.Errorf("empty qos config") + } + for _, subSys := range q.SubSys { + if _, ok := supportCgroupTypes[subSys]; !ok { + return fmt.Errorf("not support sub system %s", subSys) + } + } + return nil +} diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 4903e76..8054951 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -186,7 +186,7 @@ func (manager *ServiceManager) Setup(v api.Viewer) error { // 2. pre-start persistent services for _, s := range manager.RunningPersistentServices { if err := setupFunc(s.ID(), s); err != nil { - return manager.terminatingRunningServices(fmt.Errorf("error running services %s: %v", s.ID(), err)) + return manager.terminatingRunningServices(fmt.Errorf("error running persistent services %s: %v", s.ID(), err)) } } return nil @@ -235,7 +235,9 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) - s.AddFunc(podInfo) + if err := s.AddFunc(podInfo); err != nil { + log.Errorf("service %s add func failed: %v", s.ID(), err) + } wg.Done() } manager.RLock() @@ -261,7 +263,10 @@ func (manager *ServiceManager) updateFunc(event typedef.Event) { } runOnce := func(s api.Service, old, new *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) - s.UpdateFunc(old, new) + log.Debugf("update Func with service: %s", s.ID()) + if err := s.UpdateFunc(old, new); err != nil { + log.Errorf("service %s update func failed: %v", s.ID(), err) + } wg.Done() } manager.RLock() @@ -283,7 +288,9 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) - s.DeleteFunc(podInfo) + if err := s.DeleteFunc(podInfo); err != nil { + log.Errorf("service %s delete func failed: %v", s.ID(), err) + } wg.Done() } manager.RLock() -- Gitee From c950b848a536a02f2c1d7617f3faa7b3cb083eb8 Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 11 Feb 2023 21:03:39 +0800 Subject: [PATCH 22/73] Optimize the dependencies of the three packages of config, api, and service 1. Put the configparser definition into config to avoid exposure through api 2. Sercvice relies on the parser analysis configuration in the config package to complete part of the analysis work 3. Modify the topic enumeration variable name to keep the naming style consistent --- pkg/api/api.go | 6 ----- pkg/config/config.go | 13 +++++----- pkg/config/config_test.go | 18 +------------ pkg/config/parserfactory.go | 9 ++++--- pkg/core/typedef/event.go | 42 +++++++++++++++---------------- pkg/informer/apiserverinformer.go | 8 +++--- pkg/podmanager/podmanager.go | 26 +++++++++---------- pkg/services/servicemanager.go | 15 +++++------ 8 files changed, 59 insertions(+), 78 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 9ebd28a..4768e64 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -54,12 +54,6 @@ type PersistentService interface { Run(ctx context.Context) } -// ConfigParser is a configuration parser for different languages -type ConfigParser interface { - ParseConfig(data []byte) (map[string]interface{}, error) - UnmarshalSubConfig(data interface{}, v interface{}) error -} - // Viewer collect on/offline pods info type Viewer interface { ListOnlinePods() ([]*typedef.PodInfo, error) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8b3bb80..e8b1048 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,9 +18,7 @@ import ( "fmt" "io/ioutil" - "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/util" ) const agentKey = "agent" @@ -32,7 +30,7 @@ var sysConfKeys = map[string]struct{}{ // Config saves all configuration information of rubik type Config struct { - api.ConfigParser + ConfigParser Agent *AgentConfig Fields map[string]interface{} } @@ -104,13 +102,14 @@ func (c *Config) filterNonServiceKeys(name string) bool { return ok } +// UnwarpServiceConfig returns service configuration, indexed by service name func (c *Config) UnwarpServiceConfig() map[string]interface{} { - serviceConfig := util.DeepCopy(c.Fields).(map[string]interface{}) - for name := range c.Fields { - if !c.filterNonServiceKeys(name) { + serviceConfig := make(map[string]interface{}) + for name, conf := range c.Fields { + if c.filterNonServiceKeys(name) { continue } - delete(serviceConfig, name) + serviceConfig[name] = conf } return serviceConfig } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 21bacf4..1a16be0 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -25,10 +25,6 @@ import ( "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/services" - _ "isula.org/rubik/pkg/services/blkio" - _ "isula.org/rubik/pkg/services/cachelimit" - _ "isula.org/rubik/pkg/services/qos" ) var rubikConfig string = ` @@ -82,17 +78,5 @@ func TestServices(t *testing.T) { assert.NoError(t, err) return } - serviceManager := services.NewServiceManager() - if err := serviceManager.InitServices(c.UnwarpServiceConfig(), c.ConfigParser); err != nil { - assert.NoError(t, err) - return - } - - fmt.Printf("agent: %v\n", c.Agent) - for name, service := range serviceManager.RunningServices { - fmt.Printf("name: %s, service: %v\n", name, service) - } - for name, service := range serviceManager.RunningPersistentServices { - fmt.Printf("name: %s, persistent service: %v\n", name, service) - } + fmt.Printf("config: %v", c) } diff --git a/pkg/config/parserfactory.go b/pkg/config/parserfactory.go index 921eeba..8a533da 100644 --- a/pkg/config/parserfactory.go +++ b/pkg/config/parserfactory.go @@ -14,13 +14,16 @@ // Package config is used to manage the configuration of rubik package config -import "isula.org/rubik/pkg/api" - type ( // parserType represents the parser type parserType int8 // parserFactory is the factory class of the parser parserFactory struct{} + // ConfigParser is a configuration parser for different languages + ConfigParser interface { + ParseConfig(data []byte) (map[string]interface{}, error) + UnmarshalSubConfig(data interface{}, v interface{}) error + } ) const ( @@ -32,7 +35,7 @@ const ( var defaultParserFactory = &parserFactory{} // getParser gets parser instance according to the parser type passed in -func (factory *parserFactory) getParser(pType parserType) api.ConfigParser { +func (factory *parserFactory) getParser(pType parserType) ConfigParser { switch pType { case JSON: return getJsonParser() diff --git a/pkg/core/typedef/event.go b/pkg/core/typedef/event.go index 65b7d84..ee9d886 100644 --- a/pkg/core/typedef/event.go +++ b/pkg/core/typedef/event.go @@ -22,32 +22,32 @@ type ( ) const ( - // RAW_POD_ADD means Kubernetes starts a new Pod event - RAW_POD_ADD EventType = iota - // RAW_POD_UPDATE means Kubernetes updates Pod event - RAW_POD_UPDATE - // RAW_POD_DELETE means Kubernetes deletes Pod event - RAW_POD_DELETE - // INFO_ADD means PodManager adds pod information event - INFO_ADD - // INFO_UPDATE means PodManager updates pod information event - INFO_UPDATE - // INFO_DELETE means PodManager deletes pod information event - INFO_DELETE - // RAW_POD_SYNC_ALL means Full amount of kubernetes pods - RAW_POD_SYNC_ALL + // RAWPODADD means Kubernetes starts a new Pod event + RAWPODADD EventType = iota + // RAWPODUPDATE means Kubernetes updates Pod event + RAWPODUPDATE + // RAWPODDELETE means Kubernetes deletes Pod event + RAWPODDELETE + // INFOADD means PodManager adds pod information event + INFOADD + // INFOUPDATE means PodManager updates pod information event + INFOUPDATE + // INFODELETE means PodManager deletes pod information event + INFODELETE + // RAWPODSYNCALL means Full amount of kubernetes pods + RAWPODSYNCALL ) const undefinedType = "undefined" var eventTypeToString = map[EventType]string{ - RAW_POD_ADD: "addrawpod", - RAW_POD_UPDATE: "updaterawpod", - RAW_POD_DELETE: "deleterawpod", - INFO_ADD: "addinfo", - INFO_UPDATE: "updateinfo", - INFO_DELETE: "deleteinfo", - RAW_POD_SYNC_ALL: "syncallrawpods", + RAWPODADD: "addrawpod", + RAWPODUPDATE: "updaterawpod", + RAWPODDELETE: "deleterawpod", + INFOADD: "addinfo", + INFOUPDATE: "updateinfo", + INFODELETE: "deleteinfo", + RAWPODSYNCALL: "syncallrawpods", } // String returns the string of the current event type diff --git a/pkg/informer/apiserverinformer.go b/pkg/informer/apiserverinformer.go index 4d1c9dd..0f8daaa 100644 --- a/pkg/informer/apiserverinformer.go +++ b/pkg/informer/apiserverinformer.go @@ -94,7 +94,7 @@ func (informer *APIServerInformer) listFunc(fieldSelector string) { log.Errorf("error listing all pods: %v", err) return } - informer.Publish(typedef.RAW_POD_SYNC_ALL, pods.Items) + informer.Publish(typedef.RAWPODSYNCALL, pods.Items) } func (informer *APIServerInformer) watchFunc(ctx context.Context, fieldSelector string) { @@ -114,15 +114,15 @@ func (informer *APIServerInformer) watchFunc(ctx context.Context, fieldSelector // AddFunc handles the raw pod increase event func (informer *APIServerInformer) AddFunc(obj interface{}) { - informer.Publish(typedef.RAW_POD_ADD, obj) + informer.Publish(typedef.RAWPODADD, obj) } // UpdateFunc handles the raw pod update event func (informer *APIServerInformer) UpdateFunc(oldObj, newObj interface{}) { - informer.Publish(typedef.RAW_POD_UPDATE, newObj) + informer.Publish(typedef.RAWPODUPDATE, newObj) } // DeleteFunc handles the raw pod deletion event func (informer *APIServerInformer) DeleteFunc(obj interface{}) { - informer.Publish(typedef.RAW_POD_DELETE, obj) + informer.Publish(typedef.RAWPODDELETE, obj) } diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index b79be68..3fae9e1 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -48,9 +48,9 @@ func NewPodManager(publisher api.Publisher) *PodManager { // HandleEvent handles the event from publisher func (manager *PodManager) HandleEvent(eventType typedef.EventType, event typedef.Event) { switch eventType { - case typedef.RAW_POD_ADD, typedef.RAW_POD_UPDATE, typedef.RAW_POD_DELETE: + case typedef.RAWPODADD, typedef.RAWPODUPDATE, typedef.RAWPODDELETE: manager.handleWatchEvent(eventType, event) - case typedef.RAW_POD_SYNC_ALL: + case typedef.RAWPODSYNCALL: manager.handleListEvent(eventType, event) default: log.Infof("fail to process %s type event", eventType.String()) @@ -66,11 +66,11 @@ func (manager *PodManager) handleWatchEvent(eventType typedef.EventType, event t } switch eventType { - case typedef.RAW_POD_ADD: + case typedef.RAWPODADD: manager.addFunc(pod) - case typedef.RAW_POD_UPDATE: + case typedef.RAWPODUPDATE: manager.updateFunc(pod) - case typedef.RAW_POD_DELETE: + case typedef.RAWPODDELETE: manager.deleteFunc(pod) default: log.Errorf("code problem, should not go here...") @@ -85,7 +85,7 @@ func (manager *PodManager) handleListEvent(eventType typedef.EventType, event ty return } switch eventType { - case typedef.RAW_POD_SYNC_ALL: + case typedef.RAWPODSYNCALL: manager.sync(pods) default: log.Errorf("code problem, should not go here...") @@ -94,10 +94,10 @@ func (manager *PodManager) handleListEvent(eventType typedef.EventType, event ty // EventTypes returns the intersted event types func (manager *PodManager) EventTypes() []typedef.EventType { - return []typedef.EventType{typedef.RAW_POD_ADD, - typedef.RAW_POD_UPDATE, - typedef.RAW_POD_DELETE, - typedef.RAW_POD_SYNC_ALL, + return []typedef.EventType{typedef.RAWPODADD, + typedef.RAWPODUPDATE, + typedef.RAWPODDELETE, + typedef.RAWPODSYNCALL, } } @@ -181,7 +181,7 @@ func (manager *PodManager) tryAdd(podInfo *typedef.PodInfo) { // only add when pod is not existed if !manager.pods.podExist(podInfo.UID) { manager.pods.addPod(podInfo) - manager.Publish(typedef.INFO_ADD, podInfo.DeepCopy()) + manager.Publish(typedef.INFOADD, podInfo.DeepCopy()) } } @@ -191,7 +191,7 @@ func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { if manager.pods.podExist(podInfo.UID) { oldPod := manager.pods.getPod(podInfo.UID) manager.pods.updatePod(podInfo) - manager.Publish(typedef.INFO_UPDATE, []*typedef.PodInfo{oldPod, podInfo.DeepCopy()}) + manager.Publish(typedef.INFOUPDATE, []*typedef.PodInfo{oldPod, podInfo.DeepCopy()}) } } @@ -201,7 +201,7 @@ func (manager *PodManager) tryDelete(id string) { oldPod := manager.pods.getPod(id) if oldPod != nil { manager.pods.delPod(id) - manager.Publish(typedef.INFO_DELETE, oldPod) + manager.Publish(typedef.INFODELETE, oldPod) } } diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index dc1e7cf..834aa89 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -26,6 +26,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/subscriber" "isula.org/rubik/pkg/core/typedef" ) @@ -54,7 +55,7 @@ func NewServiceManager() *ServiceManager { } // InitServices parses the to-be-run services config and loads them to the ServiceManager -func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{}, parser api.ConfigParser) error { +func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{}, parser config.ConfigParser) error { for name, config := range serviceConfig { creator := GetServiceCreator(name) if creator == nil { @@ -90,11 +91,11 @@ func (manager *ServiceManager) AddRunningService(name string, service interface{ _, existed2 := manager.RunningPersistentServices[name] manager.RUnlock() if existed1 || existed2 { - return fmt.Errorf("service name conflict : \"%s\"", name) + return fmt.Errorf("service name conflict: %s", name) } if !manager.tryAddService(name, service) && !manager.tryAddPersistentService(name, service) { - return fmt.Errorf("invalid service : \"%s\", %T", name, service) + return fmt.Errorf("invalid service %s (type %T)", name, service) } log.Debugf("pre-start service %s", name) return nil @@ -108,11 +109,11 @@ func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event ty } }() switch eventType { - case typedef.INFO_ADD: + case typedef.INFOADD: manager.addFunc(event) - case typedef.INFO_UPDATE: + case typedef.INFOUPDATE: manager.updateFunc(event) - case typedef.INFO_DELETE: + case typedef.INFODELETE: manager.deleteFunc(event) default: log.Infof("service manager fail to process %s type", eventType.String()) @@ -121,7 +122,7 @@ func (manager *ServiceManager) HandleEvent(eventType typedef.EventType, event ty // EventTypes returns the type of event the serviceManager is interested in func (manager *ServiceManager) EventTypes() []typedef.EventType { - return []typedef.EventType{typedef.INFO_ADD, typedef.INFO_UPDATE, typedef.INFO_DELETE} + return []typedef.EventType{typedef.INFOADD, typedef.INFOUPDATE, typedef.INFODELETE} } // tryAddService determines whether it is a api.Service and adds it to the queue to be run -- Gitee From 9f19d2af99fb132b5fbb49567f792743043dc0c6 Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 11 Feb 2023 21:42:53 +0800 Subject: [PATCH 23/73] Add general cgroup operation method 1. Considering that cgroup depends on the parameters passed in by config, it is not suitable to be placed in the util package, remove it 2. Considering rubik's large number of operations on cgroups, we regard cgroups as key-value pairs and encapsulate operations on cgroups, including operations such as reading, setting, and converting. 3. The cgroup package under typedef will be regarded as a component that depends on config 4. Adapt to qos and rubik 5. other: The name of the publisher type enumeration value is changed to camel case --- pkg/common/util/cgroup.go | 53 ---------- pkg/common/util/conversion.go | 24 +++++ pkg/core/publisher/publisherfactory.go | 16 +-- pkg/core/typedef/cgroup/common.go | 134 +++++++++++++++++++++++++ pkg/core/typedef/cgroup/cpu.go | 44 ++++++++ pkg/core/typedef/containerinfo.go | 24 ++--- pkg/core/typedef/podinfo.go | 21 ++-- pkg/core/typedef/rawpod.go | 8 +- pkg/rubik/rubik.go | 15 ++- pkg/services/qos/qos.go | 38 ++----- 10 files changed, 251 insertions(+), 126 deletions(-) delete mode 100644 pkg/common/util/cgroup.go create mode 100644 pkg/core/typedef/cgroup/common.go create mode 100644 pkg/core/typedef/cgroup/cpu.go diff --git a/pkg/common/util/cgroup.go b/pkg/common/util/cgroup.go deleted file mode 100644 index d65aae6..0000000 --- a/pkg/common/util/cgroup.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2023-02-01 -// Description: This file contains commom cgroup operation - -// Package util is common utilitization -package util - -import ( - "fmt" - "path/filepath" - - "isula.org/rubik/pkg/common/constant" -) - -var ( - // CgroupRoot is the unique cgroup mount point globally - CgroupRoot = constant.DefaultCgroupRoot -) - -// AbsoluteCgroupPath returns absolute cgroup path of specified subsystem of a relative path -func AbsoluteCgroupPath(subsys string, relativePath string) string { - if subsys == "" || relativePath == "" { - return "" - } - return filepath.Join(CgroupRoot, subsys, relativePath) -} - -// ReadCgroupFile reads data from cgroup files -func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) { - cgfile := filepath.Join(CgroupRoot, subsys, cgroupParent, cgroupFileName) - if !PathExist(cgfile) { - return nil, fmt.Errorf("%v: no such file or diretory", cgfile) - } - return ReadFile(cgfile) -} - -// WriteCgroupFile writes data to cgroup file -func WriteCgroupFile(subsys, cgroupParent, cgroupFileName string, content string) error { - cgfile := filepath.Join(CgroupRoot, subsys, cgroupParent, cgroupFileName) - if !PathExist(cgfile) { - return fmt.Errorf("%v: no such file or diretory", cgfile) - } - return WriteFile(cgfile, content) -} diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go index f6c2019..fc036e8 100644 --- a/pkg/common/util/conversion.go +++ b/pkg/common/util/conversion.go @@ -15,7 +15,10 @@ package util import ( + "bufio" + "fmt" "strconv" + "strings" ) // FormatInt64 convert the int 64 type to a string @@ -39,6 +42,27 @@ func ParseFloat64(str string) (float64, error) { return strconv.ParseFloat(str, bitSize) } +// ParseInt64Map converts string to map[string]Int64: +// 1. multiple lines are allowed; +// 2. a single line consists of only two strings separated by spaces +func ParseInt64Map(data string) (map[string]int64, error) { + var res = make(map[string]int64) + scanner := bufio.NewScanner(strings.NewReader(data)) + for scanner.Scan() { + arr := strings.Fields(scanner.Text()) + const defaultLength = 2 + if len(arr) != defaultLength { + return nil, fmt.Errorf(" fail to parse a single line into two strings") + } + value, err := ParseInt64(arr[1]) + if err != nil { + return nil, err + } + res[arr[0]] = value + } + return res, nil +} + // FormatFloat64 convert the Float64 type to string func FormatFloat64(f float64) string { const ( diff --git a/pkg/core/publisher/publisherfactory.go b/pkg/core/publisher/publisherfactory.go index f5a8038..e457b66 100644 --- a/pkg/core/publisher/publisherfactory.go +++ b/pkg/core/publisher/publisherfactory.go @@ -19,25 +19,25 @@ import "isula.org/rubik/pkg/api" type publihserType int8 const ( - // TYPE_GENERIC indicates the generic publisher type - TYPE_GENERIC publihserType = iota + // GENERIC indicates the generic publisher type + GENERIC publihserType = iota ) -// PublisherFactory is the factory class of the publisher entity -type PublisherFactory struct { +// Factory is the factory class of the publisher entity +type Factory struct { } -var publisherFactory = &PublisherFactory{} +var publisherFactory = &Factory{} // GetPublisherFactory creates a publisher factory instance -func GetPublisherFactory() *PublisherFactory { +func GetPublisherFactory() *Factory { return publisherFactory } // GetPublisher returns the publisher entity according to the publisher type -func (f *PublisherFactory) GetPublisher(publisherType publihserType) api.Publisher { +func (f *Factory) GetPublisher(publisherType publihserType) api.Publisher { switch publisherType { - case TYPE_GENERIC: + case GENERIC: return getGenericPublisher() default: return nil diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go new file mode 100644 index 0000000..ada7da0 --- /dev/null +++ b/pkg/core/typedef/cgroup/common.go @@ -0,0 +1,134 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-01-05 +// Description: This file defines cgroupAttr and CgroupKey + +// Package cgroup uses map to manage cgroup parameters and provides a friendly and simple cgroup usage method +package cgroup + +import ( + "fmt" + "path/filepath" + "strconv" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" +) + +var rootDir = constant.DefaultCgroupRoot + +// ReadCgroupFile reads data from cgroup files +func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) { + cgfile := filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) + if !util.PathExist(cgfile) { + return nil, fmt.Errorf("%v: no such file or diretory", cgfile) + } + return util.ReadFile(cgfile) +} + +// WriteCgroupFile writes data to cgroup file +func WriteCgroupFile(subsys, cgroupParent, cgroupFileName string, content string) error { + cgfile := filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) + if !util.PathExist(cgfile) { + return fmt.Errorf("%v: no such file or diretory", cgfile) + } + return util.WriteFile(cgfile, content) +} + +// InitMountDir sets the mount directory of the cgroup file system +func InitMountDir(arg string) { + rootDir = arg +} + +type ( + // Key uniquely determines the cgroup value of the container or pod + Key struct { + // SubSys refers to the subsystem of the cgroup + SubSys string + // FileName represents the cgroup file name + FileName string + } + // Attr represents a single cgroup attribute, and Err represents whether the Value is available + Attr struct { + Value string + Err error + } + // SetterAndGetter is used for set and get value to/from cgroup file + SetterAndGetter interface { + SetCgroupAttr(*Key, string) error + GetCgroupAttr(*Key) *Attr + } +) + +// Expect judges whether Attr is consistent with the input +func (attr *Attr) Expect(arg interface{}) error { + if attr.Err != nil { + return attr.Err + } + + switch arg := arg.(type) { + case int: + value, err := attr.Int() + if err != nil { + return fmt.Errorf("fail to convert: %v", err) + } + if value != arg { + return fmt.Errorf("%v is not equal to %v", value, arg) + } + case string: + if attr.Value != arg { + return fmt.Errorf("%v is not equal to %v", attr.Value, arg) + } + case int64: + value, err := attr.Int64() + if err != nil { + return fmt.Errorf("fail to convert: %v", err) + } + if value != arg { + return fmt.Errorf("%v is not equal to %v", value, arg) + } + default: + return fmt.Errorf("invalid expect type: %T", arg) + } + return nil +} + +// Int64 parses CgroupAttr as int64 type +func (attr *Attr) Int64() (int64, error) { + if attr.Err != nil { + return 0, attr.Err + } + return util.ParseInt64(attr.Value) +} + +// Int parses CgroupAttr as int type +func (attr *Attr) Int() (int, error) { + if attr.Err != nil { + return 0, attr.Err + } + return strconv.Atoi(attr.Value) +} + +// Int64Map parses CgroupAttr64 as map[string]int64 type +func (attr *Attr) Int64Map() (map[string]int64, error) { + if attr.Err != nil { + return nil, attr.Err + } + return util.ParseInt64Map(attr.Value) +} + +// CPUStat parses CgroupAttr64 as CPUStat type +func (attr *Attr) CPUStat() (*CPUStat, error) { + if attr.Err != nil { + return nil, attr.Err + } + return NewCPUStat(attr.Value) +} diff --git a/pkg/core/typedef/cgroup/cpu.go b/pkg/core/typedef/cgroup/cpu.go new file mode 100644 index 0000000..0104b8c --- /dev/null +++ b/pkg/core/typedef/cgroup/cpu.go @@ -0,0 +1,44 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-11 +// Description: This file provides the relevant data structures and methods of the cgroup cpu subsystem + +package cgroup + +import "isula.org/rubik/pkg/common/util" + +type ( + // CPUStat save the cpu.stat data + CPUStat struct { + NrPeriods int64 + NrThrottled int64 + ThrottledTime int64 + } +) + +// NewCPUStat creates a new MPStat object and returns its pointer +func NewCPUStat(data string) (*CPUStat, error) { + const ( + throttlePeriodNumFieldName = "nr_periods" + throttleNumFieldName = "nr_throttled" + throttleTimeFieldName = "throttled_time" + ) + stringInt64Map, err := util.ParseInt64Map(data) + if err != nil { + return nil, err + } + + return &CPUStat{ + NrPeriods: stringInt64Map[throttlePeriodNumFieldName], + NrThrottled: stringInt64Map[throttleNumFieldName], + ThrottledTime: stringInt64Map[throttleTimeFieldName], + }, nil +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 762b652..8b52803 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -21,13 +21,11 @@ import ( "sync" "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" ) // ContainerEngineType indicates the type of container engine type ContainerEngineType int8 -type CgroupKey struct { - SubSys, FileName string -} const ( // UNDEFINED means undefined container engine @@ -98,33 +96,33 @@ func fixContainerEngine(containerID string) { // DeepCopy returns deepcopy object. func (cont *ContainerInfo) DeepCopy() *ContainerInfo { copyObject := *cont - copyObject.LimitResources = util.DeepCopy(cont.LimitResources).(ResourceMap) - copyObject.RequestResources = util.DeepCopy(cont.RequestResources).(ResourceMap) + copyObject.LimitResources, _ = util.DeepCopy(cont.LimitResources).(ResourceMap) + copyObject.RequestResources, _ = util.DeepCopy(cont.RequestResources).(ResourceMap) return ©Object } // SetCgroupAttr sets the container cgroup file -func (cont *ContainerInfo) SetCgroupAttr(key *CgroupKey, value string) error { +func (cont *ContainerInfo) SetCgroupAttr(key *cgroup.Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return util.WriteCgroupFile(key.SubSys, cont.CgroupPath, key.FileName, value) + return cgroup.WriteCgroupFile(key.SubSys, cont.CgroupPath, key.FileName, value) } // GetCgroupAttr gets container cgroup file content -func (cont *ContainerInfo) GetCgroupAttr(key *CgroupKey) (string, error) { +func (cont *ContainerInfo) GetCgroupAttr(key *cgroup.Key) *cgroup.Attr { if err := validateCgroupKey(key); err != nil { - return "", err + return &cgroup.Attr{Err: err} } - data, err := util.ReadCgroupFile(key.SubSys, cont.CgroupPath, key.FileName) + data, err := cgroup.ReadCgroupFile(key.SubSys, cont.CgroupPath, key.FileName) if err != nil { - return "", err + return &cgroup.Attr{Err: err} } - return strings.TrimSpace(string(data)), nil + return &cgroup.Attr{Value: strings.TrimSpace(string(data)), Err: nil} } // validateCgroupKey is used to verify the validity of the cgroup key -func validateCgroupKey(key *CgroupKey) error { +func validateCgroupKey(key *cgroup.Key) error { if key == nil { return fmt.Errorf("key cannot be empty") } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index aa0473b..ba73f1e 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -18,6 +18,7 @@ import ( "strings" "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" ) // PodInfo represents pod @@ -58,27 +59,21 @@ func (pod *PodInfo) DeepCopy() *PodInfo { } // SetCgroupAttr sets the container cgroup file -func (pod *PodInfo) SetCgroupAttr(key *CgroupKey, value string) error { +func (pod *PodInfo) SetCgroupAttr(key *cgroup.Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return util.WriteCgroupFile(key.SubSys, pod.CgroupPath, key.FileName, value) + return cgroup.WriteCgroupFile(key.SubSys, pod.CgroupPath, key.FileName, value) } // GetCgroupAttr gets container cgroup file content -func (pod *PodInfo) GetCgroupAttr(key *CgroupKey) (string, error) { +func (pod *PodInfo) GetCgroupAttr(key *cgroup.Key) *cgroup.Attr { if err := validateCgroupKey(key); err != nil { - return "", err + return &cgroup.Attr{Err: err} } - data, err := util.ReadCgroupFile(key.SubSys, pod.CgroupPath, key.FileName) + data, err := cgroup.ReadCgroupFile(key.SubSys, pod.CgroupPath, key.FileName) if err != nil { - return "", err + return &cgroup.Attr{Err: err} } - return strings.TrimSpace(string(data)), nil -} - -// CgroupSetterAndGetter is used for set and get value to/from cgroup file -type CgroupSetterAndGetter interface { - SetCgroupAttr(*CgroupKey, string) error - GetCgroupAttr(*CgroupKey) (string, error) + return &cgroup.Attr{Value: strings.TrimSpace(string(data)), Err: nil} } diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index a5ac2a7..c3ea5ae 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -47,13 +47,17 @@ type ( spec corev1.Container } // RawPod represents kubernetes pod structure - RawPod corev1.Pod + RawPod corev1.Pod + // ResourceType indicates the resource type, such as memory or CPU ResourceType uint8 - ResourceMap map[ResourceType]float64 + // ResourceMap represents the available value of a certain type of resource + ResourceMap map[ResourceType]float64 ) const ( + // ResourceCPU indicates CPU resources ResourceCPU ResourceType = iota + // ResourceMem represents memory resources ResourceMem ) diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 07295d3..739ec8d 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -29,6 +29,7 @@ import ( "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/publisher" + "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/informer" "isula.org/rubik/pkg/podmanager" "isula.org/rubik/pkg/services" @@ -44,7 +45,7 @@ type Agent struct { // NewAgent returns an agent for given configuration func NewAgent(cfg *config.Config) (*Agent, error) { - publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) + publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) serviceManager := services.NewServiceManager() if err := serviceManager.InitServices(cfg.UnwarpServiceConfig(), cfg); err != nil { return nil, err @@ -73,7 +74,7 @@ func (a *Agent) Run(ctx context.Context) error { // startInformer starts informer to obtain external data func (a *Agent) startInformer(ctx context.Context) error { - publisher := publisher.GetPublisherFactory().GetPublisher(publisher.TYPE_GENERIC) + publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) informer, err := informer.GetInfomerFactory().GetInformerCreator(informer.APISERVER)(publisher) if err != nil { return fmt.Errorf("fail to set informer: %v", err) @@ -109,16 +110,20 @@ func (a *Agent) stopServiceHandler() { // runAgent creates and runs rubik's agent func runAgent(ctx context.Context) error { + // 1. read configuration c := config.NewConfig(config.JSON) if err := c.LoadConfig(constant.ConfigFile); err != nil { return fmt.Errorf("error loading config: %v", err) } - // Agent parameter enable + // 2. enable log system if err := log.InitConfig(c.Agent.LogDriver, c.Agent.LogDir, c.Agent.LogLevel, c.Agent.LogSize); err != nil { return fmt.Errorf("error initializing log: %v", err) } - util.CgroupRoot = c.Agent.CgroupRoot + // 3. enable cgroup system + cgroup.InitMountDir(c.Agent.CgroupRoot) + + // 4. Create and run the agent agent, err := NewAgent(c) if err != nil { return fmt.Errorf("error new agent: %v", err) @@ -141,7 +146,7 @@ func Run() int { lock, err := util.LockFile(constant.LockFile) defer func() { lock.Close() - os.Remove(constant.LockFile) + log.DropError(os.Remove(constant.LockFile)) }() if err != nil { fmt.Printf("set rubik lock failed: %v, check if there is another rubik running\n", err) diff --git a/pkg/services/qos/qos.go b/pkg/services/qos/qos.go index 2d5ed43..902798d 100644 --- a/pkg/services/qos/qos.go +++ b/pkg/services/qos/qos.go @@ -7,10 +7,11 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/services" ) -var supportCgroupTypes = map[string]*typedef.CgroupKey{ +var supportCgroupTypes = map[string]*cgroup.Key{ "cpu": {SubSys: "cpu", FileName: constant.CPUCgroupFileName}, "memory": {SubSys: "memory", FileName: constant.MemoryCgroupFileName}, } @@ -40,16 +41,6 @@ func NewQoS() *QoS { } } -// PreStart will do some pre-start actions -func (q *QoS) PreStart(viewer api.Viewer) error { - return nil -} - -// Terminate will do some clean-up actions -func (q *QoS) Terminate(viewer api.Viewer) error { - return nil -} - // ID return qos service name func (q *QoS) ID() string { return q.Name @@ -93,30 +84,13 @@ func (q *QoS) DeleteFunc(pod *typedef.PodInfo) error { // cgroup file and the one from pod info func (q *QoS) ValidateQoS(pod *typedef.PodInfo) error { targetLevel := getQoSLevel(pod) - var compare = func(c typedef.CgroupSetterAndGetter, subSys string) bool { - value, err := c.GetCgroupAttr(supportCgroupTypes[subSys]) - if err != nil { - q.Log.Errorf("failed to get cgroup attr: %v", err) - return false - } - level, err := strconv.Atoi(value) - if err != nil { - q.Log.Errorf("failed to convert value %s to int: %v", value, err) - return false - } - if level != targetLevel { - q.Log.Errorf("different qos value between annotation %d and cgroup file %d", targetLevel, level) - return false - } - return true - } for _, subSys := range q.SubSys { - if !compare(pod, subSys) { - return fmt.Errorf("failed to validate pod %s", pod.Name) + if err := pod.GetCgroupAttr(supportCgroupTypes[subSys]).Expect(targetLevel); err != nil { + return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) } for _, container := range pod.IDContainersMap { - if !compare(container, subSys) { - return fmt.Errorf("failed to validate container %s", container.Name) + if err := container.GetCgroupAttr(supportCgroupTypes[subSys]).Expect(targetLevel); err != nil { + return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) } } } -- Gitee From 6169dc0afea7cde60c586cdb9ef6d601c7078394 Mon Sep 17 00:00:00 2001 From: vegbir Date: Mon, 13 Feb 2023 22:06:25 +0800 Subject: [PATCH 24/73] feature: support cpu utilization compute & quota save 1. add /proc/stat parse function 2. add quota comuputation and save 3. fix repo --- pkg/common/constant/constant.go | 2 + pkg/common/util/file.go | 2 +- pkg/core/typedef/cgroup/common.go | 7 + pkg/services/quotaturbo/cpu.go | 111 +++++++++++++++ pkg/services/quotaturbo/cpuquota.go | 207 ++++++++++++++++++++++++++++ 5 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 pkg/services/quotaturbo/cpu.go create mode 100644 pkg/services/quotaturbo/cpuquota.go diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index 584ce9d..de8282c 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -60,6 +60,8 @@ const ( QuotaBurstAnnotationKey = "volcano.sh/quota-burst-time" // BlkioKey is annotation key to set blkio limit BlkioKey = "volcano.sh/blkio-limit" + // QuotaAnnotationKey is annotation key to mark whether to enable the quota turbo + QuotaAnnotationKey = "volcano.sh/quota-turbo" ) // log config diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go index 22b4150..10cb472 100644 --- a/pkg/common/util/file.go +++ b/pkg/common/util/file.go @@ -122,7 +122,7 @@ func WriteFile(path, content string) error { if IsDir(path) { return fmt.Errorf("%v is not a file", path) } - // try to create Pparent directory + // try to create parent directory dirPath := filepath.Dir(path) if !PathExist(dirPath) { if err := os.MkdirAll(dirPath, constant.DefaultDirMode); err != nil { diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index ada7da0..9d0e6a9 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -25,6 +25,13 @@ import ( var rootDir = constant.DefaultCgroupRoot +func AbsoluteCgroupPath(subsys, cgroupParent, cgroupFileName string) string { + if subsys == "" { + return "" + } + return filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) +} + // ReadCgroupFile reads data from cgroup files func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) { cgfile := filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) diff --git a/pkg/services/quotaturbo/cpu.go b/pkg/services/quotaturbo/cpu.go new file mode 100644 index 0000000..8b2d3fc --- /dev/null +++ b/pkg/services/quotaturbo/cpu.go @@ -0,0 +1,111 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-08 +// Description: This file is used for computing cpu utilization + +// Package quotaturbo is for Quota Turbo +package quotaturbo + +import ( + "fmt" + "io/ioutil" + "math" + "strings" + + "isula.org/rubik/pkg/common/util" +) + +const ( + maximumUtilization float64 = 100 + minimumUtilization float64 = 0 +) + +// ProcStat store /proc/stat data +type ProcStat struct { + name string + user float64 + nice float64 + system float64 + idle float64 + iowait float64 + irq float64 + softirq float64 + steal float64 + guest float64 + guestNice float64 + total float64 + busy float64 +} + +// getProcStat create a proc stat object +func getProcStat() (ProcStat, error) { + const ( + procStatFilePath = "/proc/stat" + nameLineNum = 0 + userIndex = 0 + niceIndex = 1 + systemIndex = 2 + idleIndex = 3 + iowaitIndex = 4 + irqIndex = 5 + softirqIndex = 6 + stealIndex = 7 + guestIndex = 8 + guestNiceIndex = 9 + statsFieldsCount = 10 + supportFieldNumber = 11 + ) + data, err := ioutil.ReadFile(procStatFilePath) + if err != nil { + return ProcStat{}, err + } + // format of the first line of the file /proc/stat : + // name user nice system idle iowait irq softirq steal guest guest_nice + line := strings.Fields(strings.Split(string(data), "\n")[0]) + if len(line) < supportFieldNumber { + return ProcStat{}, fmt.Errorf("too few fields and check the kernel version") + } + var fields [statsFieldsCount]float64 + for i := 0; i < statsFieldsCount; i++ { + fields[i], err = util.ParseFloat64(line[i+1]) + if err != nil { + return ProcStat{}, err + } + } + ps := ProcStat{ + name: line[nameLineNum], + user: fields[userIndex], + nice: fields[niceIndex], + system: fields[systemIndex], + idle: fields[idleIndex], + iowait: fields[iowaitIndex], + irq: fields[irqIndex], + softirq: fields[softirqIndex], + steal: fields[stealIndex], + guest: fields[guestIndex], + guestNice: fields[guestNiceIndex], + } + ps.busy = ps.user + ps.system + ps.nice + ps.iowait + ps.irq + ps.softirq + ps.steal + ps.total = ps.busy + ps.idle + return ps, nil +} + +// calculateUtils calculate the CPU utilization rate based on the two interval /proc/stat +func calculateUtils(t1, t2 ProcStat) float64 { + if t2.busy <= t1.busy { + return minimumUtilization + } + if t2.total <= t1.total { + return maximumUtilization + } + return math.Min(maximumUtilization, + math.Max(minimumUtilization, util.Div(t2.busy-t1.busy, t2.total-t1.total)*maximumUtilization)) +} diff --git a/pkg/services/quotaturbo/cpuquota.go b/pkg/services/quotaturbo/cpuquota.go new file mode 100644 index 0000000..ab89af9 --- /dev/null +++ b/pkg/services/quotaturbo/cpuquota.go @@ -0,0 +1,207 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2022-03-18 +// Description: cpu container cpu quota data and methods + +// Package quotaturbo is for Quota Turbo +package quotaturbo + +import ( + "fmt" + "path" + "time" + + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +const ( + // numberOfRestrictedCycles is the number of periods in which the quota limits the CPU usage. + numberOfRestrictedCycles = 60 + // default cfs_period_us = 100ms + defaultCFSPeriodUs int64 = 100000 +) + +var ( + cpuPeriodKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} + cpuQuotaKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} + cpuAcctUsageKey = &cgroup.Key{SubSys: "cpuacct", FileName: "cpuacct.usage"} + cpuStatKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.stat"} +) + +// cpuUsage cpu time used by the container at timestamp +type cpuUsage struct { + timestamp int64 + usage int64 +} + +// CPUQuota stores the CPU quota information of a single container. +type CPUQuota struct { + // basic container information + *typedef.ContainerInfo + // current throttling data for the container + curThrottle *cgroup.CPUStat + // previous throttling data for container + preThrottle *cgroup.CPUStat + // container cfs_period_us + period int64 + // current cpu quota of the container + curQuota int64 + // cpu quota of the container in the next period + nextQuota int64 + // the delta of the cpu quota to be adjusted based on the decision. + quotaDelta float64 + // the upper limit of the container cpu quota + heightLimit float64 + // maximum quota that can be used by a container in the next period, + // calculated based on the total usage in the past N-1 cycles + maxQuotaNextPeriod float64 + // container cpu usage sequence + cpuUsages []cpuUsage +} + +// NewCPUQuota create a cpu quota object +func NewCPUQuota(ci *typedef.ContainerInfo) (*CPUQuota, error) { + defaultQuota := ci.LimitResources[typedef.ResourceCPU] * float64(defaultCFSPeriodUs) + cq := &CPUQuota{ + ContainerInfo: ci, + cpuUsages: make([]cpuUsage, 0), + quotaDelta: 0, + curThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, + preThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, + period: defaultCFSPeriodUs, + curQuota: int64(defaultQuota), + nextQuota: int64(defaultQuota), + heightLimit: defaultQuota, + maxQuotaNextPeriod: defaultQuota, + } + + if err := cq.updatePeriod(); err != nil { + return cq, err + } + + if err := cq.updateThrottle(); err != nil { + return cq, err + } + // The throttle data before and after the initialization is the same. + cq.preThrottle = cq.curThrottle + + if err := cq.updateQuota(); err != nil { + return cq, err + } + + if err := cq.updateUsage(); err != nil { + return cq, err + } + return cq, nil +} + +func (c *CPUQuota) updatePeriod() error { + us, err := c.ContainerInfo.GetCgroupAttr(cpuPeriodKey).Int64() + // If an error occurs, the period remains unchanged or the default value is used. + if err != nil { + return err + } + c.period = us + return nil +} + +func (c *CPUQuota) updateThrottle() error { + // update suppression times and duration + // if data cannot be obtained from cpu.stat, the value remains unchanged. + c.preThrottle = c.curThrottle + cs, err := c.ContainerInfo.GetCgroupAttr(cpuStatKey).CPUStat() + if err != nil { + return err + } + c.curThrottle = cs + return nil +} + +func (c *CPUQuota) updateQuota() error { + c.quotaDelta = 0 + curQuota, err := c.ContainerInfo.GetCgroupAttr(cpuQuotaKey).Int64() + if err != nil { + return err + } + c.curQuota = curQuota + return nil +} + +func (c *CPUQuota) updateUsage() error { + latest, err := c.ContainerInfo.GetCgroupAttr(cpuAcctUsageKey).Int64() + if err != nil { + return err + } + c.cpuUsages = append(c.cpuUsages, cpuUsage{timestamp: time.Now().UnixNano(), usage: latest}) + // ensure that the CPU usage of the container does not exceed the upper limit. + if len(c.cpuUsages) >= numberOfRestrictedCycles { + c.cpuUsages = c.cpuUsages[1:] + } + return nil +} + +func (c *CPUQuota) writePodQuota(delta int64) error { + pod := &typedef.PodInfo{ + CgroupPath: path.Dir(c.CgroupPath), + } + podQuota, err := pod.GetCgroupAttr(cpuQuotaKey).Int64() + if err == nil && podQuota == -1 { + return nil + } + podQuota += delta + return pod.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(podQuota)) +} + +func (c *CPUQuota) writeContainerQuota() error { + return c.ContainerInfo.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(c.curQuota)) +} + +// SaveQuota use to modify quota for container +func (c *CPUQuota) SaveQuota() error { + delta := c.nextQuota - c.curQuota + tmp := c.curQuota + c.curQuota = c.nextQuota + if delta < 0 { + // update container data first + if err := c.writeContainerQuota(); err != nil { + c.curQuota = tmp + return fmt.Errorf("fail to write container's quota for container %v: %v", c.ID, err) + } + // then update the pod data + if err := c.writePodQuota(delta); err != nil { + // recover + c.curQuota = tmp + if recoverErr := c.writeContainerQuota(); recoverErr != nil { + log.Errorf("fail to recover contaienr's quota for container %v: %v", c.ID, recoverErr) + } + return fmt.Errorf("fail to write pod's quota for container %v: %v", c.ID, err) + } + } else if delta > 0 { + // update pod data first + if err := c.writePodQuota(delta); err != nil { + c.curQuota = tmp + return fmt.Errorf("fail to write pod's quota for container %v: %v", c.ID, err) + } + // then update the container data + if err := c.writeContainerQuota(); err != nil { + // recover + c.curQuota = tmp + if recoverErr := c.writePodQuota(-delta); recoverErr != nil { + log.Errorf("fail to recover pod's quota for container %v: %v", c.ID, recoverErr) + } + return fmt.Errorf("fail to write container's quota for container %v: %v", c.ID, err) + } + } + return nil +} -- Gitee From 4117245a6aa62a2f8614f6600b79170526b1876b Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Mon, 13 Feb 2023 22:36:50 +0800 Subject: [PATCH 25/73] test: add pod test framework 1. add fake pod generator for test use 2. move try from "common" package to "test" package 3. add qos service test case Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/common/log/log_test.go | 2 +- pkg/core/typedef/rawpod.go | 15 +- pkg/services/qos/qos.go | 29 +++- pkg/services/qos/qos_test.go | 210 +++++++++++++++++++++++++++ test/try/pod.go | 180 +++++++++++++++++++++++ {pkg/common => test}/try/try.go | 14 +- {pkg/common => test}/try/try_test.go | 0 7 files changed, 429 insertions(+), 21 deletions(-) create mode 100644 pkg/services/qos/qos_test.go create mode 100644 test/try/pod.go rename {pkg/common => test}/try/try.go (91%) rename {pkg/common => test}/try/try_test.go (100%) diff --git a/pkg/common/log/log_test.go b/pkg/common/log/log_test.go index 2b2e3be..1156871 100644 --- a/pkg/common/log/log_test.go +++ b/pkg/common/log/log_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/assert" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/try" + "isula.org/rubik/test/try" ) // test_rubik_set_logdriver_0001 diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index c3ea5ae..b8c85a6 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -15,6 +15,7 @@ package typedef import ( + "fmt" "path/filepath" "strings" @@ -22,7 +23,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/log" ) const ( @@ -148,8 +148,8 @@ func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { // 2. generate ID-Container mapping podCgroupPath := pod.CgroupPath() for _, rawContainer := range nameRawContainersMap { - id := rawContainer.GetRealContainerID() - if id == "" { + id, err := rawContainer.GetRealContainerID() + if id == "" || err != nil { continue } idContainersMap[id] = NewContainerInfo(id, podCgroupPath, rawContainer) @@ -158,7 +158,7 @@ func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { } // GetRealContainerID parses the containerID of k8s -func (cont *RawContainer) GetRealContainerID() string { +func (cont *RawContainer) GetRealContainerID() (string, error) { /* Note: An UNDEFINED container engine was used when the function was executed for the first time @@ -169,16 +169,15 @@ func (cont *RawContainer) GetRealContainerID() string { setContainerEnginesOnce.Do(func() { fixContainerEngine(cont.status.ContainerID) }) if !currentContainerEngines.Support(cont) { - log.Errorf("fatal error : unsupported container engine") - return "" + return "", fmt.Errorf("fatal error : unsupported container engine") } cid := cont.status.ContainerID[len(currentContainerEngines.Prefix()):] // the container may be in the creation or deletion phase. if len(cid) == 0 { - return "" + return "", nil } - return cid + return cid, nil } // GetResourceMaps returns the number of requests and limits of CPU and memory resources diff --git a/pkg/services/qos/qos.go b/pkg/services/qos/qos.go index 902798d..2868acc 100644 --- a/pkg/services/qos/qos.go +++ b/pkg/services/qos/qos.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-10 +// Description: This file implement qos level setting service + +// Package qos is the service used for qos level setting package qos import ( @@ -99,8 +113,11 @@ func (q *QoS) ValidateQoS(pod *typedef.PodInfo) error { // SetQoS set pod and all containers' qos level within it func (q *QoS) SetQoS(pod *typedef.PodInfo) error { + if pod == nil { + return fmt.Errorf("pod info is empty") + } qosLevel := getQoSLevel(pod) - if qosLevel != constant.Offline { + if qosLevel == constant.Online { q.Log.Debugf("pod %s already online", pod.Name) return nil } @@ -121,19 +138,19 @@ func (q *QoS) SetQoS(pod *typedef.PodInfo) error { func getQoSLevel(pod *typedef.PodInfo) int { if pod == nil { - return 0 + return constant.Online } anno, ok := pod.Annotations[constant.PriorityAnnotationKey] if !ok { - return 0 + return constant.Online } switch anno { case "true": - return -1 + return constant.Offline case "false": - return 0 + return constant.Online default: - return 0 + return constant.Online } } diff --git a/pkg/services/qos/qos_test.go b/pkg/services/qos/qos_test.go new file mode 100644 index 0000000..62fdf4f --- /dev/null +++ b/pkg/services/qos/qos_test.go @@ -0,0 +1,210 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-10 +// Description: This file test qos level setting service + +// Package qos is the service used for qos level setting +package qos + +import ( + "context" + "testing" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +func init() { + cgroup.InitMountDir(try.TestRoot) +} + +type fields struct { + Name string + Log api.Logger + Config Config +} +type args struct { + old *typedef.PodInfo + new *typedef.PodInfo +} + +type test struct { + name string + fields fields + args args + wantErr bool + preHook func(*typedef.PodInfo) *typedef.PodInfo +} + +var getCommonField = func(subSys []string) fields { + return fields{ + Name: "qos", + Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), + Config: Config{SubSys: subSys}, + } +} +var addFuncTC = []test{ + { + name: "TC1-set offline pod qos ok", + fields: getCommonField([]string{"cpu", "memory"}), + args: args{ + new: try.GenFakeOfflinePod([]*cgroup.Key{ + supportCgroupTypes["cpu"], + supportCgroupTypes["memory"], + }).PodInfo, + }, + }, + { + name: "TC2-set online pod qos ok", + fields: getCommonField([]string{"cpu", "memory"}), + args: args{ + new: try.GenFakeOnlinePod([]*cgroup.Key{ + supportCgroupTypes["cpu"], + supportCgroupTypes["memory"], + }).WithContainers(3).PodInfo, + }, + }, + { + name: "TC3-empty pod info", + fields: getCommonField([]string{"cpu", "memory"}), + wantErr: true, + }, + { + name: "TC4-invalid annotation key", + fields: getCommonField([]string{"cpu"}), + args: args{ + new: try.GenFakeBestEffortPod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo, + }, + preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + newPod := pod.DeepCopy() + newPod.Annotations["undefine"] = "true" + return newPod + }, + }, + { + name: "TC5-invalid annotation value", + fields: getCommonField([]string{"cpu"}), + args: args{ + new: try.GenFakeBestEffortPod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo, + }, + preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + newPod := pod.DeepCopy() + newPod.Annotations[constant.PriorityAnnotationKey] = "undefine" + return newPod + }, + }, +} + +func TestQoS_AddFunc(t *testing.T) { + for _, tt := range addFuncTC { + t.Run(tt.name, func(t *testing.T) { + q := &QoS{ + Name: tt.fields.Name, + Log: tt.fields.Log, + Config: tt.fields.Config, + } + if tt.preHook != nil { + tt.preHook(tt.args.new) + } + if err := q.AddFunc(tt.args.new); (err != nil) != tt.wantErr { + t.Errorf("QoS.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +var updateFuncTC = []test{ + { + name: "TC1-online to offline", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOnlinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).WithContainers(3).PodInfo}, + preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + newPod := pod.DeepCopy() + // TODO: need fix pod.DeepCopy + newAnnotation := make(map[string]string, 0) + newAnnotation[constant.PriorityAnnotationKey] = "true" + newPod.Annotations = newAnnotation + return newPod + }, + }, + { + name: "TC2-offline to online", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOfflinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo}, + wantErr: true, + }, + { + name: "TC3-online to online", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOnlinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo}, + }, +} + +func TestQoS_UpdateFunc(t *testing.T) { + for _, tt := range updateFuncTC { + t.Run(tt.name, func(t *testing.T) { + q := &QoS{ + Name: tt.fields.Name, + Log: tt.fields.Log, + Config: tt.fields.Config, + } + if tt.preHook != nil { + tt.args.new = tt.preHook(tt.args.old) + } + if err := q.UpdateFunc(tt.args.old, tt.args.new); (err != nil) != tt.wantErr { + t.Errorf("QoS.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +var validateTC = []test{ + { + name: "TC1-normal config", + fields: fields{ + Name: "qos", + Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), + Config: Config{SubSys: []string{"cpu", "memory"}}, + }, + }, + { + name: "TC2-abnormal config", + fields: fields{ + Name: "undefine", + Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), + Config: Config{SubSys: []string{"undefine"}}, + }, + wantErr: true, + }, + { + name: "TC3-empty config", + wantErr: true, + }, +} + +func TestQoS_Validate(t *testing.T) { + for _, tt := range validateTC { + t.Run(tt.name, func(t *testing.T) { + q := &QoS{ + Name: tt.fields.Name, + Log: tt.fields.Log, + Config: tt.fields.Config, + } + if err := q.Validate(); (err != nil) != tt.wantErr { + t.Errorf("QoS.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/test/try/pod.go b/test/try/pod.go new file mode 100644 index 0000000..ec16287 --- /dev/null +++ b/test/try/pod.go @@ -0,0 +1,180 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-10 +// Description: This file contains pod info and cgroup construct + +package try + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +// FakePod is used for pod testing +type FakePod struct { + *typedef.PodInfo + // Keys is cgroup key list + Keys []*cgroup.Key +} + +const idLen = 8 + +func genFakeContainerInfo(parentCGPath string) *typedef.ContainerInfo { + containerID := genContainerID() + var fakeContainer = &typedef.ContainerInfo{ + Name: fmt.Sprintf("fakeContainer-%s", containerID[:idLen]), + ID: containerID, + CgroupPath: filepath.Join(parentCGPath, containerID), + RequestResources: make(typedef.ResourceMap, 0), + LimitResources: make(typedef.ResourceMap, 0), + } + return fakeContainer +} + +func genFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { + podID := uuid.New().String() + // generate fake pod info + var fakePod = &typedef.PodInfo{ + Name: fmt.Sprintf("fakepod-%s", podID[:idLen]), + Namespace: "test", + UID: constant.PodCgroupNamePrefix + podID, + CgroupPath: genRelativeCgroupPath(qosClass, podID), + Annotations: make(map[string]string, 0), + } + return fakePod +} + +// NewFakePod return fake pod info struct +func NewFakePod(keys []*cgroup.Key, qosClass corev1.PodQOSClass) *FakePod { + return &FakePod{ + Keys: keys, + PodInfo: genFakePodInfo(qosClass), + } +} + +func (pod *FakePod) genFakePodCgroupPath() { + if !util.PathExist(TestRoot) { + MkdirAll(TestRoot, constant.DefaultDirMode).OrDie() + } + // generate fake cgroup path + for _, key := range pod.Keys { + // generate pod absolute cgroup path + podCGPath := filepath.Join(TestRoot, key.SubSys, pod.CgroupPath) + MkdirAll(podCGPath, constant.DefaultDirMode).OrDie() + // create pod cgroup file, default qos level is "0" + podCGFilePath := filepath.Join(podCGPath, key.FileName) + WriteFile(podCGFilePath, []byte(string("0")), constant.DefaultFileMode).OrDie() + } + pod.genFakeContainersCgroupPath() +} + +func (pod *FakePod) genFakeContainersCgroupPath() { + if len(pod.IDContainersMap) == 0 { + return + } + + for _, key := range pod.Keys { + podCGPath := filepath.Join(TestRoot, key.SubSys, pod.CgroupPath) + if !util.PathExist(podCGPath) { + return + } + for _, container := range pod.IDContainersMap { + // generate container absolute cgroup path + containerCGPath := filepath.Join(podCGPath, container.ID) + MkdirAll(containerCGPath, constant.DefaultDirMode).OrDie() + // create container cgroup file, default qos level is "0" + containerCGFilePath := filepath.Join(containerCGPath, key.FileName) + WriteFile(containerCGFilePath, []byte(string("0")), constant.DefaultFileMode).OrDie() + } + } +} + +// WithContainers will generate containers under pod with container num +func (pod *FakePod) WithContainers(containerNum int) *FakePod { + pod.IDContainersMap = make(map[string]*typedef.ContainerInfo, containerNum) + for i := 0; i < containerNum; i++ { + fakeContainer := genFakeContainerInfo(pod.CgroupPath) + pod.IDContainersMap[fakeContainer.ID] = fakeContainer + } + pod.genFakeContainersCgroupPath() + return pod +} + +func genContainerID() string { + const delimiter = "-" + const containerIDLenLimit = 64 + uuid1 := uuid.New().String() + uuid2 := uuid.New().String() + containerID := strings.ReplaceAll(uuid1, delimiter, "") + strings.ReplaceAll(uuid2, delimiter, "") + if len(containerID) < containerIDLenLimit { + containerID += strings.Repeat("a", containerIDLenLimit-len(containerID)) + } + return containerID[:containerIDLenLimit] +} + +// GenFakePod gen fake pod info +func GenFakePod(keys []*cgroup.Key, qosClass corev1.PodQOSClass) *FakePod { + fakePod := NewFakePod(keys, qosClass) + fakePod.genFakePodCgroupPath() + return fakePod +} + +// GenFakeBurstablePod generate pod with qos class burstable +func GenFakeBurstablePod(keys []*cgroup.Key) *FakePod { + return GenFakePod(keys, corev1.PodQOSBurstable) +} + +// GenFakeBestEffortPod generate pod with qos class best effort +func GenFakeBestEffortPod(keys []*cgroup.Key) *FakePod { + return GenFakePod(keys, corev1.PodQOSBestEffort) +} + +// GenFakeGuaranteedPod generate pod with qos class guaranteed +func GenFakeGuaranteedPod(keys []*cgroup.Key) *FakePod { + return GenFakePod(keys, corev1.PodQOSGuaranteed) +} + +// GenFakeOnlinePod generate online pod +func GenFakeOnlinePod(keys []*cgroup.Key) *FakePod { + fakePod := GenFakeGuaranteedPod(keys) + fakePod.Annotations[constant.PriorityAnnotationKey] = "false" + return fakePod +} + +// GenFakeOfflinePod generate offline pod +func GenFakeOfflinePod(keys []*cgroup.Key) *FakePod { + fakePod := GenFakeBurstablePod(keys) + fakePod.Annotations[constant.PriorityAnnotationKey] = "true" + return fakePod +} + +func genRelativeCgroupPath(qosClass corev1.PodQOSClass, id string) string { + path := "" + switch qosClass { + case corev1.PodQOSGuaranteed: + case corev1.PodQOSBurstable: + path = strings.ToLower(string(corev1.PodQOSBurstable)) + case corev1.PodQOSBestEffort: + path = strings.ToLower(string(corev1.PodQOSBestEffort)) + default: + return "" + } + return filepath.Join(constant.KubepodsCgroup, path, constant.PodCgroupNamePrefix+id) +} diff --git a/pkg/common/try/try.go b/test/try/try.go similarity index 91% rename from pkg/common/try/try.go rename to test/try/try.go index 5f84dfa..7ab0c3c 100644 --- a/pkg/common/try/try.go +++ b/test/try/try.go @@ -10,7 +10,7 @@ // Author: jingrui // Create: 2022-04-17 // Description: try provide some helper functions for unit-test. -// + // Package try provide some helper function for unit-test, if you want // to use try outside unit-test, please add notes. // @@ -27,6 +27,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" securejoin "github.com/cyphar/filepath-securejoin" "github.com/google/uuid" @@ -110,18 +111,19 @@ func WriteFile(filename string, data []byte, perm os.FileMode) Ret { } const ( - testdir = "/tmp/rubik-test" + // TestRoot is the root path for all test cases + TestRoot = "/tmp/rubik-test" ) // GenTestDir gen testdir func GenTestDir() Ret { - name := fmt.Sprintf("%s/%s", testdir, uuid.New().String()) - ret := MkdirAll(name, constant.DefaultDirMode) - ret.val = name + path := filepath.Join(TestRoot, uuid.New().String()) + ret := MkdirAll(path, constant.DefaultDirMode) + ret.val = path return ret } // DelTestDir del testdir, this function only need call once. func DelTestDir() Ret { - return RemoveAll(testdir) + return RemoveAll(TestRoot) } diff --git a/pkg/common/try/try_test.go b/test/try/try_test.go similarity index 100% rename from pkg/common/try/try_test.go rename to test/try/try_test.go -- Gitee From de7389652da12622f2013947fb9c52743238a87a Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Tue, 14 Feb 2023 11:46:36 +0800 Subject: [PATCH 26/73] test: add testcase for pod test freamwork 1. add testcase for try package 2. add ReadFile function for try package 3. replace WriteFile with util.WriteFile in try package 4. add pod path clean function Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/common/log/log_test.go | 4 +- pkg/services/qos/qos_test.go | 65 +++++---- test/try/pod.go | 99 ++++++++----- test/try/pod_test.go | 261 +++++++++++++++++++++++++++++++++++ test/try/try.go | 17 ++- test/try/try_test.go | 6 +- 6 files changed, 382 insertions(+), 70 deletions(-) create mode 100644 test/try/pod_test.go diff --git a/pkg/common/log/log_test.go b/pkg/common/log/log_test.go index 1156871..8476b17 100644 --- a/pkg/common/log/log_test.go +++ b/pkg/common/log/log_test.go @@ -34,7 +34,7 @@ func TestInitConfigLogDriver(t *testing.T) { logFilePath := filepath.Join(logDir, "rubik.log") // case: rubik.log already exist. - try.WriteFile(logFilePath, []byte(""), constant.DefaultFileMode) + try.WriteFile(logFilePath, "") err := InitConfig("file", logDir, "", logSize) assert.NoError(t, err) @@ -248,7 +248,7 @@ func TestDropError(t *testing.T) { // TestLogOthers is log other tests func TestLogOthers(t *testing.T) { logDir := filepath.Join(try.GenTestDir().String(), "regular-file") - try.WriteFile(logDir, []byte{}, constant.DefaultFileMode) + try.WriteFile(logDir, "") err := makeLogDir(logDir) assert.Equal(t, true, err != nil) diff --git a/pkg/services/qos/qos_test.go b/pkg/services/qos/qos_test.go index 62fdf4f..8128c37 100644 --- a/pkg/services/qos/qos_test.go +++ b/pkg/services/qos/qos_test.go @@ -21,7 +21,6 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/test/try" ) @@ -36,8 +35,8 @@ type fields struct { Config Config } type args struct { - old *typedef.PodInfo - new *typedef.PodInfo + old *try.FakePod + new *try.FakePod } type test struct { @@ -45,7 +44,7 @@ type test struct { fields fields args args wantErr bool - preHook func(*typedef.PodInfo) *typedef.PodInfo + preHook func(*try.FakePod) *try.FakePod } var getCommonField = func(subSys []string) fields { @@ -60,20 +59,20 @@ var addFuncTC = []test{ name: "TC1-set offline pod qos ok", fields: getCommonField([]string{"cpu", "memory"}), args: args{ - new: try.GenFakeOfflinePod([]*cgroup.Key{ - supportCgroupTypes["cpu"], - supportCgroupTypes["memory"], - }).PodInfo, + new: try.GenFakeOfflinePod(map[*cgroup.Key]string{ + supportCgroupTypes["cpu"]: "0", + supportCgroupTypes["memory"]: "0", + }), }, }, { name: "TC2-set online pod qos ok", fields: getCommonField([]string{"cpu", "memory"}), args: args{ - new: try.GenFakeOnlinePod([]*cgroup.Key{ - supportCgroupTypes["cpu"], - supportCgroupTypes["memory"], - }).WithContainers(3).PodInfo, + new: try.GenFakeOnlinePod(map[*cgroup.Key]string{ + supportCgroupTypes["cpu"]: "0", + supportCgroupTypes["memory"]: "0", + }).WithContainers(3), }, }, { @@ -85,9 +84,9 @@ var addFuncTC = []test{ name: "TC4-invalid annotation key", fields: getCommonField([]string{"cpu"}), args: args{ - new: try.GenFakeBestEffortPod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo, + new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), }, - preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + preHook: func(pod *try.FakePod) *try.FakePod { newPod := pod.DeepCopy() newPod.Annotations["undefine"] = "true" return newPod @@ -97,9 +96,9 @@ var addFuncTC = []test{ name: "TC5-invalid annotation value", fields: getCommonField([]string{"cpu"}), args: args{ - new: try.GenFakeBestEffortPod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo, + new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), }, - preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + preHook: func(pod *try.FakePod) *try.FakePod { newPod := pod.DeepCopy() newPod.Annotations[constant.PriorityAnnotationKey] = "undefine" return newPod @@ -118,9 +117,13 @@ func TestQoS_AddFunc(t *testing.T) { if tt.preHook != nil { tt.preHook(tt.args.new) } - if err := q.AddFunc(tt.args.new); (err != nil) != tt.wantErr { - t.Errorf("QoS.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + if tt.args.new != nil { + if err := q.AddFunc(tt.args.new.PodInfo); (err != nil) != tt.wantErr { + t.Errorf("QoS.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + } + } + tt.args.new.CleanPath().OrDie() }) } } @@ -129,8 +132,8 @@ var updateFuncTC = []test{ { name: "TC1-online to offline", fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOnlinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).WithContainers(3).PodInfo}, - preHook: func(pod *typedef.PodInfo) *typedef.PodInfo { + args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}).WithContainers(3)}, + preHook: func(pod *try.FakePod) *try.FakePod { newPod := pod.DeepCopy() // TODO: need fix pod.DeepCopy newAnnotation := make(map[string]string, 0) @@ -140,15 +143,25 @@ var updateFuncTC = []test{ }, }, { - name: "TC2-offline to online", - fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOfflinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo}, + name: "TC2-offline to online", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOfflinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, + preHook: func(pod *try.FakePod) *try.FakePod { + newPod := pod.DeepCopy() + newAnnotation := make(map[string]string, 0) + newAnnotation[constant.PriorityAnnotationKey] = "false" + newPod.Annotations = newAnnotation + return newPod + }, wantErr: true, }, { name: "TC3-online to online", fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOnlinePod([]*cgroup.Key{supportCgroupTypes["cpu"]}).PodInfo}, + args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, + preHook: func(pod *try.FakePod) *try.FakePod { + return pod.DeepCopy() + }, }, } @@ -163,9 +176,11 @@ func TestQoS_UpdateFunc(t *testing.T) { if tt.preHook != nil { tt.args.new = tt.preHook(tt.args.old) } - if err := q.UpdateFunc(tt.args.old, tt.args.new); (err != nil) != tt.wantErr { + if err := q.UpdateFunc(tt.args.old.PodInfo, tt.args.new.PodInfo); (err != nil) != tt.wantErr { t.Errorf("QoS.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) } + tt.args.new.CleanPath().OrDie() + tt.args.old.CleanPath().OrDie() }) } } diff --git a/test/try/pod.go b/test/try/pod.go index ec16287..05b7a26 100644 --- a/test/try/pod.go +++ b/test/try/pod.go @@ -31,7 +31,7 @@ import ( type FakePod struct { *typedef.PodInfo // Keys is cgroup key list - Keys []*cgroup.Key + Keys map[*cgroup.Key]string } const idLen = 8 @@ -62,48 +62,44 @@ func genFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { } // NewFakePod return fake pod info struct -func NewFakePod(keys []*cgroup.Key, qosClass corev1.PodQOSClass) *FakePod { +func NewFakePod(keys map[*cgroup.Key]string, qosClass corev1.PodQOSClass) *FakePod { return &FakePod{ Keys: keys, PodInfo: genFakePodInfo(qosClass), } } -func (pod *FakePod) genFakePodCgroupPath() { +func (pod *FakePod) genFakePodCgroupPath() Ret { if !util.PathExist(TestRoot) { MkdirAll(TestRoot, constant.DefaultDirMode).OrDie() } + cgroup.InitMountDir(TestRoot) // generate fake cgroup path - for _, key := range pod.Keys { + for key, value := range pod.Keys { // generate pod absolute cgroup path - podCGPath := filepath.Join(TestRoot, key.SubSys, pod.CgroupPath) - MkdirAll(podCGPath, constant.DefaultDirMode).OrDie() - // create pod cgroup file, default qos level is "0" - podCGFilePath := filepath.Join(podCGPath, key.FileName) - WriteFile(podCGFilePath, []byte(string("0")), constant.DefaultFileMode).OrDie() + podCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, pod.CgroupPath, key.FileName) + if err := WriteFile(podCGFilePath, value); err.err != nil { + return err + } } - pod.genFakeContainersCgroupPath() + return pod.genFakeContainersCgroupPath() } -func (pod *FakePod) genFakeContainersCgroupPath() { +func (pod *FakePod) genFakeContainersCgroupPath() Ret { if len(pod.IDContainersMap) == 0 { - return + return newRet(nil) } - for _, key := range pod.Keys { - podCGPath := filepath.Join(TestRoot, key.SubSys, pod.CgroupPath) - if !util.PathExist(podCGPath) { - return - } + for key, value := range pod.Keys { for _, container := range pod.IDContainersMap { // generate container absolute cgroup path - containerCGPath := filepath.Join(podCGPath, container.ID) - MkdirAll(containerCGPath, constant.DefaultDirMode).OrDie() - // create container cgroup file, default qos level is "0" - containerCGFilePath := filepath.Join(containerCGPath, key.FileName) - WriteFile(containerCGFilePath, []byte(string("0")), constant.DefaultFileMode).OrDie() + containerCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, container.CgroupPath, key.FileName) + if err := WriteFile(containerCGFilePath, value); err.err != nil { + return err + } } } + return newRet(nil) } // WithContainers will generate containers under pod with container num @@ -117,49 +113,63 @@ func (pod *FakePod) WithContainers(containerNum int) *FakePod { return pod } +// CleanPath will delete fakepod's cgroup folders and files +func (pod *FakePod) CleanPath() Ret { + if pod == nil { + return newRet(nil) + } + for key := range pod.Keys { + path := cgroup.AbsoluteCgroupPath(key.SubSys, pod.CgroupPath, key.FileName) + if err := RemoveAll(filepath.Dir(path)); err.err != nil { + return err + } + } + return newRet(nil) +} + func genContainerID() string { const delimiter = "-" - const containerIDLenLimit = 64 + // format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + // length: 36 + // delimiter no: 4 uuid1 := uuid.New().String() uuid2 := uuid.New().String() + // now one uuid length is 64 for sure containerID := strings.ReplaceAll(uuid1, delimiter, "") + strings.ReplaceAll(uuid2, delimiter, "") - if len(containerID) < containerIDLenLimit { - containerID += strings.Repeat("a", containerIDLenLimit-len(containerID)) - } - return containerID[:containerIDLenLimit] + return containerID } // GenFakePod gen fake pod info -func GenFakePod(keys []*cgroup.Key, qosClass corev1.PodQOSClass) *FakePod { +func GenFakePod(keys map[*cgroup.Key]string, qosClass corev1.PodQOSClass) *FakePod { fakePod := NewFakePod(keys, qosClass) - fakePod.genFakePodCgroupPath() + fakePod.genFakePodCgroupPath().OrDie() return fakePod } // GenFakeBurstablePod generate pod with qos class burstable -func GenFakeBurstablePod(keys []*cgroup.Key) *FakePod { +func GenFakeBurstablePod(keys map[*cgroup.Key]string) *FakePod { return GenFakePod(keys, corev1.PodQOSBurstable) } // GenFakeBestEffortPod generate pod with qos class best effort -func GenFakeBestEffortPod(keys []*cgroup.Key) *FakePod { +func GenFakeBestEffortPod(keys map[*cgroup.Key]string) *FakePod { return GenFakePod(keys, corev1.PodQOSBestEffort) } // GenFakeGuaranteedPod generate pod with qos class guaranteed -func GenFakeGuaranteedPod(keys []*cgroup.Key) *FakePod { +func GenFakeGuaranteedPod(keys map[*cgroup.Key]string) *FakePod { return GenFakePod(keys, corev1.PodQOSGuaranteed) } // GenFakeOnlinePod generate online pod -func GenFakeOnlinePod(keys []*cgroup.Key) *FakePod { +func GenFakeOnlinePod(keys map[*cgroup.Key]string) *FakePod { fakePod := GenFakeGuaranteedPod(keys) fakePod.Annotations[constant.PriorityAnnotationKey] = "false" return fakePod } // GenFakeOfflinePod generate offline pod -func GenFakeOfflinePod(keys []*cgroup.Key) *FakePod { +func GenFakeOfflinePod(keys map[*cgroup.Key]string) *FakePod { fakePod := GenFakeBurstablePod(keys) fakePod.Annotations[constant.PriorityAnnotationKey] = "true" return fakePod @@ -169,12 +179,29 @@ func genRelativeCgroupPath(qosClass corev1.PodQOSClass, id string) string { path := "" switch qosClass { case corev1.PodQOSGuaranteed: + path = "" case corev1.PodQOSBurstable: path = strings.ToLower(string(corev1.PodQOSBurstable)) case corev1.PodQOSBestEffort: path = strings.ToLower(string(corev1.PodQOSBestEffort)) - default: - return "" } return filepath.Join(constant.KubepodsCgroup, path, constant.PodCgroupNamePrefix+id) } + +// DeepCopy returns fake pod deepcopy object +func (pod *FakePod) DeepCopy() *FakePod { + if pod == nil || pod.PodInfo == nil { + return nil + } + return &FakePod{ + Keys: util.DeepCopy(pod.Keys).(map[*cgroup.Key]string), + PodInfo: &typedef.PodInfo{ + Name: pod.Name, + UID: pod.UID, + CgroupPath: pod.CgroupPath, + Namespace: pod.Namespace, + Annotations: util.DeepCopy(pod.Annotations).(map[string]string), + IDContainersMap: util.DeepCopy(pod.IDContainersMap).(map[string]*typedef.ContainerInfo), + }, + } +} diff --git a/test/try/pod_test.go b/test/try/pod_test.go new file mode 100644 index 0000000..bde226b --- /dev/null +++ b/test/try/pod_test.go @@ -0,0 +1,261 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-10 +// Description: This file contains pod info and cgroup construct + +package try + +import ( + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + corev1 "k8s.io/api/core/v1" +) + +func TestNewFakePod(t *testing.T) { + id := constant.PodCgroupNamePrefix + uuid.New().String() + type args struct { + keys map[*cgroup.Key]string + qosClass corev1.PodQOSClass + } + tests := []struct { + name string + args args + want *FakePod + }{ + { + name: "TC1-new fake best effort pod", + args: args{ + keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + qosClass: corev1.PodQOSBestEffort, + }, + want: &FakePod{ + Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + PodInfo: &typedef.PodInfo{ + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + CgroupPath: filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), id), + }, + }, + }, + { + name: "TC2-new fake guaranteed pod", + args: args{ + keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + qosClass: corev1.PodQOSGuaranteed, + }, + want: &FakePod{ + Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + PodInfo: &typedef.PodInfo{ + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + CgroupPath: filepath.Join(constant.KubepodsCgroup, id), + }, + }, + }, + { + name: "TC3-new fake burstable pod", + args: args{ + keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + qosClass: corev1.PodQOSBurstable, + }, + want: &FakePod{ + Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + PodInfo: &typedef.PodInfo{ + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + CgroupPath: filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), id), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakePod := NewFakePod(tt.args.keys, tt.args.qosClass) + assert.Equal(t, fakePod.Namespace, tt.want.Namespace) + assert.Equal(t, len(fakePod.Name), len(tt.want.Name)) + assert.Equal(t, len(fakePod.UID), len(tt.want.UID)) + assert.Equal(t, len(fakePod.CgroupPath), len(tt.want.CgroupPath)) + }) + } +} + +func TestGenFakePod(t *testing.T) { + type args struct { + keys map[*cgroup.Key]string + qosClass corev1.PodQOSClass + containerNum int + } + tests := []struct { + name string + args args + }{ + { + name: "TC1-generate burstable pod", + args: args{ + keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + qosClass: corev1.PodQOSBestEffort, + }, + }, + { + name: "TC2-generate guaranteed pod with 3 containers", + args: args{ + keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, + qosClass: corev1.PodQOSGuaranteed, + containerNum: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakePod := GenFakePod(tt.args.keys, tt.args.qosClass) + if tt.args.qosClass != corev1.PodQOSGuaranteed { + // guaranteed pod does not have path prefix like "guaranteed/podxxx" + assert.Equal(t, true, strings.Contains(fakePod.CgroupPath, strings.ToLower(string(corev1.PodQOSBestEffort)))) + + } + for key, val := range tt.args.keys { + podCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, fakePod.CgroupPath, key.FileName) + assert.Equal(t, true, util.PathExist(podCgroupFile)) + ret := ReadFile(podCgroupFile) + assert.NoError(t, ret.err) + assert.Equal(t, val, ret.val) + } + if tt.args.containerNum != 0 { + fakePod.WithContainers(tt.args.containerNum) + for key, val := range tt.args.keys { + for _, c := range fakePod.IDContainersMap { + containerCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, c.CgroupPath, key.FileName) + assert.Equal(t, true, util.PathExist(containerCgroupFile)) + ret := ReadFile(containerCgroupFile) + assert.NoError(t, ret.err) + assert.Equal(t, val, ret.val) + } + } + } + fakePod.CleanPath().OrDie() + }) + } +} + +func TestGenParticularFakePod(t *testing.T) { + type args struct { + keys map[*cgroup.Key]string + } + tests := []struct { + name string + kind string + wantAnnotation string + }{ + { + name: "TC1-generate online pod", + kind: "online", + wantAnnotation: "false", + }, + { + name: "TC2-generate offline pod", + kind: "offline", + wantAnnotation: "true", + }, + { + name: "TC3-generate burstable pod", + kind: "burstable", + wantAnnotation: "", + }, + { + name: "TC4-generate besteffort pod", + kind: "besteffort", + wantAnnotation: "", + }, + { + name: "TC5-generate guaranteed pod", + kind: "guaranteed", + wantAnnotation: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + keys := map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"} + var fakePod *FakePod + switch tt.kind { + case "online": + fakePod = GenFakeOnlinePod(keys) + case "offline": + fakePod = GenFakeOfflinePod(keys) + case "burstable": + fakePod = GenFakeBurstablePod(keys) + case "besteffort": + fakePod = GenFakeBestEffortPod(keys) + case "guaranteed": + fakePod = GenFakeGuaranteedPod(keys) + } + assert.Equal(t, tt.wantAnnotation, fakePod.Annotations[constant.PriorityAnnotationKey]) + fakePod.CleanPath().OrDie() + }) + } +} + +func TestFakePod_DeepCopy(t *testing.T) { + type fields struct { + PodInfo *typedef.PodInfo + Keys map[*cgroup.Key]string + } + + keys := map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"} + podInfo := NewFakePod(keys, corev1.PodQOSGuaranteed).PodInfo + tests := []struct { + name string + fields fields + want *FakePod + }{ + { + name: "TC1-deep copy", + fields: fields{ + PodInfo: podInfo, + Keys: keys, + }, + want: &FakePod{ + PodInfo: podInfo, + Keys: keys, + }, + }, + { + name: "TC2-empty copy", + fields: fields{ + PodInfo: nil, + Keys: nil, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pod := &FakePod{ + PodInfo: tt.fields.PodInfo, + Keys: tt.fields.Keys, + } + if got := pod.DeepCopy(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("FakePod.DeepCopy() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test/try/try.go b/test/try/try.go index 7ab0c3c..438665f 100644 --- a/test/try/try.go +++ b/test/try/try.go @@ -25,7 +25,6 @@ package try import ( "fmt" - "io/ioutil" "os" "path/filepath" @@ -33,6 +32,7 @@ import ( "github.com/google/uuid" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" ) // Ret provide some action for error. @@ -101,15 +101,26 @@ func RemoveAll(path string) Ret { } // WriteFile wrap error to Ret. -func WriteFile(filename string, data []byte, perm os.FileMode) Ret { +func WriteFile(filename string, data string) Ret { ret := newRet(nil) ret.val = filename - if err := ioutil.WriteFile(filename, data, perm); err != nil { + if err := util.WriteFile(filename, data); err != nil { ret.err = err } return ret } +// ReadFile wrap error to Ret +func ReadFile(filename string) Ret { + ret := newRet(nil) + val, err := util.ReadFile(filename) + if err != nil { + ret.err = err + } + ret.val = string(val) + return ret +} + const ( // TestRoot is the root path for all test cases TestRoot = "/tmp/rubik-test" diff --git a/test/try/try_test.go b/test/try/try_test.go index 9fbbc33..7fbc257 100644 --- a/test/try/try_test.go +++ b/test/try/try_test.go @@ -20,8 +20,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/common/constant" ) // Test_OrDie test try some-func or die. @@ -29,7 +27,7 @@ func Test_OrDie(t *testing.T) { ret := GenTestDir() ret.OrDie() dname := ret.String() - WriteFile(SecureJoin(dname, "die.txt").String(), []byte("ok"), constant.DefaultFileMode).OrDie() + WriteFile(SecureJoin(dname, "die.txt").String(), "ok").OrDie() RemoveAll(dname).OrDie() } @@ -38,6 +36,6 @@ func Test_ErrMessage(t *testing.T) { ret := GenTestDir() assert.Equal(t, ret.ErrMessage(), "") dname := ret.String() - WriteFile(SecureJoin(dname, "log.txt").String(), []byte("ok"), constant.DefaultFileMode).ErrMessage() + WriteFile(SecureJoin(dname, "log.txt").String(), "ok").ErrMessage() assert.Equal(t, RemoveAll(dname).ErrMessage(), "") } -- Gitee From a409a773e75d59f59af28352973f583c6d7e16e6 Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 14 Feb 2023 21:02:38 +0800 Subject: [PATCH 27/73] bugfix: fix deepCopy() of podInfo&containerInfo --- pkg/common/util/conversion.go | 18 ---- pkg/common/util/conversion_test.go | 162 +++++++++++++++++++++++++++++ pkg/core/typedef/containerinfo.go | 5 +- pkg/core/typedef/podinfo.go | 13 ++- pkg/core/typedef/podinfo_test.go | 74 +++++++++++++ pkg/core/typedef/rawpod.go | 11 ++ 6 files changed, 259 insertions(+), 24 deletions(-) create mode 100644 pkg/common/util/conversion_test.go create mode 100644 pkg/core/typedef/podinfo_test.go diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go index fc036e8..86faa0a 100644 --- a/pkg/common/util/conversion.go +++ b/pkg/common/util/conversion.go @@ -78,21 +78,3 @@ func PercentageToDecimal(num float64) float64 { const percentageofOne float64 = 100 return Div(num, percentageofOne) } - -// DeepCopy deep copy slice or map type data -func DeepCopy(value interface{}) interface{} { - if valueMap, ok := value.(map[string]interface{}); ok { - newMap := make(map[string]interface{}) - for k, v := range valueMap { - newMap[k] = DeepCopy(v) - } - return newMap - } else if valueSlice, ok := value.([]interface{}); ok { - newSlice := make([]interface{}, len(valueSlice)) - for k, v := range valueSlice { - newSlice[k] = DeepCopy(v) - } - return newSlice - } - return value -} diff --git a/pkg/common/util/conversion_test.go b/pkg/common/util/conversion_test.go new file mode 100644 index 0000000..491e47c --- /dev/null +++ b/pkg/common/util/conversion_test.go @@ -0,0 +1,162 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-14 +// Description: This file is used for testing conversion functions + +// Package util is common utilitization +package util + +import ( + "math" + "testing" +) + +// TestFormatInt64 is testcase for FormatInt64 +func TestFormatInt64(t *testing.T) { + type args struct { + n int64 + } + validNum := 100 + tests := []struct { + name string + args args + want string + }{ + { + name: "TC-convert the int 64 to string", + args: args{n: int64(validNum)}, + want: "100", + }, + { + name: "TC-convert the big int", + args: args{math.MaxInt64}, + want: "9223372036854775807", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FormatInt64(tt.args.n); got != tt.want { + t.Errorf("FormatInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestParseInt64 is testcase for ParseInt64 +func TestParseInt64(t *testing.T) { + type args struct { + str string + } + validNum := 100 + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + { + name: "TC-convert the int 64 to string", + args: args{str: "100"}, + want: int64(validNum), + }, + { + name: "TC-convert the big int", + args: args{str: "9223372036854775807"}, + want: math.MaxInt64, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseInt64(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("ParseInt64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestFormatFloat64 is testcase for FormatFloat64 +func TestFormatFloat64(t *testing.T) { + type args struct { + f float64 + } + validNum := 100.0 + tests := []struct { + name string + args args + want string + }{ + { + name: "TC-convert the float64 to string", + args: args{f: validNum}, + want: "100", + }, + { + name: "TC-convert the big float", + args: args{math.MaxFloat64}, + want: "179769313486231570000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "00000000000000000000000000000000000000000000000000000", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FormatFloat64(tt.args.f); got != tt.want { + t.Errorf("FormatFloat64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestParseFloat64 is testcase for ParseFloat64 +func TestParseFloat64(t *testing.T) { + type args struct { + str string + } + validNum := 100.0 + tests := []struct { + name string + args args + want float64 + wantErr bool + }{ + { + name: "TC-convert the string to float64", + args: args{str: "100"}, + want: validNum, + }, + { + name: "TC-convert the big float", + args: args{str: "1797693134862315700000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + + "000000000000000000000000000000000000000000000000000000000"}, + want: math.MaxFloat64, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseFloat64(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("ParseFloat64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseFloat64() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 8b52803..a43b0eb 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -20,7 +20,6 @@ import ( "strings" "sync" - "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -96,8 +95,8 @@ func fixContainerEngine(containerID string) { // DeepCopy returns deepcopy object. func (cont *ContainerInfo) DeepCopy() *ContainerInfo { copyObject := *cont - copyObject.LimitResources, _ = util.DeepCopy(cont.LimitResources).(ResourceMap) - copyObject.RequestResources, _ = util.DeepCopy(cont.RequestResources).(ResourceMap) + copyObject.LimitResources = cont.LimitResources.DeepCopy() + copyObject.RequestResources = cont.RequestResources.DeepCopy() return ©Object } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index ba73f1e..a97bcbd 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -17,7 +17,6 @@ package typedef import ( "strings" - "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -48,13 +47,21 @@ func (pod *PodInfo) DeepCopy() *PodInfo { if pod == nil { return nil } + contMap := make(map[string]*ContainerInfo, len(pod.IDContainersMap)) + for id, cont := range pod.IDContainersMap { + contMap[id] = cont.DeepCopy() + } + annoMap := make(map[string]string, len(pod.Annotations)) + for k, v := range pod.Annotations { + annoMap[k] = v + } return &PodInfo{ Name: pod.Name, UID: pod.UID, CgroupPath: pod.CgroupPath, Namespace: pod.Namespace, - Annotations: util.DeepCopy(pod.Annotations).(map[string]string), - IDContainersMap: util.DeepCopy(pod.IDContainersMap).(map[string]*ContainerInfo), + Annotations: annoMap, + IDContainersMap: contMap, } } diff --git a/pkg/core/typedef/podinfo_test.go b/pkg/core/typedef/podinfo_test.go new file mode 100644 index 0000000..92088a4 --- /dev/null +++ b/pkg/core/typedef/podinfo_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-14 +// Description: This file tests podInfo + +// Package typedef defines core struct and methods for rubik +package typedef + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" +) + +func TestPodInfo_DeepCopy(t *testing.T) { + const ( + oldPodName = "FooPod" + newPodName = "NewFooPod" + oldPodID = "testPod1" + newPodID = "newTestPod1" + oldQuota = "true" + newQuota = "false" + oldContName = "FooCon" + newContName = "NewFooPod" + oldReqCPU float64 = 1.2 + newReqCPU float64 = 2.7 + oldReqMem float64 = 500 + newReqMem float64 = 350 + contID = "testCon1" + oldLimitCPU float64 = 9.0 + oldLimitMem float64 = 300 + ) + oldPod := &PodInfo{ + Name: oldPodName, + UID: oldPodID, + Annotations: map[string]string{ + constant.QuotaAnnotationKey: oldQuota, + constant.PriorityAnnotationKey: "true", + }, + IDContainersMap: map[string]*ContainerInfo{ + contID: { + Name: oldContName, + RequestResources: ResourceMap{ResourceCPU: oldReqCPU, ResourceMem: oldReqMem}, + LimitResources: ResourceMap{ResourceCPU: 9.0, ResourceMem: 300}, + }, + }, + } + copyPod := oldPod.DeepCopy() + copyPod.Name = newContName + copyPod.UID = newPodID + copyPod.Annotations[constant.QuotaAnnotationKey] = newQuota + copyPod.IDContainersMap[contID].Name = newContName + copyPod.IDContainersMap[contID].RequestResources[ResourceCPU] = newReqCPU + copyPod.IDContainersMap[contID].RequestResources[ResourceMem] = newReqMem + + assert.Equal(t, oldPodName, oldPod.Name) + assert.Equal(t, oldPodID, oldPod.UID) + assert.Equal(t, oldContName, oldPod.IDContainersMap[contID].Name) + assert.Equal(t, oldQuota, oldPod.Annotations[constant.QuotaAnnotationKey]) + assert.Equal(t, oldReqCPU, oldPod.IDContainersMap[contID].RequestResources[ResourceCPU]) + assert.Equal(t, oldReqMem, oldPod.IDContainersMap[contID].RequestResources[ResourceMem]) + assert.Equal(t, oldLimitCPU, oldPod.IDContainersMap[contID].LimitResources[ResourceCPU]) + assert.Equal(t, oldLimitMem, oldPod.IDContainersMap[contID].LimitResources[ResourceMem]) + +} diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index b8c85a6..a96a2e1 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -197,3 +197,14 @@ func (cont *RawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { ) return iterator(&cont.spec.Resources.Requests), iterator(&cont.spec.Resources.Limits) } + +func (m ResourceMap) DeepCopy() ResourceMap { + if m == nil { + return nil + } + res := make(ResourceMap, len(m)) + for k, v := range m { + res[k] = v + } + return res +} -- Gitee From 53d02e19ff68059e23d45fe3018d4ea94323f38e Mon Sep 17 00:00:00 2001 From: vegbir Date: Wed, 15 Feb 2023 11:22:39 +0800 Subject: [PATCH 28/73] bugfix: use reflect to implement DeepCopy 1. fix bug of podInfo DeepCopy method: support nil map case 2. support easy DeepCopy --- pkg/common/util/conversion.go | 30 ++++++++++++++++++++++++++++++ pkg/common/util/conversion_test.go | 25 +++++++++++++++++++++++++ pkg/core/typedef/podinfo.go | 22 ++++++++++++++++------ pkg/core/typedef/podinfo_test.go | 10 ++++++++++ test/try/pod.go | 18 +++++++++--------- test/try/pod_test.go | 1 + 6 files changed, 91 insertions(+), 15 deletions(-) diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go index 86faa0a..b686872 100644 --- a/pkg/common/util/conversion.go +++ b/pkg/common/util/conversion.go @@ -17,6 +17,7 @@ package util import ( "bufio" "fmt" + "reflect" "strconv" "strings" ) @@ -78,3 +79,32 @@ func PercentageToDecimal(num float64) float64 { const percentageofOne float64 = 100 return Div(num, percentageofOne) } + +// DeepCopy deep copy slice or map data with basic type +// DeepCopy is a simple deep copy function that only supports copying of basic types of maps and slices, +// such as map[string]string, []int, etc., and does not support nesting. +// **Since the reflection mechanism with poor performance is used, +// please use this function with caution after balancing performance and ease of use** +func DeepCopy(value interface{}) interface{} { + defer func() { + if err := recover(); err != nil { + fmt.Printf("err: %v\n", err) + } + }() + typ := reflect.TypeOf(value) + val := reflect.ValueOf(value) + switch typ.Kind() { + case reflect.Map: + newMap := reflect.MakeMapWithSize(typ, val.Len()) + it := val.MapRange() + for it.Next() { + newMap.SetMapIndex(it.Key(), DeepCopy(it.Value()).(reflect.Value)) + } + return newMap.Interface() + case reflect.Slice: + newSlice := reflect.MakeSlice(typ, val.Len(), val.Cap()) + reflect.Copy(newSlice, val) + return newSlice.Interface() + } + return value +} diff --git a/pkg/common/util/conversion_test.go b/pkg/common/util/conversion_test.go index 491e47c..29b7e8c 100644 --- a/pkg/common/util/conversion_test.go +++ b/pkg/common/util/conversion_test.go @@ -17,6 +17,8 @@ package util import ( "math" "testing" + + "github.com/stretchr/testify/assert" ) // TestFormatInt64 is testcase for FormatInt64 @@ -160,3 +162,26 @@ func TestParseFloat64(t *testing.T) { }) } } + +// TestDeepCopy tests DeepCopy +func TestDeepCopy(t *testing.T) { + oldMap := map[string]string{ + "a": "1", + "b": "2", + } + newMap := DeepCopy(oldMap).(map[string]string) + newMap["a"] = "3" + newMap["b"] = "4" + assert.Equal(t, oldMap["a"], "1") + assert.Equal(t, oldMap["b"], "2") + assert.Equal(t, newMap["a"], "3") + assert.Equal(t, newMap["b"], "4") + + oldSlice := []string{"a", "b", "c"} + newSlice := DeepCopy(oldSlice).([]string) + for i, _ := range newSlice { + newSlice[i] += "z" + } + assert.Equal(t, oldSlice, []string{"a", "b", "c"}) + assert.Equal(t, newSlice, []string{"az", "bz", "cz"}) +} diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index a97bcbd..b622715 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -47,14 +47,24 @@ func (pod *PodInfo) DeepCopy() *PodInfo { if pod == nil { return nil } - contMap := make(map[string]*ContainerInfo, len(pod.IDContainersMap)) - for id, cont := range pod.IDContainersMap { - contMap[id] = cont.DeepCopy() + var ( + contMap map[string]*ContainerInfo + annoMap map[string]string + ) + // nil is different from empty value in golang + if pod.IDContainersMap != nil { + contMap = make(map[string]*ContainerInfo, len(pod.IDContainersMap)) + for id, cont := range pod.IDContainersMap { + contMap[id] = cont.DeepCopy() + } } - annoMap := make(map[string]string, len(pod.Annotations)) - for k, v := range pod.Annotations { - annoMap[k] = v + if pod.Annotations != nil { + annoMap = make(map[string]string, len(pod.Annotations)) + for k, v := range pod.Annotations { + annoMap[k] = v + } } + return &PodInfo{ Name: pod.Name, UID: pod.UID, diff --git a/pkg/core/typedef/podinfo_test.go b/pkg/core/typedef/podinfo_test.go index 92088a4..26e314f 100644 --- a/pkg/core/typedef/podinfo_test.go +++ b/pkg/core/typedef/podinfo_test.go @@ -71,4 +71,14 @@ func TestPodInfo_DeepCopy(t *testing.T) { assert.Equal(t, oldLimitCPU, oldPod.IDContainersMap[contID].LimitResources[ResourceCPU]) assert.Equal(t, oldLimitMem, oldPod.IDContainersMap[contID].LimitResources[ResourceMem]) + oldNilMapPod := &PodInfo{ + Name: oldPodName, + UID: oldPodID, + Annotations: map[string]string{ + constant.QuotaAnnotationKey: oldQuota, + }, + } + copyPod = oldNilMapPod.DeepCopy() + assert.Equal(t, copyPod, oldNilMapPod) + } diff --git a/test/try/pod.go b/test/try/pod.go index 05b7a26..6150cf2 100644 --- a/test/try/pod.go +++ b/test/try/pod.go @@ -193,15 +193,15 @@ func (pod *FakePod) DeepCopy() *FakePod { if pod == nil || pod.PodInfo == nil { return nil } + var copyKeys map[*cgroup.Key]string + if pod.Keys != nil { + copyKeys = make(map[*cgroup.Key]string, len(pod.Keys)) + for k, v := range pod.Keys { + copyKeys[k] = v + } + } return &FakePod{ - Keys: util.DeepCopy(pod.Keys).(map[*cgroup.Key]string), - PodInfo: &typedef.PodInfo{ - Name: pod.Name, - UID: pod.UID, - CgroupPath: pod.CgroupPath, - Namespace: pod.Namespace, - Annotations: util.DeepCopy(pod.Annotations).(map[string]string), - IDContainersMap: util.DeepCopy(pod.IDContainersMap).(map[string]*typedef.ContainerInfo), - }, + Keys: copyKeys, + PodInfo: pod.PodInfo.DeepCopy(), } } diff --git a/test/try/pod_test.go b/test/try/pod_test.go index bde226b..b65ce2d 100644 --- a/test/try/pod_test.go +++ b/test/try/pod_test.go @@ -254,6 +254,7 @@ func TestFakePod_DeepCopy(t *testing.T) { Keys: tt.fields.Keys, } if got := pod.DeepCopy(); !reflect.DeepEqual(got, tt.want) { + assert.Equal(t, got, tt.want) t.Errorf("FakePod.DeepCopy() = %v, want %v", got, tt.want) } }) -- Gitee From e1e4327c8c89cab40c05715639cdde9cbca5625e Mon Sep 17 00:00:00 2001 From: wujing Date: Wed, 15 Feb 2023 11:00:47 +0800 Subject: [PATCH 29/73] unified unit tests style Signed-off-by: wujing --- pkg/services/qos/qos_test.go | 207 ++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 103 deletions(-) diff --git a/pkg/services/qos/qos_test.go b/pkg/services/qos/qos_test.go index 8128c37..92106a1 100644 --- a/pkg/services/qos/qos_test.go +++ b/pkg/services/qos/qos_test.go @@ -54,59 +54,60 @@ var getCommonField = func(subSys []string) fields { Config: Config{SubSys: subSys}, } } -var addFuncTC = []test{ - { - name: "TC1-set offline pod qos ok", - fields: getCommonField([]string{"cpu", "memory"}), - args: args{ - new: try.GenFakeOfflinePod(map[*cgroup.Key]string{ - supportCgroupTypes["cpu"]: "0", - supportCgroupTypes["memory"]: "0", - }), - }, - }, - { - name: "TC2-set online pod qos ok", - fields: getCommonField([]string{"cpu", "memory"}), - args: args{ - new: try.GenFakeOnlinePod(map[*cgroup.Key]string{ - supportCgroupTypes["cpu"]: "0", - supportCgroupTypes["memory"]: "0", - }).WithContainers(3), + +func TestQoS_AddFunc(t *testing.T) { + var addFuncTC = []test{ + { + name: "TC1-set offline pod qos ok", + fields: getCommonField([]string{"cpu", "memory"}), + args: args{ + new: try.GenFakeOfflinePod(map[*cgroup.Key]string{ + supportCgroupTypes["cpu"]: "0", + supportCgroupTypes["memory"]: "0", + }), + }, }, - }, - { - name: "TC3-empty pod info", - fields: getCommonField([]string{"cpu", "memory"}), - wantErr: true, - }, - { - name: "TC4-invalid annotation key", - fields: getCommonField([]string{"cpu"}), - args: args{ - new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), + { + name: "TC2-set online pod qos ok", + fields: getCommonField([]string{"cpu", "memory"}), + args: args{ + new: try.GenFakeOnlinePod(map[*cgroup.Key]string{ + supportCgroupTypes["cpu"]: "0", + supportCgroupTypes["memory"]: "0", + }).WithContainers(3), + }, }, - preHook: func(pod *try.FakePod) *try.FakePod { - newPod := pod.DeepCopy() - newPod.Annotations["undefine"] = "true" - return newPod + { + name: "TC3-empty pod info", + fields: getCommonField([]string{"cpu", "memory"}), + wantErr: true, }, - }, - { - name: "TC5-invalid annotation value", - fields: getCommonField([]string{"cpu"}), - args: args{ - new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), + { + name: "TC4-invalid annotation key", + fields: getCommonField([]string{"cpu"}), + args: args{ + new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), + }, + preHook: func(pod *try.FakePod) *try.FakePod { + newPod := pod.DeepCopy() + newPod.Annotations["undefine"] = "true" + return newPod + }, }, - preHook: func(pod *try.FakePod) *try.FakePod { - newPod := pod.DeepCopy() - newPod.Annotations[constant.PriorityAnnotationKey] = "undefine" - return newPod + { + name: "TC5-invalid annotation value", + fields: getCommonField([]string{"cpu"}), + args: args{ + new: try.GenFakeBestEffortPod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}), + }, + preHook: func(pod *try.FakePod) *try.FakePod { + newPod := pod.DeepCopy() + newPod.Annotations[constant.PriorityAnnotationKey] = "undefine" + return newPod + }, }, - }, -} + } -func TestQoS_AddFunc(t *testing.T) { for _, tt := range addFuncTC { t.Run(tt.name, func(t *testing.T) { q := &QoS{ @@ -128,44 +129,44 @@ func TestQoS_AddFunc(t *testing.T) { } } -var updateFuncTC = []test{ - { - name: "TC1-online to offline", - fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}).WithContainers(3)}, - preHook: func(pod *try.FakePod) *try.FakePod { - newPod := pod.DeepCopy() - // TODO: need fix pod.DeepCopy - newAnnotation := make(map[string]string, 0) - newAnnotation[constant.PriorityAnnotationKey] = "true" - newPod.Annotations = newAnnotation - return newPod +func TestQoS_UpdateFunc(t *testing.T) { + var updateFuncTC = []test{ + { + name: "TC1-online to offline", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}).WithContainers(3)}, + preHook: func(pod *try.FakePod) *try.FakePod { + newPod := pod.DeepCopy() + // TODO: need fix pod.DeepCopy + newAnnotation := make(map[string]string, 0) + newAnnotation[constant.PriorityAnnotationKey] = "true" + newPod.Annotations = newAnnotation + return newPod + }, }, - }, - { - name: "TC2-offline to online", - fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOfflinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, - preHook: func(pod *try.FakePod) *try.FakePod { - newPod := pod.DeepCopy() - newAnnotation := make(map[string]string, 0) - newAnnotation[constant.PriorityAnnotationKey] = "false" - newPod.Annotations = newAnnotation - return newPod + { + name: "TC2-offline to online", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOfflinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, + preHook: func(pod *try.FakePod) *try.FakePod { + newPod := pod.DeepCopy() + newAnnotation := make(map[string]string, 0) + newAnnotation[constant.PriorityAnnotationKey] = "false" + newPod.Annotations = newAnnotation + return newPod + }, + wantErr: true, }, - wantErr: true, - }, - { - name: "TC3-online to online", - fields: getCommonField([]string{"cpu"}), - args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, - preHook: func(pod *try.FakePod) *try.FakePod { - return pod.DeepCopy() + { + name: "TC3-online to online", + fields: getCommonField([]string{"cpu"}), + args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"})}, + preHook: func(pod *try.FakePod) *try.FakePod { + return pod.DeepCopy() + }, }, - }, -} + } -func TestQoS_UpdateFunc(t *testing.T) { for _, tt := range updateFuncTC { t.Run(tt.name, func(t *testing.T) { q := &QoS{ @@ -185,31 +186,31 @@ func TestQoS_UpdateFunc(t *testing.T) { } } -var validateTC = []test{ - { - name: "TC1-normal config", - fields: fields{ - Name: "qos", - Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), - Config: Config{SubSys: []string{"cpu", "memory"}}, +func TestQoS_Validate(t *testing.T) { + var validateTC = []test{ + { + name: "TC1-normal config", + fields: fields{ + Name: "qos", + Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), + Config: Config{SubSys: []string{"cpu", "memory"}}, + }, }, - }, - { - name: "TC2-abnormal config", - fields: fields{ - Name: "undefine", - Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), - Config: Config{SubSys: []string{"undefine"}}, + { + name: "TC2-abnormal config", + fields: fields{ + Name: "undefine", + Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), + Config: Config{SubSys: []string{"undefine"}}, + }, + wantErr: true, }, - wantErr: true, - }, - { - name: "TC3-empty config", - wantErr: true, - }, -} + { + name: "TC3-empty config", + wantErr: true, + }, + } -func TestQoS_Validate(t *testing.T) { for _, tt := range validateTC { t.Run(tt.name, func(t *testing.T) { q := &QoS{ -- Gitee From 37751abfbe7d6fecaca716e01f2fb1800c6d60e7 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 17 Feb 2023 17:01:47 +0800 Subject: [PATCH 30/73] feature: support apiViewer for podmanager & service SetupLog function 1. Add service log selection: 1) serviceManager supports calling the SetupLog function provided by the service to initialize the service log. 2) log provides the EmptyLog class (api.Logger interface) for placeholders. 2. serviceManager supports the addition of both persistent and one-time services, and ensures that only Prestart and Terminate are executed. 3. Add and modify some tests 4. Podmanager implements api.Viewer --- pkg/api/api.go | 7 ++- pkg/common/log/log.go | 15 +++++ pkg/config/config_test.go | 2 +- pkg/podmanager/podcache.go | 11 ++++ pkg/podmanager/podmanager.go | 34 ++++++++++ pkg/podmanager/podmanager_test.go | 100 ++++++++++++++++++++++++++++++ pkg/services/servicemanager.go | 34 +++++++--- 7 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 pkg/podmanager/podmanager_test.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 4768e64..da4228e 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -54,10 +54,13 @@ type PersistentService interface { Run(ctx context.Context) } +// ListOption is for filtering podInfo +type ListOption func(pi *typedef.PodInfo) bool + // Viewer collect on/offline pods info type Viewer interface { - ListOnlinePods() ([]*typedef.PodInfo, error) - ListOfflinePods() ([]*typedef.PodInfo, error) + ListContainersWithOptions(options ...ListOption) map[string]*typedef.ContainerInfo + ListPodsWithOptions(options ...ListOption) map[string]*typedef.PodInfo } // Publisher is a generic interface for Observables diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index 85e9e61..02b12ee 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -305,3 +305,18 @@ func (e *Entry) Errorf(f string, args ...interface{}) { } output(e.level(logError), f, args...) } + +// EmptyLog is an empty log structure without any log processing +type EmptyLog struct{} + +// Warnf write logs +func (e *EmptyLog) Warnf(f string, args ...interface{}) {} + +// Infof write logs +func (e *EmptyLog) Infof(f string, args ...interface{}) {} + +// Debugf write verbose logs +func (e *EmptyLog) Debugf(f string, args ...interface{}) {} + +// Errorf write error logs +func (e *EmptyLog) Errorf(f string, args ...interface{}) {} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1a16be0..dbbd2e4 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -57,7 +57,7 @@ var rubikConfig string = ` } ` -func TestServices(t *testing.T) { +func TestNewConfig(t *testing.T) { if !util.PathExist(constant.TmpTestDir) { if err := os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode); err != nil { assert.NoError(t, err) diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go index 7179f92..fced314 100644 --- a/pkg/podmanager/podcache.go +++ b/pkg/podmanager/podcache.go @@ -103,3 +103,14 @@ func (cache *podCache) substitute(pods []*typedef.PodInfo) { log.Debugf("substituting pod %v", pod.UID) } } + +// listPod returns the deepcopy object of all pod +func (cache *podCache) listPod() map[string]*typedef.PodInfo { + res := make(map[string]*typedef.PodInfo, len(cache.Pods)) + cache.RLock() + for id, pi := range cache.Pods { + res[id] = pi.DeepCopy() + } + cache.RUnlock() + return res +} diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index 3fae9e1..3dc9e67 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -226,3 +226,37 @@ func (manager *PodManager) ListOfflinePods() ([]*typedef.PodInfo, error) { func (manager *PodManager) ListOnlinePods() ([]*typedef.PodInfo, error) { return nil, nil } + +func withOption(pi *typedef.PodInfo, opts []api.ListOption) bool { + for _, opt := range opts { + if !opt(pi) { + return false + } + } + return true +} + +// ListContainersWithOptions filters and returns deep copy objects of all containers +func (manager *PodManager) ListContainersWithOptions(options ...api.ListOption) map[string]*typedef.ContainerInfo { + conts := make(map[string]*typedef.ContainerInfo) + for _, pod := range manager.ListPodsWithOptions(options...) { + for _, ci := range pod.IDContainersMap { + conts[ci.ID] = ci + } + } + return conts +} + +// ListPodsWithOptions filters and returns deep copy objects of all pods +func (manager *PodManager) ListPodsWithOptions(options ...api.ListOption) map[string]*typedef.PodInfo { + // already deep copied + allPods := manager.pods.listPod() + pods := make(map[string]*typedef.PodInfo, len(allPods)) + for _, pod := range allPods { + if !withOption(pod, options) { + continue + } + pods[pod.UID] = pod + } + return pods +} diff --git a/pkg/podmanager/podmanager_test.go b/pkg/podmanager/podmanager_test.go new file mode 100644 index 0000000..eab7fa0 --- /dev/null +++ b/pkg/podmanager/podmanager_test.go @@ -0,0 +1,100 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-17 +// Description: This file is used for testing podmanager + +package podmanager + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef" +) + +func TestPodManager_ListContainersWithOptions(t *testing.T) { + var ( + cont1 = &typedef.ContainerInfo{ + ID: "testCon1", + } + cont2 = &typedef.ContainerInfo{ + ID: "testCon2", + } + cont3 = &typedef.ContainerInfo{ + ID: "testCon3", + } + ) + + type fields struct { + pods *podCache + } + type args struct { + options []api.ListOption + } + tests := []struct { + name string + fields fields + args args + want map[string]*typedef.ContainerInfo + }{ + // TODO: Add test cases. + { + name: "TC1-filter priority container", + args: args{ + []api.ListOption{ + func(pi *typedef.PodInfo) bool { + return pi.Annotations[constant.PriorityAnnotationKey] == "true" + }, + }, + }, + fields: fields{ + pods: &podCache{ + Pods: map[string]*typedef.PodInfo{ + "testPod1": { + UID: "testPod1", + IDContainersMap: map[string]*typedef.ContainerInfo{ + cont1.ID: cont1, + cont2.ID: cont2, + }, + Annotations: map[string]string{ + constant.PriorityAnnotationKey: "true", + }, + }, + "testPod2": { + IDContainersMap: map[string]*typedef.ContainerInfo{ + cont3.ID: cont3, + }, + }, + }, + }, + }, + want: map[string]*typedef.ContainerInfo{ + cont1.ID: cont1, + cont2.ID: cont2, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := &PodManager{ + pods: tt.fields.pods, + } + if got := manager.ListContainersWithOptions(tt.args.options...); !reflect.DeepEqual(got, tt.want) { + assert.Equal(t, tt.want, got) + t.Errorf("PodManager.ListContainersWithOptions() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 834aa89..3e2b502 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -94,11 +94,13 @@ func (manager *ServiceManager) AddRunningService(name string, service interface{ return fmt.Errorf("service name conflict: %s", name) } - if !manager.tryAddService(name, service) && !manager.tryAddPersistentService(name, service) { - return fmt.Errorf("invalid service %s (type %T)", name, service) + addService := manager.tryAddService(name, service) + addPersistentService := manager.tryAddPersistentService(name, service) + if addPersistentService || addService { + log.Debugf("pre-start service %s", name) + return nil } - log.Debugf("pre-start service %s", name) - return nil + return fmt.Errorf("invalid service %s (type %T)", name, service) } // HandleEvent is used to handle PodInfo events pushed by the publisher @@ -164,13 +166,21 @@ func (manager *ServiceManager) terminatingRunningServices(err error) error { // SetLoggerOnService assigns a value to the variable Log member if there is a Log field func SetLoggerOnService(value interface{}, logger api.Logger) bool { - // look for a member variable named Log + // 1. call the SetupLog function to set up the log + method := reflect.ValueOf(value).MethodByName("SetupLog") + if method.IsValid() && !method.IsZero() && !method.IsNil() { + method.Call([]reflect.Value{reflect.ValueOf(logger)}) + return true + } + + // 2. look for a member variable named Log field := reflect.ValueOf(value).Elem().FieldByName("Log") - if !field.IsValid() || !field.CanSet() || field.Type().String() != "api.Logger" { - return false + if field.IsValid() && field.CanSet() && field.Type().String() == "api.Logger" { + field.Set(reflect.ValueOf(logger)) + return true } - field.Set(reflect.ValueOf(logger)) - return true + + return false } // Setup pre-starts services, such as preparing the environment, etc. @@ -179,6 +189,7 @@ func (manager *ServiceManager) Setup(v api.Viewer) error { if v == nil { return nil } + preStarted := make(map[string]struct{}, 0) manager.Viewer = v manager.TerminateFuncs = make(map[string]Terminator) setupFunc := func(id string, s interface{}) error { @@ -191,9 +202,14 @@ func (manager *ServiceManager) Setup(v api.Viewer) error { if !ok { return nil } + // execute the prestart method only once if the service is both a persistent service and a service + if _, ok := preStarted[id]; ok { + return nil + } if err := p.PreStart(manager.Viewer); err != nil { return err } + preStarted[id] = struct{}{} return nil } -- Gitee From c0eff23b2dbf66454b63c9885a5461e4294c4a3d Mon Sep 17 00:00:00 2001 From: vegbir Date: Mon, 20 Feb 2023 10:58:51 +0800 Subject: [PATCH 31/73] feature: support quotaTurbo 1. We have implemented an event-driven quota Turbo framework, which is a method of dynamically adjusting business CPU quotas by considering the overall CPU resource level and business resource requirements. 1) NodaData stores the whole machine information, including node container information, user configuration, etc. 2) EventDriver performs slow raising, fast falling, and slow callback operations based on the CPU water level. 3) QuotaTurbo is responsible for operating the entire lifecycle. 2. We added some DT test cases --- pkg/services/quotaturbo/cpu_test.go | 57 ++ pkg/services/quotaturbo/cpuquota.go | 7 +- pkg/services/quotaturbo/cpuquota_test.go | 236 ++++++++ pkg/services/quotaturbo/data.go | 216 +++++++ pkg/services/quotaturbo/data_test.go | 232 ++++++++ pkg/services/quotaturbo/driverevent.go | 196 +++++++ pkg/services/quotaturbo/driverevent_test.go | 593 ++++++++++++++++++++ pkg/services/quotaturbo/quotaturbo.go | 108 ++++ 8 files changed, 1641 insertions(+), 4 deletions(-) create mode 100644 pkg/services/quotaturbo/cpu_test.go create mode 100644 pkg/services/quotaturbo/cpuquota_test.go create mode 100644 pkg/services/quotaturbo/data.go create mode 100644 pkg/services/quotaturbo/data_test.go create mode 100644 pkg/services/quotaturbo/driverevent.go create mode 100644 pkg/services/quotaturbo/driverevent_test.go create mode 100644 pkg/services/quotaturbo/quotaturbo.go diff --git a/pkg/services/quotaturbo/cpu_test.go b/pkg/services/quotaturbo/cpu_test.go new file mode 100644 index 0000000..ead2d05 --- /dev/null +++ b/pkg/services/quotaturbo/cpu_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for testing cpu.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestCalculateUtils tests calculateUtils +func TestCalculateUtils(t *testing.T) { + var ( + n1 float64 = 1 + n2 float64 = 2 + n3 float64 = 3 + n4 float64 = 4 + ) + + var ( + t1 = ProcStat{ + total: n2, + busy: n1, + } + t2 = ProcStat{ + total: n4, + busy: n2, + } + t3 = ProcStat{ + total: n3, + busy: n3, + } + ) + // normal return result + const ( + util float64 = 50 + minimumUtilization float64 = 0 + maximumUtilization float64 = 100 + ) + assert.Equal(t, util, calculateUtils(t1, t2)) + // busy errors + assert.Equal(t, minimumUtilization, calculateUtils(t2, t1)) + // total errors + assert.Equal(t, maximumUtilization, calculateUtils(t2, t3)) +} diff --git a/pkg/services/quotaturbo/cpuquota.go b/pkg/services/quotaturbo/cpuquota.go index ab89af9..08402a3 100644 --- a/pkg/services/quotaturbo/cpuquota.go +++ b/pkg/services/quotaturbo/cpuquota.go @@ -1,4 +1,4 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. // rubik licensed under the Mulan PSL v2. // You can use this software according to the terms and conditions of the Mulan PSL v2. // You may obtain a copy of Mulan PSL v2 at: @@ -8,7 +8,7 @@ // PURPOSE. // See the Mulan PSL v2 for more details. // Author: Jiaqi Yang -// Create: 2022-03-18 +// Date: 2023-02-20 // Description: cpu container cpu quota data and methods // Package quotaturbo is for Quota Turbo @@ -19,7 +19,6 @@ import ( "path" "time" - "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" @@ -28,7 +27,7 @@ import ( const ( // numberOfRestrictedCycles is the number of periods in which the quota limits the CPU usage. numberOfRestrictedCycles = 60 - // default cfs_period_us = 100ms + // The default value of the cfs_period_us file is 100ms defaultCFSPeriodUs int64 = 100000 ) diff --git a/pkg/services/quotaturbo/cpuquota_test.go b/pkg/services/quotaturbo/cpuquota_test.go new file mode 100644 index 0000000..89584c1 --- /dev/null +++ b/pkg/services/quotaturbo/cpuquota_test.go @@ -0,0 +1,236 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for test cpu_quota + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "path" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +// TestSaveQuota tests SaveQuota of CPUQuota +func TestSaveQuota(t *testing.T) { + const ( + largerQuota = "200000" + largerQuotaVal int64 = 200000 + smallerQuota = "100000" + smallerQuotaVal int64 = 100000 + unlimitedQuota = "-1" + unlimitedQuotaVal int64 = -1 + periodUs = "100000" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + ) + cgroup.InitMountDir(constant.TmpTestDir) + var ( + cq = &CPUQuota{ + ContainerInfo: &typedef.ContainerInfo{ + Name: "Foo", + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + }, + } + contPath = cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath, "") + podPeriodPath = filepath.Join(filepath.Dir(contPath), cpuPeriodFile) + podQuotaPath = filepath.Join(filepath.Dir(contPath), cpuQuotaFile) + contPeriodPath = filepath.Join(contPath, cpuPeriodFile) + contQuotaPath = filepath.Join(contPath, cpuQuotaFile) + ) + + try.RemoveAll(constant.TmpTestDir) + defer try.RemoveAll(constant.TmpTestDir) + + assertValue := func(t *testing.T, paths []string, value string) { + for _, p := range paths { + data, err := util.ReadFile(p) + assert.NoError(t, err) + assert.Equal(t, value, strings.TrimSpace(string(data))) + } + } + + // case1: Only one pod or container exists at a time + func() { + try.MkdirAll(contPath, constant.DefaultDirMode) + defer try.RemoveAll(path.Dir(contPath)) + // None of the paths exist + cq.nextQuota = largerQuotaVal + cq.curQuota = smallerQuotaVal + assert.Error(t, cq.SaveQuota()) + cq.nextQuota = smallerQuotaVal + cq.curQuota = largerQuotaVal + assert.Error(t, cq.SaveQuota()) + + // only Pod path existed + cq.nextQuota = largerQuotaVal + cq.curQuota = smallerQuotaVal + try.WriteFile(podQuotaPath, unlimitedQuota) + try.WriteFile(podPeriodPath, periodUs) + try.RemoveAll(contQuotaPath) + try.RemoveAll(contPeriodPath) + assert.Error(t, cq.SaveQuota()) + + // only container path existed + cq.nextQuota = smallerQuotaVal + cq.curQuota = largerQuotaVal + try.RemoveAll(podQuotaPath) + try.RemoveAll(podPeriodPath) + try.WriteFile(contQuotaPath, largerQuota) + try.WriteFile(contPeriodPath, periodUs) + assert.Error(t, cq.SaveQuota()) + + }() + + // case2: success + func() { + try.MkdirAll(contPath, constant.DefaultDirMode) + defer try.RemoveAll(path.Dir(contPath)) + + try.WriteFile(podQuotaPath, smallerQuota) + try.WriteFile(podPeriodPath, periodUs) + try.WriteFile(contQuotaPath, smallerQuota) + try.WriteFile(contPeriodPath, periodUs) + + // delta > 0 + cq.nextQuota = largerQuotaVal + cq.curQuota = smallerQuotaVal + assert.NoError(t, cq.SaveQuota()) + assertValue(t, []string{podQuotaPath, contQuotaPath}, largerQuota) + // delta < 0 + cq.nextQuota = smallerQuotaVal + cq.curQuota = largerQuotaVal + assert.NoError(t, cq.SaveQuota()) + assertValue(t, []string{podQuotaPath, contQuotaPath}, smallerQuota) + }() + + cgroup.InitMountDir(constant.DefaultCgroupRoot) +} + +// TestNewCPUQuota tests NewCPUQuota +func TestNewCPUQuota(t *testing.T) { + const ( + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + validStat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + throttleTime int64 = 1 + quota = "200000" + quotaValue int64 = 200000 + period = "100000" + periodValue int64 = 100000 + usage = "1234567" + usageValue int64 = 1234567 + ) + cgroup.InitMountDir(constant.TmpTestDir) + + var ( + ci = &typedef.ContainerInfo{ + Name: "Foo", + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + } + contPath = cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") + contPeriodPath = filepath.Join(contPath, cpuPeriodFile) + contQuotaPath = filepath.Join(contPath, cpuQuotaFile) + contUsagePath = cgroup.AbsoluteCgroupPath("cpuacct", ci.CgroupPath, cpuUsageFile) + contStatPath = filepath.Join(contPath, cpuStatFile) + ) + + try.RemoveAll(constant.TmpTestDir) + try.MkdirAll(contPath, constant.DefaultDirMode) + try.MkdirAll(path.Dir(contUsagePath), constant.DefaultDirMode) + defer try.RemoveAll(constant.TmpTestDir) + + // absent of period file + try.RemoveAll(contPeriodPath) + _, err := NewCPUQuota(ci) + assert.Error(t, err, "should lacking of period file") + try.WriteFile(contPeriodPath, period) + + // absent of throttle file + try.RemoveAll(contStatPath) + _, err = NewCPUQuota(ci) + assert.Error(t, err, "should lacking of throttle file") + try.WriteFile(contStatPath, validStat) + + // absent of quota file + try.RemoveAll(contQuotaPath) + _, err = NewCPUQuota(ci) + assert.Error(t, err, "should lacking of quota file") + try.WriteFile(contQuotaPath, quota) + + // absent of usage file + try.RemoveAll(contUsagePath) + _, err = NewCPUQuota(ci) + assert.Error(t, err, "should lacking of usage file") + try.WriteFile(contUsagePath, usage) + + cq, err := NewCPUQuota(ci) + assert.NoError(t, err) + assert.Equal(t, usageValue, cq.cpuUsages[0].usage) + assert.Equal(t, quotaValue, cq.curQuota) + assert.Equal(t, periodValue, cq.period) + + cu := make([]cpuUsage, numberOfRestrictedCycles) + for i := 0; i < numberOfRestrictedCycles; i++ { + cu[i] = cpuUsage{} + } + cq.cpuUsages = cu + assert.NoError(t, cq.updateUsage()) + cgroup.InitMountDir(constant.DefaultCgroupRoot) +} + +var containerInfos = []*typedef.ContainerInfo{ + { + Name: "Foo", + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, + RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, + }, + { + Name: "Bar", + ID: "testCon2", + CgroupPath: "kubepods/testPod2/testCon2", + LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 3}, + RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 3}, + }, + { + Name: "Biu", + ID: "testCon3", + CgroupPath: "kubepods/testPod3/testCon3", + LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, + RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, + }, + { + Name: "Pah", + ID: "testCon4", + CgroupPath: "kubepods/testPod4/testCon4", + LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 0}, + RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 0}, + }, +} diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go new file mode 100644 index 0000000..99eac56 --- /dev/null +++ b/pkg/services/quotaturbo/data.go @@ -0,0 +1,216 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: QuotaTurbo driver interface + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "runtime" + "sync" + "time" + + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +const ( + defaultHightWaterMark = 60 + defaultAlarmWaterMark = 80 + defaultQuotaTurboSyncInterval = 100 +) + +// cpuUtil is used to store the cpu usage at a specific time +type cpuUtil struct { + timestamp int64 + util float64 +} + +// Config defines configuration of QuotaTurbo +type Config struct { + HighWaterMark int `json:"highWaterMark,omitempty"` + AlarmWaterMark int `json:"alarmWaterMark,omitempty"` + SyncInterval int `json:"syncInterval,omitempty"` +} + +// NodeData is the information of node/containers obtained for quotaTurbo +type NodeData struct { + // configuration of the QuotaTurbo + *Config + // ensuring Concurrent Sequential Consistency + sync.RWMutex + // map between container IDs and container CPU quota + containers map[string]*CPUQuota + // cpu utilization sequence for N consecutive cycles + cpuUtils []cpuUtil + // /proc/stat of the previous period + lastProcStat ProcStat +} + +// NewNodeData returns a pointer to NodeData +func NewNodeData() *NodeData { + return &NodeData{ + Config: &Config{ + HighWaterMark: defaultHightWaterMark, + AlarmWaterMark: defaultAlarmWaterMark, + SyncInterval: defaultQuotaTurboSyncInterval, + }, + lastProcStat: ProcStat{ + total: -1, + busy: -1, + }, + containers: make(map[string]*CPUQuota, 0), + cpuUtils: make([]cpuUtil, 0), + } + +} + +// getLastCPUUtil obtain the latest cpu utilization +func (d *NodeData) getLastCPUUtil() float64 { + if len(d.cpuUtils) == 0 { + return 0 + } + return d.cpuUtils[len(d.cpuUtils)-1].util +} + +// removeContainer deletes the list of pods that do not need to be adjusted. +func (d *NodeData) removeContainer(id string) error { + d.RLock() + cq, ok := d.containers[id] + d.RUnlock() + if !ok { + return nil + } + safeDel := func(id string) error { + d.Lock() + delete(d.containers, id) + d.Unlock() + return nil + } + + if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath, "")) { + return safeDel(id) + } + // cq.Period ranges from 1000(us) to 1000000(us) and does not overflow. + origin := int64(cq.LimitResources[typedef.ResourceCPU] * float64(cq.period)) + if err := cq.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(origin)); err != nil { + return fmt.Errorf("fail to recover cpu.cfs_quota_us for container %s : %v", cq.Name, err) + } + return safeDel(id) +} + +// updateCPUUtils updates the cpu usage of a node +func (d *NodeData) updateCPUUtils() error { + var ( + curUtil float64 = 0 + index = 0 + t cpuUtil + ) + ps, err := getProcStat() + if err != nil { + return err + } + if d.lastProcStat.total >= 0 { + curUtil = calculateUtils(d.lastProcStat, ps) + } + d.lastProcStat = ps + cur := time.Now().UnixNano() + d.cpuUtils = append(d.cpuUtils, cpuUtil{ + timestamp: cur, + util: curUtil, + }) + // retain utilization data for only one minute + const minuteTimeDelta = int64(time.Minute) + for index, t = range d.cpuUtils { + if cur-t.timestamp <= minuteTimeDelta { + break + } + } + if index > 0 { + d.cpuUtils = d.cpuUtils[index:] + } + return nil +} + +// UpdateClusterContainers synchronizes data from given containers +func (d *NodeData) UpdateClusterContainers(conts map[string]*typedef.ContainerInfo) error { + var toBeDeletedList []string + for _, cont := range conts { + old, ok := d.containers[cont.ID] + // delete or skip containers that do not meet the conditions. + if !isAdjustmentAllowed(cont) { + if ok { + toBeDeletedList = append(toBeDeletedList, cont.ID) + } + continue + } + // add container + if !ok { + log.Debugf("add container %v (name : %v)", cont.ID, cont.Name) + if newQuota, err := NewCPUQuota(cont); err != nil { + log.Errorf("failed to create cpu quota object %v, error: %v", cont.Name, err) + } else { + d.containers[cont.ID] = newQuota + } + continue + } + // update data container in the quotaTurboList + old.ContainerInfo = cont + if err := old.updatePeriod(); err != nil { + log.Errorf("fail to update period : %v", err) + } + if err := old.updateThrottle(); err != nil { + log.Errorf("fail to update throttle time : %v", err) + } + if err := old.updateQuota(); err != nil { + log.Errorf("fail to update quota : %v", err) + } + if err := old.updateUsage(); err != nil { + log.Errorf("fail to update cpu usage : %v", err) + } + } + // non trust list container + for id := range d.containers { + // the container is removed from the trust list + if _, ok := conts[id]; !ok { + toBeDeletedList = append(toBeDeletedList, id) + } + } + + for _, id := range toBeDeletedList { + if err := d.removeContainer(id); err != nil { + log.Errorf(err.Error()) + } + } + return nil +} + +// isAdjustmentAllowed judges whether quota adjustment is allowed +func isAdjustmentAllowed(ci *typedef.ContainerInfo) bool { + // 1. containers whose cgroup path does not exist are not considered. + if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "")) { + return false + } + + // 2. abnormal CPULimit + // a). containers that do not limit the quota + // b). CPULimit = 0 : k8s allows the CPULimit to be 0, but the quota is not limited. + if ci.LimitResources[typedef.ResourceCPU] <= 0 || + ci.RequestResources[typedef.ResourceCPU] <= 0 || + ci.LimitResources[typedef.ResourceCPU] == float64(runtime.NumCPU()) || + ci.RequestResources[typedef.ResourceCPU] == float64(runtime.NumCPU()) { + return false + } + return true +} diff --git a/pkg/services/quotaturbo/data_test.go b/pkg/services/quotaturbo/data_test.go new file mode 100644 index 0000000..5f516c3 --- /dev/null +++ b/pkg/services/quotaturbo/data_test.go @@ -0,0 +1,232 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: This file is used for testing data.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "path" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +// TestNodeDataGetLastCPUUtil tests getLastCPUUtil of NodeData +func TestNodeDataGetLastCPUUtil(t *testing.T) { + // 1. empty CPU Utils + d := &NodeData{} + t.Run("TC1-empty CPU Util", func(t *testing.T) { + util := float64(0.0) + assert.Equal(t, util, d.getLastCPUUtil()) + }) + // 2. CPU Utils + cpuUtil20 := 20 + d = &NodeData{cpuUtils: []cpuUtil{{ + util: float64(cpuUtil20), + }}} + t.Run("TC2-CPU Util is 20", func(t *testing.T) { + util := float64(20.0) + assert.Equal(t, util, d.getLastCPUUtil()) + }) +} + +// TestNodeDataRemoveContainer tests removeContainer of NodeData +func TestNodeDataRemoveContainer(t *testing.T) { + var nodeDataRemoveContainerTests = []struct { + data *NodeData + name string + id string + num int + }{ + { + name: "TC1-no container exists", + id: "", + num: 1, + data: &NodeData{ + containers: map[string]*CPUQuota{ + "testCon1": {}, + }, + }, + }, + { + name: "TC2-the container path does not exist", + id: "testCon3", + num: 1, + data: &NodeData{ + containers: map[string]*CPUQuota{ + "testCon1": {}, + "testCon3": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon3", + CgroupPath: "kubepods/testPod3/testCon3", + }, + }, + }, + }, + }, + { + name: "TC3-delete a container normally", + id: containerInfos[0].ID, + num: 0, + data: &NodeData{ + containers: map[string]*CPUQuota{ + containerInfos[0].ID: { + ContainerInfo: containerInfos[0].DeepCopy(), + period: 100000, + }, + }, + }, + }, + { + name: "TC4-write an invalid value", + id: containerInfos[0].ID, + num: 1, + data: &NodeData{ + containers: map[string]*CPUQuota{ + containerInfos[0].ID: { + ContainerInfo: containerInfos[0].DeepCopy(), + period: 1000000000000000000, + }, + }, + }, + }, + } + cis := []*typedef.ContainerInfo{containerInfos[0].DeepCopy()} + mkCgDirs(cis) + defer rmCgDirs(cis) + + for _, tt := range nodeDataRemoveContainerTests { + t.Run(tt.name, func(t *testing.T) { + tt.data.removeContainer(tt.id) + assert.Equal(t, tt.num, len(tt.data.containers)) + }) + } + us, err := cis[0].GetCgroupAttr(cpuQuotaKey).Int64() + if err != nil { + assert.NoError(t, err) + } + assert.Equal(t, int64(cis[0].LimitResources[typedef.ResourceCPU]*100000), us) +} + +// TestQuotaTurboUpdateCPUUtils tests updateCPUUtils of QuotaTurbo and NewProcStat +func TestQuotaTurboUpdateCPUUtils(t *testing.T) { + data := NewNodeData() + // 1. obtain the cpu usage for the first time + if err := data.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + num1 := 1 + assert.Equal(t, num1, len(data.cpuUtils)) + // 2. obtain the cpu usage for the second time + if err := data.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + num2 := 2 + assert.Equal(t, num2, len(data.cpuUtils)) + // 3. obtain the cpu usage after 1 minute + var minuteTimeDelta int64 = 60000000001 + data.cpuUtils[0].timestamp -= minuteTimeDelta + if err := data.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + assert.Equal(t, num2, len(data.cpuUtils)) +} + +// TestIsAdjustmentAllowed tests isAdjustmentAllowed +func TestIsAdjustmentAllowed(t *testing.T) { + cis := []*typedef.ContainerInfo{containerInfos[0].DeepCopy(), containerInfos[3].DeepCopy()} + mkCgDirs(cis) + defer rmCgDirs(cis) + + tests := []struct { + ci *typedef.ContainerInfo + name string + want bool + }{ + { + name: "TC1-allow adjustment", + ci: cis[0], + want: true, + }, + { + name: "TC2-cgroup path is not existed", + ci: containerInfos[1], + want: false, + }, + { + name: "TC3-cpulimit = 0", + ci: cis[1], + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, isAdjustmentAllowed(tt.ci), tt.want) + }) + } +} + +// TestUpdateClusterContainers tests UpdateClusterContainers +func TestUpdateClusterContainers(t *testing.T) { + cis := []*typedef.ContainerInfo{containerInfos[0]} + mkCgDirs(cis) + defer rmCgDirs(cis) + + qt := &QuotaTurbo{ + NodeData: &NodeData{ + containers: make(map[string]*CPUQuota, 0), + }, + } + + // 1. add container + qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{cis[0].ID: cis[0]}) + conNum1 := 1 + assert.Equal(t, conNum1, len(qt.containers)) + + // 2. updated container + qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{cis[0].ID: cis[0]}) + assert.Equal(t, conNum1, len(qt.containers)) + + // 3. deleting a container that does not meet the conditions (cgroup path is not existed) + // 4. delete a container whose checkpoint does not exist. + qt.containers[containerInfos[2].ID] = &CPUQuota{ContainerInfo: containerInfos[2].DeepCopy()} + conNum2 := 2 + assert.Equal(t, conNum2, len(qt.containers)) + qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{containerInfos[2].ID: containerInfos[2].DeepCopy()}) + conNum0 := 0 + assert.Equal(t, conNum0, len(qt.containers)) +} + +// mkCgDirs creates the cgroup folder for the container +func mkCgDirs(cc []*typedef.ContainerInfo) { + for _, ci := range cc { + dirPath := cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") + fmt.Println("path : " + dirPath) + try.MkdirAll(dirPath, constant.DefaultDirMode) + } +} + +// rmCgDirs deletes the cgroup folder of the container +func rmCgDirs(cc []*typedef.ContainerInfo) { + for _, ci := range cc { + dirPath := cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") + try.RemoveAll(dirPath) + try.RemoveAll(path.Dir(dirPath)) + } +} diff --git a/pkg/services/quotaturbo/driverevent.go b/pkg/services/quotaturbo/driverevent.go new file mode 100644 index 0000000..a13d259 --- /dev/null +++ b/pkg/services/quotaturbo/driverevent.go @@ -0,0 +1,196 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: event driver method for quota turbo + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "math" + "runtime" + + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" +) + +// Driver uses different methods based on different policies. +type Driver interface { + // adjustQuota calculate the quota in the next period based on the customized policy, upper limit, and quota. + adjustQuota(data *NodeData) +} + +const ( + // upperLimitOfIncrease is the maximum percentage of. + // the total amount of a single promotion to the total amount of nodes. + upperLimitOfIncrease = 1.0 + // slowFallbackRatio is the rate of slow fallback. + slowFallbackRatio = 0.1 + // limitOfNodeCPUUsageChangeWithin1Minute is the upper limit of the node change rate in one minute. + limitOfNodeCPUUsageChangeWithin1Minute float64 = 10 +) + +// EventDriver event based quota adjustment driver. +type EventDriver struct{} + +// adjustQuota calculates quota delta based on events +func (e *EventDriver) adjustQuota(data *NodeData) { + e.slowFallback(data) + e.fastFallback(data) + // Ensure that the CPU usage does not change by more than 10% within one minute. + // Otherwise, the available quota rollback continues but does not increase. + if !sharpFluctuates(data) { + e.elevate(data) + } else { + log.Infof("the CPU usage changes by more than %.2f, "+ + "the quota will not adjusted in this round", limitOfNodeCPUUsageChangeWithin1Minute) + } + for _, c := range data.containers { + // get height limit + const easingMultiple = 2.0 + // c.Period + c.heightLimit = easingMultiple * c.LimitResources[typedef.ResourceCPU] * float64(c.period) + // get the maximum available ensuring that the overall utilization does not exceed the limit. + c.maxQuotaNextPeriod = getMaxQuota(c) + // c.Period ranges from 1000(us) to 1000000(us) and does not overflow. + c.nextQuota = int64(math.Max(math.Min(float64(c.curQuota)+c.quotaDelta, c.maxQuotaNextPeriod), + c.LimitResources[typedef.ResourceCPU]*float64(c.period))) + } +} + +// elevate boosts when cpu is suppressed +func (e *EventDriver) elevate(data *NodeData) { + // the CPU usage of the current node is lower than the warning watermark. + // U + R <= a & a > U ======> a - U >= R && a - U > 0 =====> a - U >= R + if float64(data.AlarmWaterMark)-data.getLastCPUUtil() < upperLimitOfIncrease { + return + } + // sumDelta : total number of cores to be adjusted + var sumDelta float64 = 0 + delta := make(map[string]float64, 0) + for _, c := range data.containers { + if c.curThrottle.NrThrottled > c.preThrottle.NrThrottled { + delta[c.ID] = NsToUs(c.curThrottle.ThrottledTime-c.preThrottle.ThrottledTime) / + float64(c.curThrottle.NrThrottled-c.preThrottle.NrThrottled) / float64(c.period) + sumDelta += delta[c.ID] + } + } + // the container quota does not need to be increased in this round. + if sumDelta == 0 { + return + } + // the total increase cannot exceed (upperLimitOfIncrease%) of the total available CPUs of the node. + A := math.Min(sumDelta, util.PercentageToDecimal(upperLimitOfIncrease)*float64(runtime.NumCPU())) + coefficient := A / sumDelta + for id, quotaDelta := range delta { + data.containers[id].quotaDelta += coefficient * quotaDelta * float64(data.containers[id].period) + } +} + +// fastFallback decreases the quota to ensure that the CPU utilization of the node is below the warning water level +// when the water level exceeds the warning water level. +func (e *EventDriver) fastFallback(data *NodeData) { + // The CPU usage of the current node is greater than the warning watermark, triggering a fast rollback. + if float64(data.AlarmWaterMark) > data.getLastCPUUtil() { + return + } + // sub : the total number of CPU quotas to be reduced on a node. + sub := util.PercentageToDecimal(float64(data.AlarmWaterMark)-data.getLastCPUUtil()) * float64(runtime.NumCPU()) + // sumDelta ：total number of cpu cores that can be decreased. + var sumDelta float64 = 0 + delta := make(map[string]float64, 0) + for _, c := range data.containers { + delta[c.ID] = float64(c.curQuota)/float64(c.period) - c.LimitResources[typedef.ResourceCPU] + sumDelta += delta[c.ID] + } + if sumDelta <= 0 { + return + } + // proportional adjustment of each business quota. + for id, quotaDelta := range delta { + data.containers[id].quotaDelta += (quotaDelta / sumDelta) * sub * float64(data.containers[id].period) + } +} + +// slowFallback triggers quota callback of unpressed containers when the CPU utilization exceeds the control watermark. +func (e *EventDriver) slowFallback(data *NodeData) { + // The CPU usage of the current node is greater than the high watermark, triggering a slow rollback. + if float64(data.HighWaterMark) > data.getLastCPUUtil() { + return + } + coefficient := (data.getLastCPUUtil() - float64(data.HighWaterMark)) / + float64(data.AlarmWaterMark-data.HighWaterMark) * slowFallbackRatio + for id, c := range data.containers { + originQuota := int64(c.LimitResources[typedef.ResourceCPU] * float64(c.period)) + if c.curQuota > originQuota && c.curThrottle.NrThrottled == c.preThrottle.NrThrottled { + data.containers[id].quotaDelta += coefficient * float64(originQuota-c.curQuota) + } + } +} + +// sharpFluctuates checks whether the node CPU utilization exceeds the specified value within one minute. +func sharpFluctuates(data *NodeData) bool { + var ( + min float64 = maximumUtilization + max float64 = minimumUtilization + ) + for _, u := range data.cpuUtils { + min = math.Min(min, u.util) + max = math.Max(max, u.util) + } + if max-min > limitOfNodeCPUUsageChangeWithin1Minute { + log.Debugf("The resources changed drastically, and no adjustment will be made this time") + return true + } + return false +} + +// getMaxQuota calculate the maximum available quota in the next period based on the container CPU usage in N-1 periods. +func getMaxQuota(c *CPUQuota) float64 { + if len(c.cpuUsages) <= 1 { + return c.heightLimit + } + // the time unit is nanosecond + first := c.cpuUsages[0] + last := c.cpuUsages[len(c.cpuUsages)-1] + timeDelta := NsToUs(last.timestamp - first.timestamp) + coefficient := float64(len(c.cpuUsages)) / float64(len(c.cpuUsages)-1) + maxAvailable := c.LimitResources[typedef.ResourceCPU] * timeDelta * coefficient + used := NsToUs(last.usage - first.usage) + remainingUsage := maxAvailable - used + origin := c.LimitResources[typedef.ResourceCPU] * float64(c.period) + const ( + // To prevent sharp service jitters, the Rubik proactively decreases the traffic in advance + // when the available balance reaches a certain threshold. + // The limitMultiplier is used to control the relationship between the upper limit and the threshold. + // Experiments show that the value 3 is efficient and secure. + limitMultiplier = 3 + precision = 1e-10 + ) + var threshold = limitMultiplier * c.heightLimit + remainingQuota := util.Div(remainingUsage, timeDelta, math.MaxFloat64, precision) * + float64(len(c.cpuUsages)-1) * float64(c.period) + + // gradually decrease beyond the threshold to prevent sudden dips. + res := remainingQuota + if remainingQuota <= threshold { + res = origin + util.Div((c.heightLimit-origin)*remainingQuota, threshold, threshold, precision) + } + // The utilization must not exceed the height limit and must not be less than the cpuLimit. + return math.Max(math.Min(res, c.heightLimit), origin) +} + +// NsToUs converts nanoseconds into microseconds +func NsToUs(ns int64) float64 { + // number of nanoseconds contained in 1 microsecond + const nanoSecPerMicroSec float64 = 1000 + return util.Div(float64(ns), nanoSecPerMicroSec) +} diff --git a/pkg/services/quotaturbo/driverevent_test.go b/pkg/services/quotaturbo/driverevent_test.go new file mode 100644 index 0000000..5be213d --- /dev/null +++ b/pkg/services/quotaturbo/driverevent_test.go @@ -0,0 +1,593 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for testing driverevent.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "math" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +// TestEventDriverElevate tests elevate of EventDriver +func TestEventDriverElevate(t *testing.T) { + var elevateTests = []struct { + data *NodeData + judgements func(t *testing.T, data *NodeData) + name string + }{ + { + name: "TC1 - CPU usage >= the alarmWaterMark.", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 60, + }, + containers: map[string]*CPUQuota{ + "testCon1": {}, + }, + cpuUtils: []cpuUtil{ + { + util: 90, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + var delta float64 = 0 + conID := "testCon1" + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC2 - the container is not suppressed.", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 70, + }, + containers: map[string]*CPUQuota{ + "testCon2": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon2", + }, + // currently not suppressed + curThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 10, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 10, + }, + period: 100000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 60, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + var delta float64 = 0 + conID := "testCon2" + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC3 - increase the quota of the suppressed container", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 60, + }, + containers: map[string]*CPUQuota{ + "testCon3": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon3", + }, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 50, + ThrottledTime: 200000, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 40, + ThrottledTime: 100000, + }, + period: 100000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 40, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon3" + c := data.containers[conID] + coefficient := math.Min(float64(0.0001), util.PercentageToDecimal(upperLimitOfIncrease)* + float64(runtime.NumCPU())) / float64(0.0001) + delta := coefficient * float64(0.0001) * float64(c.period) + assert.True(t, data.containers[conID].quotaDelta == delta) + }, + }, + } + + e := &EventDriver{} + for _, tt := range elevateTests { + t.Run(tt.name, func(t *testing.T) { + e.elevate(tt.data) + tt.judgements(t, tt.data) + }) + } +} + +// TestSlowFallback tests slowFallback of EventDriver +func TestSlowFallback(t *testing.T) { + var slowFallBackTests = []struct { + data *NodeData + judgements func(t *testing.T, data *NodeData) + name string + }{ + { + name: "TC1-CPU usage <= the highWaterMark.", + data: &NodeData{ + Config: &Config{ + HighWaterMark: 60, + }, + containers: map[string]*CPUQuota{ + "testCon4": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon4", + }, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 40, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon4" + var delta float64 = 0 + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC2-the container is suppressed.", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 50, + }, + containers: map[string]*CPUQuota{ + "testCon5": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon5", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 1, + }, + }, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 0, + }, + period: 100000, + curQuota: 200000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 70, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + var delta float64 = 0 + conID := "testCon5" + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC3-decrease the quota of the uncompressed containers", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 90, + HighWaterMark: 40, + }, + containers: map[string]*CPUQuota{ + "testCon6": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon6", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 2, + }, + }, + // currently not suppressed + curThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + ThrottledTime: 100000, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + ThrottledTime: 100000, + }, + period: 100000, + curQuota: 400000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 60.0, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon6" + c := data.containers[conID] + coefficient := (data.getLastCPUUtil() - float64(data.HighWaterMark)) / + float64(data.AlarmWaterMark-data.HighWaterMark) * slowFallbackRatio + delta := coefficient * + ((float64(c.LimitResources[typedef.ResourceCPU]) * float64(c.period)) - float64(c.curQuota)) + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range slowFallBackTests { + t.Run(tt.name, func(t *testing.T) { + e.slowFallback(tt.data) + tt.judgements(t, tt.data) + }) + } +} + +// TestFastFallback tests fastFallback of EventDriver +func TestFastFallback(t *testing.T) { + var fastFallBackTests = []struct { + data *NodeData + judgements func(t *testing.T, data *NodeData) + name string + }{ + { + name: "TC1-CPU usage <= the AlarmWaterMark.", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 30, + }, + containers: map[string]*CPUQuota{ + "testCon7": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon7", + }, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 10, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon7" + var delta float64 = 0 + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC2-the quota of container is not increased.", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 30, + }, + containers: map[string]*CPUQuota{ + "testCon8": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon8", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 1, + }, + }, + period: 100, + curQuota: 100, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 48, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + var delta float64 = 0 + conID := "testCon8" + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC3-decrease the quota of the containers", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 65, + }, + containers: map[string]*CPUQuota{ + "testCon9": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon9", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 3, + }, + }, + period: 10000, + curQuota: 40000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 90, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon9" + c := data.containers[conID] + delta := util.PercentageToDecimal(float64(data.AlarmWaterMark)-data.getLastCPUUtil()) * + float64(runtime.NumCPU()) * float64(c.period) + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range fastFallBackTests { + t.Run(tt.name, func(t *testing.T) { + e.fastFallback(tt.data) + tt.judgements(t, tt.data) + }) + } +} + +// TestSharpFluctuates tests sharpFluctuates +func TestSharpFluctuates(t *testing.T) { + var sharpFluctuatesTests = []struct { + data *NodeData + want bool + name string + }{ + { + name: "TC1-the cpu changes rapidly", + data: &NodeData{ + cpuUtils: []cpuUtil{ + { + util: 90, + }, + { + util: 90 - limitOfNodeCPUUsageChangeWithin1Minute - 1, + }, + }, + }, + want: true, + }, + { + name: "TC2-the cpu changes steadily", + data: &NodeData{ + cpuUtils: []cpuUtil{ + { + util: 90, + }, + { + util: 90 - limitOfNodeCPUUsageChangeWithin1Minute + 1, + }, + }, + }, + want: false, + }, + } + for _, tt := range sharpFluctuatesTests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, sharpFluctuates(tt.data) == tt.want) + }) + } +} + +// TestEventDriverAdjustQuota tests adjustQuota of EventDriver +func TestEventDriverAdjustQuota(t *testing.T) { + var eDriverAdjustQuotaTests = []struct { + data *NodeData + judgements func(t *testing.T, data *NodeData) + name string + }{ + { + name: "TC1-no promotion", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 73, + }, + containers: map[string]*CPUQuota{ + "testCon10": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon10", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 1, + }, + }, + period: 80, + curQuota: 100, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 1, + }, + { + util: -limitOfNodeCPUUsageChangeWithin1Minute, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + var delta float64 = 0 + conID := "testCon10" + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + { + name: "TC2-make a promotion", + data: &NodeData{ + Config: &Config{ + AlarmWaterMark: 97, + HighWaterMark: 73, + }, + containers: map[string]*CPUQuota{ + "testCon11": { + ContainerInfo: &typedef.ContainerInfo{ + ID: "testCon11", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 2, + }, + }, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 200, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 0, + ThrottledTime: 100, + }, + period: 2000, + curQuota: 5000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 10, + }, + }, + }, + judgements: func(t *testing.T, data *NodeData) { + conID := "testCon11" + c := data.containers[conID] + coefficient := math.Min(float64(0.00005), util.PercentageToDecimal(upperLimitOfIncrease)* + float64(runtime.NumCPU())) / float64(0.00005) + delta := coefficient * float64(0.00005) * float64(c.period) + assert.Equal(t, delta, data.containers[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range eDriverAdjustQuotaTests { + t.Run(tt.name, func(t *testing.T) { + e.adjustQuota(tt.data) + tt.judgements(t, tt.data) + }) + } +} + +// TestGetMaxQuota tests getMaxQuota +func TestGetMaxQuota(t *testing.T) { + var getMaxQuotaTests = []struct { + cq *CPUQuota + judgements func(t *testing.T, cq *CPUQuota) + name string + }{ + { + name: "TC1-empty cpu usage", + cq: &CPUQuota{ + heightLimit: 100, + cpuUsages: []cpuUsage{}, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 100 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC2-The remaining value is less than 3 times the upper limit.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {100000, 100000}, + {200000, 200000}, + }, + ContainerInfo: &typedef.ContainerInfo{ + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 4, + }, + }, + period: 100, + heightLimit: 800, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + const res = 400 + float64(400*700)/float64(3*800) + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC3-The remaining value is greater than 3 times the limit height.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {10000, 0}, + {20000, 0}, + {30000, 0}, + {40000, 0}, + {50000, 0}, + {60000, 0}, + {70000, 0}, + {80000, 100}, + }, + ContainerInfo: &typedef.ContainerInfo{ + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 1, + }, + }, + period: 100, + heightLimit: 200, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 200 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC4-The remaining value is less than the initial value.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {100, 0}, + {200, 1000000}, + }, + ContainerInfo: &typedef.ContainerInfo{ + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 10, + }, + }, + period: 10, + heightLimit: 150, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 100 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + } + for _, tt := range getMaxQuotaTests { + t.Run(tt.name, func(t *testing.T) { + tt.judgements(t, tt.cq) + }) + } +} diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go new file mode 100644 index 0000000..b7febf0 --- /dev/null +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -0,0 +1,108 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: quota turbo method（dynamically adjusting container quotas） + +// Package quotaturbo is for Quota Turbo +package quotaturbo + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/services" +) + +const moduleName = "quotaturbo" + +var log api.Logger + +func init() { + log = &Log.EmptyLog{} + services.Register(moduleName, func() interface{} { + return NewQuotaTurbo() + }) +} + +// QuotaTurbo manages all container CPU quota data on the current node. +type QuotaTurbo struct { + // NodeData including the container data, CPU usage, and QuotaTurbo configuration of the local node + *NodeData + // interfaces with different policies + Driver + // referenced object to list pods + Viewer api.Viewer +} + +// SetupLog initializes the log interface for the module +func (qt *QuotaTurbo) SetupLog(logger api.Logger) { + log = logger +} + +// NewQuotaTurbo generate quota turbo objects +func NewQuotaTurbo() *QuotaTurbo { + return &QuotaTurbo{ + NodeData: NewNodeData(), + } +} + +// ID returns the module name +func (qt *QuotaTurbo) ID() string { + return moduleName +} + +// PreStart is the pre-start action +func (qt *QuotaTurbo) PreStart(v api.Viewer) error { + qt.Viewer = v + return nil +} + +// saveQuota saves the quota value of the container +func (qt *QuotaTurbo) saveQuota() { + for _, c := range qt.containers { + if err := c.SaveQuota(); err != nil { + log.Errorf(err.Error()) + } + } +} + +// AdjustQuota adjusts the quota of a container at a time +func (qt *QuotaTurbo) AdjustQuota(cc map[string]*typedef.ContainerInfo) { + qt.UpdateClusterContainers(cc) + if err := qt.updateCPUUtils(); err != nil { + log.Errorf("fail to get current cpu utilization : %v", err) + return + } + if len(qt.containers) == 0 { + return + } + qt.adjustQuota(qt.NodeData) + qt.saveQuota() +} + +// Run adjusts the quota of the trust list container cyclically. +func (qt *QuotaTurbo) Run(ctx context.Context) { + wait.Until( + func() { + qt.AdjustQuota(qt.Viewer.ListContainersWithOptions( + func(pod *typedef.PodInfo) bool { + return pod.Annotations[constant.QuotaAnnotationKey] == "true" + })) + }, + time.Millisecond*time.Duration(qt.SyncInterval), + ctx.Done()) +} -- Gitee From 2987cf23f2ff268249163d5b0f709ca975cf16aa Mon Sep 17 00:00:00 2001 From: vegbir Date: Tue, 21 Feb 2023 11:05:28 +0800 Subject: [PATCH 32/73] feature: support quotaTurbo Prestart\Validate\Terminate add some testcases --- pkg/podmanager/podcache.go | 22 +- pkg/podmanager/podmanager.go | 24 +- pkg/podmanager/podmanager_test.go | 6 +- pkg/services/quotaturbo/cpuquota.go | 4 +- pkg/services/quotaturbo/cpuquota_test.go | 12 +- pkg/services/quotaturbo/data.go | 10 +- pkg/services/quotaturbo/quotaturbo.go | 118 ++++++-- pkg/services/quotaturbo/quotaturbo_test.go | 330 +++++++++++++++++++++ 8 files changed, 470 insertions(+), 56 deletions(-) create mode 100644 pkg/services/quotaturbo/quotaturbo_test.go diff --git a/pkg/podmanager/podcache.go b/pkg/podmanager/podcache.go index fced314..9e021c4 100644 --- a/pkg/podmanager/podcache.go +++ b/pkg/podmanager/podcache.go @@ -21,28 +21,28 @@ import ( "isula.org/rubik/pkg/core/typedef" ) -// podCache is used to store PodInfo -type podCache struct { +// PodCache is used to store PodInfo +type PodCache struct { sync.RWMutex Pods map[string]*typedef.PodInfo } // NewPodCache returns a PodCache object (pointer) -func NewPodCache() *podCache { - return &podCache{ +func NewPodCache() *PodCache { + return &PodCache{ Pods: make(map[string]*typedef.PodInfo, 0), } } // getPod returns the deepcopy object of pod -func (cache *podCache) getPod(podID string) *typedef.PodInfo { +func (cache *PodCache) getPod(podID string) *typedef.PodInfo { cache.RLock() defer cache.RUnlock() return cache.Pods[podID].DeepCopy() } // podExist returns true if there is a pod whose key is podID in the pods -func (cache *podCache) podExist(podID string) bool { +func (cache *PodCache) podExist(podID string) bool { cache.RLock() _, ok := cache.Pods[podID] cache.RUnlock() @@ -50,7 +50,7 @@ func (cache *podCache) podExist(podID string) bool { } // addPod adds pod information -func (cache *podCache) addPod(pod *typedef.PodInfo) { +func (cache *PodCache) addPod(pod *typedef.PodInfo) { if pod == nil || pod.UID == "" { return } @@ -65,7 +65,7 @@ func (cache *podCache) addPod(pod *typedef.PodInfo) { } // delPod deletes pod information -func (cache *podCache) delPod(podID string) { +func (cache *PodCache) delPod(podID string) { if ok := cache.podExist(podID); !ok { log.Debugf("pod %v is not existed", string(podID)) return @@ -77,7 +77,7 @@ func (cache *podCache) delPod(podID string) { } // updatePod updates pod information -func (cache *podCache) updatePod(pod *typedef.PodInfo) { +func (cache *PodCache) updatePod(pod *typedef.PodInfo) { if pod == nil || pod.UID == "" { return } @@ -88,7 +88,7 @@ func (cache *podCache) updatePod(pod *typedef.PodInfo) { } // substitute replaces all the data in the cache -func (cache *podCache) substitute(pods []*typedef.PodInfo) { +func (cache *PodCache) substitute(pods []*typedef.PodInfo) { cache.Lock() defer cache.Unlock() cache.Pods = make(map[string]*typedef.PodInfo, 0) @@ -105,7 +105,7 @@ func (cache *podCache) substitute(pods []*typedef.PodInfo) { } // listPod returns the deepcopy object of all pod -func (cache *podCache) listPod() map[string]*typedef.PodInfo { +func (cache *PodCache) listPod() map[string]*typedef.PodInfo { res := make(map[string]*typedef.PodInfo, len(cache.Pods)) cache.RLock() for id, pi := range cache.Pods { diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index 3dc9e67..8e3eef2 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -32,13 +32,13 @@ const PodManagerName = "DefaultPodManager" type PodManager struct { api.Subscriber api.Publisher - pods *podCache + Pods *PodCache } // NewPodManager returns a PodManager pointer func NewPodManager(publisher api.Publisher) *PodManager { manager := &PodManager{ - pods: NewPodCache(), + Pods: NewPodCache(), Publisher: publisher, } manager.Subscriber = subscriber.NewGenericSubscriber(manager, PodManagerName) @@ -136,7 +136,7 @@ func (manager *PodManager) addFunc(pod *typedef.RawPod) { return } // condition2: pod is not existed - if manager.pods.podExist(pod.ID()) { + if manager.Pods.podExist(pod.ID()) { log.Debugf("pod %v has added", pod.UID) return } @@ -179,8 +179,8 @@ func (manager *PodManager) deleteFunc(pod *typedef.RawPod) { // tryAdd tries to add pod info which is not added func (manager *PodManager) tryAdd(podInfo *typedef.PodInfo) { // only add when pod is not existed - if !manager.pods.podExist(podInfo.UID) { - manager.pods.addPod(podInfo) + if !manager.Pods.podExist(podInfo.UID) { + manager.Pods.addPod(podInfo) manager.Publish(typedef.INFOADD, podInfo.DeepCopy()) } } @@ -188,9 +188,9 @@ func (manager *PodManager) tryAdd(podInfo *typedef.PodInfo) { // tryUpdate tries to update podinfo which is existed func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { // only update when pod is existed - if manager.pods.podExist(podInfo.UID) { - oldPod := manager.pods.getPod(podInfo.UID) - manager.pods.updatePod(podInfo) + if manager.Pods.podExist(podInfo.UID) { + oldPod := manager.Pods.getPod(podInfo.UID) + manager.Pods.updatePod(podInfo) manager.Publish(typedef.INFOUPDATE, []*typedef.PodInfo{oldPod, podInfo.DeepCopy()}) } } @@ -198,9 +198,9 @@ func (manager *PodManager) tryUpdate(podInfo *typedef.PodInfo) { // tryDelete tries to delete podinfo which is existed func (manager *PodManager) tryDelete(id string) { // only delete when pod is existed - oldPod := manager.pods.getPod(id) + oldPod := manager.Pods.getPod(id) if oldPod != nil { - manager.pods.delPod(id) + manager.Pods.delPod(id) manager.Publish(typedef.INFODELETE, oldPod) } } @@ -214,7 +214,7 @@ func (manager *PodManager) sync(pods []*typedef.RawPod) { } newPods = append(newPods, pod.ExtractPodInfo()) } - manager.pods.substitute(newPods) + manager.Pods.substitute(newPods) } // ListOfflinePods returns offline pods @@ -250,7 +250,7 @@ func (manager *PodManager) ListContainersWithOptions(options ...api.ListOption) // ListPodsWithOptions filters and returns deep copy objects of all pods func (manager *PodManager) ListPodsWithOptions(options ...api.ListOption) map[string]*typedef.PodInfo { // already deep copied - allPods := manager.pods.listPod() + allPods := manager.Pods.listPod() pods := make(map[string]*typedef.PodInfo, len(allPods)) for _, pod := range allPods { if !withOption(pod, options) { diff --git a/pkg/podmanager/podmanager_test.go b/pkg/podmanager/podmanager_test.go index eab7fa0..85e4ffa 100644 --- a/pkg/podmanager/podmanager_test.go +++ b/pkg/podmanager/podmanager_test.go @@ -38,7 +38,7 @@ func TestPodManager_ListContainersWithOptions(t *testing.T) { ) type fields struct { - pods *podCache + pods *PodCache } type args struct { options []api.ListOption @@ -60,7 +60,7 @@ func TestPodManager_ListContainersWithOptions(t *testing.T) { }, }, fields: fields{ - pods: &podCache{ + pods: &PodCache{ Pods: map[string]*typedef.PodInfo{ "testPod1": { UID: "testPod1", @@ -89,7 +89,7 @@ func TestPodManager_ListContainersWithOptions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { manager := &PodManager{ - pods: tt.fields.pods, + Pods: tt.fields.pods, } if got := manager.ListContainersWithOptions(tt.args.options...); !reflect.DeepEqual(got, tt.want) { assert.Equal(t, tt.want, got) diff --git a/pkg/services/quotaturbo/cpuquota.go b/pkg/services/quotaturbo/cpuquota.go index 08402a3..dcb079f 100644 --- a/pkg/services/quotaturbo/cpuquota.go +++ b/pkg/services/quotaturbo/cpuquota.go @@ -166,8 +166,8 @@ func (c *CPUQuota) writeContainerQuota() error { return c.ContainerInfo.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(c.curQuota)) } -// SaveQuota use to modify quota for container -func (c *CPUQuota) SaveQuota() error { +// WriteQuota use to modify quota for container +func (c *CPUQuota) WriteQuota() error { delta := c.nextQuota - c.curQuota tmp := c.curQuota c.curQuota = c.nextQuota diff --git a/pkg/services/quotaturbo/cpuquota_test.go b/pkg/services/quotaturbo/cpuquota_test.go index 89584c1..847b1d6 100644 --- a/pkg/services/quotaturbo/cpuquota_test.go +++ b/pkg/services/quotaturbo/cpuquota_test.go @@ -76,10 +76,10 @@ func TestSaveQuota(t *testing.T) { // None of the paths exist cq.nextQuota = largerQuotaVal cq.curQuota = smallerQuotaVal - assert.Error(t, cq.SaveQuota()) + assert.Error(t, cq.WriteQuota()) cq.nextQuota = smallerQuotaVal cq.curQuota = largerQuotaVal - assert.Error(t, cq.SaveQuota()) + assert.Error(t, cq.WriteQuota()) // only Pod path existed cq.nextQuota = largerQuotaVal @@ -88,7 +88,7 @@ func TestSaveQuota(t *testing.T) { try.WriteFile(podPeriodPath, periodUs) try.RemoveAll(contQuotaPath) try.RemoveAll(contPeriodPath) - assert.Error(t, cq.SaveQuota()) + assert.Error(t, cq.WriteQuota()) // only container path existed cq.nextQuota = smallerQuotaVal @@ -97,7 +97,7 @@ func TestSaveQuota(t *testing.T) { try.RemoveAll(podPeriodPath) try.WriteFile(contQuotaPath, largerQuota) try.WriteFile(contPeriodPath, periodUs) - assert.Error(t, cq.SaveQuota()) + assert.Error(t, cq.WriteQuota()) }() @@ -114,12 +114,12 @@ func TestSaveQuota(t *testing.T) { // delta > 0 cq.nextQuota = largerQuotaVal cq.curQuota = smallerQuotaVal - assert.NoError(t, cq.SaveQuota()) + assert.NoError(t, cq.WriteQuota()) assertValue(t, []string{podQuotaPath, contQuotaPath}, largerQuota) // delta < 0 cq.nextQuota = smallerQuotaVal cq.curQuota = largerQuotaVal - assert.NoError(t, cq.SaveQuota()) + assert.NoError(t, cq.WriteQuota()) assertValue(t, []string{podQuotaPath, contQuotaPath}, smallerQuota) }() diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go index 99eac56..d4589f8 100644 --- a/pkg/services/quotaturbo/data.go +++ b/pkg/services/quotaturbo/data.go @@ -73,7 +73,6 @@ func NewNodeData() *NodeData { containers: make(map[string]*CPUQuota, 0), cpuUtils: make([]cpuUtil, 0), } - } // getLastCPUUtil obtain the latest cpu utilization @@ -196,6 +195,15 @@ func (d *NodeData) UpdateClusterContainers(conts map[string]*typedef.ContainerIn return nil } +// WriteQuota saves the quota value of the container +func (d *NodeData) WriteQuota() { + for _, c := range d.containers { + if err := c.WriteQuota(); err != nil { + log.Errorf(err.Error()) + } + } +} + // isAdjustmentAllowed judges whether quota adjustment is allowed func isAdjustmentAllowed(ci *typedef.ContainerInfo) bool { // 1. containers whose cgroup path does not exist are not considered. diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index b7febf0..289b332 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -16,6 +16,7 @@ package quotaturbo import ( "context" + "fmt" "time" "k8s.io/apimachinery/pkg/util/wait" @@ -23,6 +24,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" ) @@ -48,11 +50,6 @@ type QuotaTurbo struct { Viewer api.Viewer } -// SetupLog initializes the log interface for the module -func (qt *QuotaTurbo) SetupLog(logger api.Logger) { - log = logger -} - // NewQuotaTurbo generate quota turbo objects func NewQuotaTurbo() *QuotaTurbo { return &QuotaTurbo{ @@ -60,26 +57,16 @@ func NewQuotaTurbo() *QuotaTurbo { } } +// SetupLog initializes the log interface for the module +func (qt *QuotaTurbo) SetupLog(logger api.Logger) { + log = logger +} + // ID returns the module name func (qt *QuotaTurbo) ID() string { return moduleName } -// PreStart is the pre-start action -func (qt *QuotaTurbo) PreStart(v api.Viewer) error { - qt.Viewer = v - return nil -} - -// saveQuota saves the quota value of the container -func (qt *QuotaTurbo) saveQuota() { - for _, c := range qt.containers { - if err := c.SaveQuota(); err != nil { - log.Errorf(err.Error()) - } - } -} - // AdjustQuota adjusts the quota of a container at a time func (qt *QuotaTurbo) AdjustQuota(cc map[string]*typedef.ContainerInfo) { qt.UpdateClusterContainers(cc) @@ -91,7 +78,7 @@ func (qt *QuotaTurbo) AdjustQuota(cc map[string]*typedef.ContainerInfo) { return } qt.adjustQuota(qt.NodeData) - qt.saveQuota() + qt.WriteQuota() } // Run adjusts the quota of the trust list container cyclically. @@ -106,3 +93,92 @@ func (qt *QuotaTurbo) Run(ctx context.Context) { time.Millisecond*time.Duration(qt.SyncInterval), ctx.Done()) } + +// Validate Validate verifies that the quotaTurbo parameter is set correctly +func (qt *QuotaTurbo) Validate() error { + const ( + minQuotaTurboWaterMark, maxQuotaTurboWaterMark = 0, 100 + minQuotaTurboSyncInterval, maxQuotaTurboSyncInterval = 100, 10000 + ) + outOfRange := func(num, min, max int) bool { + if num < min || num > max { + return true + } + return false + } + if qt.AlarmWaterMark <= qt.HighWaterMark || + outOfRange(qt.HighWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) || + outOfRange(qt.AlarmWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) { + return fmt.Errorf("alarmWaterMark >= highWaterMark, both of which ranges from 0 to 100") + } + if outOfRange(qt.SyncInterval, minQuotaTurboSyncInterval, maxQuotaTurboSyncInterval) { + return fmt.Errorf("synchronization time ranges from 100 (0.1s) to 10000 (10s)") + } + return nil +} + +// PreStart is the pre-start action +func (qt *QuotaTurbo) PreStart(viewer api.Viewer) error { + qt.Viewer = viewer + pods := viewer.ListPodsWithOptions() + for _, pod := range pods { + recoverOnePodQuota(pod) + } + return nil +} + +// Terminate enters the service termination process +func (qt *QuotaTurbo) Terminate(viewer api.Viewer) error { + pods := viewer.ListPodsWithOptions() + for _, pod := range pods { + recoverOnePodQuota(pod) + } + return nil +} + +func recoverOnePodQuota(pod *typedef.PodInfo) { + const unlimited = "-1" + if err := pod.SetCgroupAttr(cpuQuotaKey, unlimited); err != nil { + log.Errorf("Fail to set the cpu quota of the pod %v to -1: %v", pod.UID, err) + return + } + + var ( + podQuota int64 = 0 + unlimitedContExistd = false + ) + + for _, cont := range pod.IDContainersMap { + // cpulimit is 0 means no quota limit + if cont.LimitResources[typedef.ResourceCPU] == 0 { + unlimitedContExistd = true + if err := cont.SetCgroupAttr(cpuQuotaKey, unlimited); err != nil { + log.Errorf("Fail to set the cpu quota of the container %v to -1: %v", cont.ID, err) + continue + } + log.Debugf("Set the cpu quota of the container %v to -1", cont.ID) + continue + } + + period, err := cont.GetCgroupAttr(cpuPeriodKey).Int64() + if err != nil { + log.Errorf("Fail to get cpu period of container %v : %v", cont.ID, err) + continue + } + + contQuota := int64(cont.LimitResources[typedef.ResourceCPU] * float64(period)) + podQuota += contQuota + if err := cont.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(contQuota)); err != nil { + log.Errorf("Fail to set the cpu quota of the container %v: %v", cont.ID, err) + continue + } + log.Debugf("Set the cpu quota of the container %v to %v", cont.ID, contQuota) + } + if !unlimitedContExistd { + if err := pod.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(podQuota)); err != nil { + log.Errorf("Fail to set the cpu quota of the pod %v to -1: %v", pod.UID, err) + return + } + log.Debugf("Set the cpu quota of the pod %v to %v", pod.UID, podQuota) + } +} diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go new file mode 100644 index 0000000..cf9c987 --- /dev/null +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -0,0 +1,330 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: This file is used for test quota turbo + +// Package quotaturbo is for Quota Turbo +package quotaturbo + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/podmanager" +) + +// TestQuotaTurbo_Validatec test Validate function +func TestQuotaTurbo_Validate(t *testing.T) { + tests := []struct { + name string + NodeData *NodeData + wantErr bool + }{ + { + name: "TC1-alarmWaterMark is less or equal to highWaterMark", + NodeData: &NodeData{Config: &Config{ + HighWaterMark: 100, + AlarmWaterMark: 60, + SyncInterval: 1000, + }}, + wantErr: true, + }, + { + name: "TC2-highWater mark exceed the max quota turbo water mark(100)", + NodeData: &NodeData{Config: &Config{ + HighWaterMark: 1000, + AlarmWaterMark: 100000, + SyncInterval: 1000, + }}, + wantErr: true, + }, + { + name: "TC3-sync interval out of range(100-10000)", + NodeData: &NodeData{Config: &Config{ + HighWaterMark: 60, + AlarmWaterMark: 80, + SyncInterval: 1, + }}, + wantErr: true, + }, + { + name: "TC4-normal case", + NodeData: &NodeData{Config: &Config{ + HighWaterMark: 60, + AlarmWaterMark: 100, + SyncInterval: 1000, + }}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qt := &QuotaTurbo{ + NodeData: tt.NodeData, + } + if err := qt.Validate(); (err != nil) != tt.wantErr { + t.Errorf("QuotaTurbo.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func sameQuota(t *testing.T, path string, want int64) bool { + const cfsQuotaUsFileName = "cpu.cfs_quota_us" + data, err := util.ReadSmallFile(filepath.Join(path, cfsQuotaUsFileName)) + if err != nil { + assert.NoError(t, err) + return false + } + quota, err := util.ParseInt64(strings.ReplaceAll(string(data), "\n", "")) + if err != nil { + assert.NoError(t, err) + return false + } + if quota != want { + return false + } + return true +} + +// TestQuotaTurbo_Terminate tests Terminate function +func TestQuotaTurbo_Terminate(t *testing.T) { + const ( + fooContName = "Foo" + barContName = "Bar" + podUID = "testPod1" + wrongPodQuota = "600000" + wrongFooQuota = "300000" + wrongBarQuota = "200000" + podPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/" + fooPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon1" + barPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon2" + ) + + var ( + tests = []struct { + postfunc func(t *testing.T) + fooCPULimit float64 + barCPULimit float64 + name string + }{ + { + name: "TC1-one unlimited container is existed", + fooCPULimit: 2, + barCPULimit: 0, + postfunc: func(t *testing.T) { + var ( + unlimited int64 = -1 + correctFooQuota int64 = 200000 + ) + assert.True(t, sameQuota(t, podPath, unlimited)) + assert.True(t, sameQuota(t, fooPath, correctFooQuota)) + assert.True(t, sameQuota(t, barPath, unlimited)) + }, + }, + { + name: "TC2-all containers are unlimited", + fooCPULimit: 2, + barCPULimit: 1, + postfunc: func(t *testing.T) { + var ( + correctPodQuota int64 = 300000 + correctFooQuota int64 = 200000 + correctBarQuota int64 = 100000 + ) + assert.True(t, sameQuota(t, podPath, correctPodQuota)) + assert.True(t, sameQuota(t, fooPath, correctFooQuota)) + assert.True(t, sameQuota(t, barPath, correctBarQuota)) + }, + }, + { + name: "TC3-all containers are limited", + fooCPULimit: 0, + barCPULimit: 0, + postfunc: func(t *testing.T) { + var unLimited int64 = -1 + assert.True(t, sameQuota(t, podPath, unLimited)) + assert.True(t, sameQuota(t, fooPath, unLimited)) + assert.True(t, sameQuota(t, barPath, unLimited)) + }, + }, + } + ) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + fooCont = &typedef.ContainerInfo{ + Name: fooContName, + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: make(typedef.ResourceMap), + } + barCont = &typedef.ContainerInfo{ + Name: barContName, + ID: "testCon2", + CgroupPath: "kubepods/testPod1/testCon2", + LimitResources: make(typedef.ResourceMap), + } + contList = []*typedef.ContainerInfo{ + fooCont, + barCont, + } + pod = &typedef.PodInfo{ + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + IDContainersMap: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + barCont.ID: barCont, + }, + } + pm = &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + podUID: pod, + }, + }, + } + qt = &QuotaTurbo{ + Viewer: pm, + } + ) + + mkCgDirs(contList) + defer rmCgDirs(contList) + + fooCont.LimitResources[typedef.ResourceCPU] = tt.fooCPULimit + barCont.LimitResources[typedef.ResourceCPU] = tt.barCPULimit + + assert.NoError(t, pod.SetCgroupAttr(cpuQuotaKey, wrongPodQuota)) + assert.NoError(t, fooCont.SetCgroupAttr(cpuQuotaKey, wrongFooQuota)) + assert.NoError(t, barCont.SetCgroupAttr(cpuQuotaKey, wrongBarQuota)) + qt.Terminate(pm) + tt.postfunc(t) + }) + } +} + +func TestQuotaTurbo_AdjustQuota(t *testing.T) { + type fields struct { + NodeData *NodeData + Driver Driver + pm *podmanager.PodManager + } + type args struct { + cc map[string]*typedef.ContainerInfo + } + var ( + fooCont = &typedef.ContainerInfo{ + Name: "Foo", + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: 2, + }, + RequestResources: typedef.ResourceMap{ + typedef.ResourceCPU: 2, + }, + } + pod1 = &typedef.PodInfo{ + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + IDContainersMap: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + }, + } + pod2 = &typedef.PodInfo{ + UID: "testPod2", + CgroupPath: "kubepods/testPod2", + IDContainersMap: map[string]*typedef.ContainerInfo{}, + } + ) + + tests := []struct { + name string + fields fields + args args + }{ + { + name: "TC1-empty data", + args: args{ + cc: map[string]*typedef.ContainerInfo{}, + }, + fields: fields{ + NodeData: &NodeData{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 60, + }, + containers: make(map[string]*CPUQuota), + }, + Driver: &EventDriver{}, + pm: &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + pod1.UID: pod1, + pod2.UID: pod2, + }, + }, + }, + }, + }, + { + name: "TC2-existed data", + args: args{ + cc: map[string]*typedef.ContainerInfo{ + "testCon1": fooCont, + }, + }, + fields: fields{ + NodeData: &NodeData{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 60, + }, + containers: make(map[string]*CPUQuota), + }, + Driver: &EventDriver{}, + pm: &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + pod1.UID: pod1, + pod2.UID: pod2, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qt := &QuotaTurbo{ + NodeData: tt.fields.NodeData, + Driver: tt.fields.Driver, + Viewer: tt.fields.pm, + } + conts := []*typedef.ContainerInfo{} + for _, p := range tt.fields.pm.Pods.Pods { + for _, c := range p.IDContainersMap { + conts = append(conts, c) + } + } + mkCgDirs(conts) + defer rmCgDirs(conts) + + qt.AdjustQuota(tt.args.cc) + }) + } +} -- Gitee From 52480e087af17d5a0079c4d8e94eaa929e332f97 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 23 Feb 2023 15:25:50 +0800 Subject: [PATCH 33/73] tests: support quotaTurbo DT test The quota Turbo package achieved 91.8% code line coverage --- pkg/rubik/import.go | 1 + pkg/services/quotaturbo/driverevent.go | 8 +-- pkg/services/quotaturbo/driverevent_test.go | 6 +-- pkg/services/quotaturbo/quotaturbo.go | 1 + pkg/services/quotaturbo/quotaturbo_test.go | 55 +++++++++++++++++++++ 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index 82e65c6..d3bf133 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -19,5 +19,6 @@ import ( _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/services/cachelimit" _ "isula.org/rubik/pkg/services/qos" + _ "isula.org/rubik/pkg/services/quotaturbo" _ "isula.org/rubik/pkg/version" ) diff --git a/pkg/services/quotaturbo/driverevent.go b/pkg/services/quotaturbo/driverevent.go index a13d259..abba6a8 100644 --- a/pkg/services/quotaturbo/driverevent.go +++ b/pkg/services/quotaturbo/driverevent.go @@ -34,8 +34,8 @@ const ( upperLimitOfIncrease = 1.0 // slowFallbackRatio is the rate of slow fallback. slowFallbackRatio = 0.1 - // limitOfNodeCPUUsageChangeWithin1Minute is the upper limit of the node change rate in one minute. - limitOfNodeCPUUsageChangeWithin1Minute float64 = 10 + // cpuUtilMaxChange is the upper limit of the node change rate in one minute. + cpuUtilMaxChange float64 = 10 ) // EventDriver event based quota adjustment driver. @@ -51,7 +51,7 @@ func (e *EventDriver) adjustQuota(data *NodeData) { e.elevate(data) } else { log.Infof("the CPU usage changes by more than %.2f, "+ - "the quota will not adjusted in this round", limitOfNodeCPUUsageChangeWithin1Minute) + "the quota will not adjusted in this round", cpuUtilMaxChange) } for _, c := range data.containers { // get height limit @@ -146,7 +146,7 @@ func sharpFluctuates(data *NodeData) bool { min = math.Min(min, u.util) max = math.Max(max, u.util) } - if max-min > limitOfNodeCPUUsageChangeWithin1Minute { + if max-min > cpuUtilMaxChange { log.Debugf("The resources changed drastically, and no adjustment will be made this time") return true } diff --git a/pkg/services/quotaturbo/driverevent_test.go b/pkg/services/quotaturbo/driverevent_test.go index 5be213d..a951e0d 100644 --- a/pkg/services/quotaturbo/driverevent_test.go +++ b/pkg/services/quotaturbo/driverevent_test.go @@ -379,7 +379,7 @@ func TestSharpFluctuates(t *testing.T) { util: 90, }, { - util: 90 - limitOfNodeCPUUsageChangeWithin1Minute - 1, + util: 90 - cpuUtilMaxChange - 1, }, }, }, @@ -393,7 +393,7 @@ func TestSharpFluctuates(t *testing.T) { util: 90, }, { - util: 90 - limitOfNodeCPUUsageChangeWithin1Minute + 1, + util: 90 - cpuUtilMaxChange + 1, }, }, }, @@ -438,7 +438,7 @@ func TestEventDriverAdjustQuota(t *testing.T) { util: 1, }, { - util: -limitOfNodeCPUUsageChangeWithin1Minute, + util: -cpuUtilMaxChange, }, }, }, diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index 289b332..fcb2702 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -54,6 +54,7 @@ type QuotaTurbo struct { func NewQuotaTurbo() *QuotaTurbo { return &QuotaTurbo{ NodeData: NewNodeData(), + Driver: &EventDriver{}, } } diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index cf9c987..52ca780 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -15,11 +15,15 @@ package quotaturbo import ( + "context" "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + Log "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/podmanager" @@ -217,6 +221,7 @@ func TestQuotaTurbo_Terminate(t *testing.T) { } } +// TestQuotaTurbo_AdjustQuota tests AdjustQuota function func TestQuotaTurbo_AdjustQuota(t *testing.T) { type fields struct { NodeData *NodeData @@ -328,3 +333,53 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { }) } } + +// TestNewQuotaTurbo tests NewQuotaTurbo +func TestNewQuotaTurbo(t *testing.T) { + testName := "TC1-test otherv functions" + t.Run(testName, func(t *testing.T) { + got := NewQuotaTurbo() + got.SetupLog(&Log.EmptyLog{}) + assert.Equal(t, moduleName, got.ID()) + got.Viewer = &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + "testPod1": { + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + Annotations: map[string]string{ + constant.QuotaAnnotationKey: "true", + }, + }, + }, + }, + } + + ctx, cancle := context.WithCancel(context.Background()) + go got.Run(ctx) + time.Sleep(time.Second) + cancle() + }) +} + +// TestQuotaTurbo_PreStart tests PreStart +func TestQuotaTurbo_PreStart(t *testing.T) { + var ( + pm = &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + "testPod1": { + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + IDContainersMap: make(map[string]*typedef.ContainerInfo), + }, + }, + }, + } + qt = &QuotaTurbo{} + ) + testName := "TC1- test Prestart" + t.Run(testName, func(t *testing.T) { + qt.PreStart(pm) + }) +} -- Gitee From 394fe9a8eba189faf370e4cb3518ff18e4b3d55e Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 16 Feb 2023 19:44:12 +0800 Subject: [PATCH 34/73] feat: add cache limit service 1. add perf package used for collect cache-miss info 2. get global cgroup path 3. add cache limit service Signed-off-by: DCCooper <1866858@gmail.com> --- hack/rubik-daemonset.yaml | 15 ++ pkg/common/perf/perf.go | 285 +++++++++++++++++++++ pkg/common/perf/perf_test.go | 70 +++++ pkg/common/util/file.go | 1 + pkg/services/cachelimit/cachelimit.go | 179 ++++++++++--- pkg/services/cachelimit/cachelimit_init.go | 193 ++++++++++++++ pkg/services/cachelimit/dynamic.go | 127 +++++++++ pkg/services/cachelimit/sync.go | 116 +++++++++ 8 files changed, 954 insertions(+), 32 deletions(-) create mode 100644 pkg/common/perf/perf.go create mode 100644 pkg/common/perf/perf_test.go create mode 100644 pkg/services/cachelimit/cachelimit_init.go create mode 100644 pkg/services/cachelimit/dynamic.go create mode 100644 pkg/services/cachelimit/sync.go diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml index e6f6386..1803349 100644 --- a/hack/rubik-daemonset.yaml +++ b/hack/rubik-daemonset.yaml @@ -43,6 +43,21 @@ data: }, "qos": { "subSys": ["cpu", "memory"] + }, + "cacheLimit":{ + "defaultLimitMode": "static", + "adjustInterval": 1000, + "perfDuration": 1000, + "l3Percent": { + "low": 20, + "mid": 30, + "high": 50 + }, + "memBandPercent": { + "low": 10, + "mid": 30, + "high": 50 + } } } --- diff --git a/pkg/common/perf/perf.go b/pkg/common/perf/perf.go new file mode 100644 index 0000000..68fe6e3 --- /dev/null +++ b/pkg/common/perf/perf.go @@ -0,0 +1,285 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: JingRui +// Create: 2022-01-26 +// Description: cgroup perf stats + +// Package perf provide perf functions +package perf + +import ( + "encoding/binary" + "runtime" + "time" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +var ( + hwSupport = false +) + +// Support tell if the os support perf hw pmu events. +func Support() bool { + return hwSupport +} + +// Stat is perf stat info +type Stat struct { + Instructions uint64 + CPUCycles uint64 + CacheMisses uint64 + CacheReferences uint64 + LLCAccess uint64 + LLCMiss uint64 +} + +type cgEvent struct { + cgfd int + cpu int + fds map[string]int + leader int +} + +type eventConfig struct { + eventName string + eType uint32 + config uint64 +} + +func getEventConfig() []eventConfig { + const eight, sixteen = 8, 16 + return []eventConfig{ + { + eventName: "instructions", + eType: unix.PERF_TYPE_HARDWARE, + config: unix.PERF_COUNT_HW_INSTRUCTIONS, + }, + { + eventName: "cycles", + eType: unix.PERF_TYPE_HARDWARE, + config: unix.PERF_COUNT_HW_CPU_CYCLES, + }, + { + eventName: "cachereferences", + eType: unix.PERF_TYPE_HARDWARE, + config: unix.PERF_COUNT_HW_CACHE_REFERENCES, + }, + { + eventName: "cachemiss", + eType: unix.PERF_TYPE_HARDWARE, + config: unix.PERF_COUNT_HW_CACHE_MISSES, + }, + { + eventName: "llcmiss", + eType: unix.PERF_TYPE_HW_CACHE, + config: unix.PERF_COUNT_HW_CACHE_LL | unix.PERF_COUNT_HW_CACHE_OP_READ< maxAdjustInterval { + return fmt.Errorf("adjust interval %d out of range [%d,%d]", + c.Config.AdjustInterval, minAdjustInterval, maxAdjustInterval) + } + if c.Config.PerfDuration < minPerfDur || c.Config.PerfDuration > maxPerfDur { + return fmt.Errorf("perf duration %d out of range [%d,%d]", c.Config.PerfDuration, minPerfDur, maxPerfDur) + } + for _, per := range []int{c.Config.L3Percent.Low, c.Config.L3Percent.Mid, c.Config.L3Percent.High, c.Config.MemBandPercent.Low, + c.Config.MemBandPercent.Mid, c.Config.MemBandPercent.High} { + if per < minPercent || per > maxPercent { + return fmt.Errorf("cache limit percentage %d out of range [%d,%d]", per, minPercent, maxPercent) + } + } + if c.Config.L3Percent.Low > c.Config.L3Percent.Mid || c.Config.L3Percent.Mid > c.Config.L3Percent.High { + return fmt.Errorf("cache limit config L3Percent does not satisfy constraint low<=mid<=high") + } + if c.Config.MemBandPercent.Low > c.Config.MemBandPercent.Mid || c.Config.MemBandPercent.Mid > c.Config.MemBandPercent.High { + return fmt.Errorf("cache limit config MemBandPercent does not satisfy constraint low<=mid<=high") + } return nil } diff --git a/pkg/services/cachelimit/cachelimit_init.go b/pkg/services/cachelimit/cachelimit_init.go new file mode 100644 index 0000000..eb6d3bb --- /dev/null +++ b/pkg/services/cachelimit/cachelimit_init.go @@ -0,0 +1,193 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file will init cache limit directories before services running + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/perf" + "isula.org/rubik/pkg/common/util" +) + +const ( + base2, base16, bitSize = 2, 16, 32 +) + +type limitSet struct { + dir string + level string + l3Percent int + mbPercent int +} + +// InitCacheLimitDir init multi-level cache limit directories +func (c *CacheLimit) InitCacheLimitDir() error { + log.Debugf("init cache limit directory") + const ( + defaultL3PercentMax = 100 + defaultMbPercentMax = 100 + ) + if !perf.Support() { + return fmt.Errorf("perf event need by service %s not supported", c.ID()) + } + if err := checkHostPidns(c.Config.DefaultPidNameSpace); err != nil { + return err + } + if err := checkResctrlPath(c.Config.DefaultResctrlDir); err != nil { + return err + } + numaNum, err := getNUMANum(c.Attr.NumaNodeDir) + if err != nil { + return fmt.Errorf("get NUMA nodes number error: %v", err) + } + c.Attr.NumaNum = numaNum + c.Attr.L3PercentDynamic = c.Config.L3Percent.Low + c.Attr.MemBandPercentDynamic = c.Config.MemBandPercent.Low + + cacheLimitList := []*limitSet{ + c.newCacheLimitSet(levelDynamic, c.Attr.L3PercentDynamic, c.Attr.MemBandPercentDynamic), + c.newCacheLimitSet(levelLow, c.Config.L3Percent.Low, c.Config.MemBandPercent.Low), + c.newCacheLimitSet(levelMiddle, c.Config.L3Percent.Mid, c.Config.MemBandPercent.Mid), + c.newCacheLimitSet(levelHigh, c.Config.L3Percent.High, c.Config.MemBandPercent.High), + c.newCacheLimitSet(levelMax, defaultL3PercentMax, defaultMbPercentMax), + } + + for _, cl := range cacheLimitList { + if err := cl.writeResctrlSchemata(c.Attr.NumaNum); err != nil { + return err + } + } + + log.Debugf("init cache limit directory success") + return nil +} + +func (c *CacheLimit) newCacheLimitSet(level string, l3Per, mbPer int) *limitSet { + return &limitSet{ + level: level, + l3Percent: l3Per, + mbPercent: mbPer, + dir: filepath.Join(filepath.Clean(c.Config.DefaultResctrlDir), resctrlDirPrefix+level), + } +} + +func (cl *limitSet) setDir() error { + if err := os.Mkdir(cl.dir, constant.DefaultDirMode); err != nil && !os.IsExist(err) { + return fmt.Errorf("create cache limit directory error: %v", err) + } + return nil +} + +func (cl *limitSet) writeResctrlSchemata(numaNum int) error { + // get cbm mask like "fffff" means 20 cache way + maskFile := filepath.Join(filepath.Dir(cl.dir), "info", "L3", "cbm_mask") + llc, err := calcLimitedCacheValue(maskFile, cl.l3Percent) + if err != nil { + return fmt.Errorf("get limited cache value from L3 percent error: %v", err) + } + + if err := cl.setDir(); err != nil { + return err + } + schemetaFile := filepath.Join(cl.dir, schemataFileName) + var content string + var l3List, mbList []string + for i := 0; i < numaNum; i++ { + l3List = append(l3List, fmt.Sprintf("%d=%s", i, llc)) + mbList = append(mbList, fmt.Sprintf("%d=%d", i, cl.mbPercent)) + } + l3 := fmt.Sprintf("L3:%s\n", strings.Join(l3List, ";")) + mb := fmt.Sprintf("MB:%s\n", strings.Join(mbList, ";")) + content = l3 + mb + if err := util.WriteFile(schemetaFile, content); err != nil { + return fmt.Errorf("write %s to file %s error: %v", content, schemetaFile, err) + } + + return nil +} + +func getNUMANum(path string) (int, error) { + files, err := filepath.Glob(filepath.Join(path, "node*")) + if err != nil { + return 0, err + } + return len(files), nil +} + +// getBinaryMask get l3 limit mask like "7ff" and transfer it to binary like "111 1111 1111", return binary length 11 +func getBinaryMask(path string) (int, error) { + maskValue, err := util.ReadFile(path) + if err != nil { + return -1, fmt.Errorf("get L3 mask value error: %v", err) + } + + // transfer mask to binary format + decMask, err := strconv.ParseInt(strings.TrimSpace(string(maskValue)), base16, bitSize) + if err != nil { + return -1, fmt.Errorf("transfer L3 mask value %v to decimal format error: %v", string(maskValue), err) + } + return len(strconv.FormatInt(decMask, base2)), nil +} + +// calcLimitedCacheValue calculate number of cache way could be used according to L3 limit percent +func calcLimitedCacheValue(path string, l3Percent int) (string, error) { + l3BinaryMask, err := getBinaryMask(path) + if err != nil { + return "", err + } + ten, hundred, binValue := 10, 100, 0 + binLen := l3BinaryMask * l3Percent / hundred + if binLen == 0 { + binLen = 1 + } + for i := 0; i < binLen; i++ { + binValue = binValue*ten + 1 + } + decValue, err := strconv.ParseInt(strconv.Itoa(binValue), base2, bitSize) + if err != nil { + return "", fmt.Errorf("transfer %v to decimal format error: %v", binValue, err) + } + + return strconv.FormatInt(decValue, base16), nil +} + +func checkHostPidns(path string) error { + ns, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("get pid namespace inode error: %v", err) + } + hostPidInode := "4026531836" + if strings.Trim(ns, "pid:[]") != hostPidInode { + return fmt.Errorf("not share pid ns with host") + } + return nil +} + +func checkResctrlPath(path string) error { + if !util.PathExist(path) { + return fmt.Errorf("resctrl path %v not exist, not support cache limit", path) + } + schemataPath := filepath.Join(path, schemataFileName) + if !util.PathExist(schemataPath) { + return fmt.Errorf("resctrl schemata file %v not exist, check if %v directory is mounted", + schemataPath, path) + } + return nil +} diff --git a/pkg/services/cachelimit/dynamic.go b/pkg/services/cachelimit/dynamic.go new file mode 100644 index 0000000..f5839da --- /dev/null +++ b/pkg/services/cachelimit/dynamic.go @@ -0,0 +1,127 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file is used for dynamic cache limit level setting + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "fmt" + "time" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/perf" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +// StartDynamic will continuously run to detect online pod cache miss and +// limit offline pod cache usage +func (c *CacheLimit) StartDynamic() { + if !c.dynamicExist() { + return + } + + stepMore, stepLess := 5, -50 + needMore := true + limiter := c.newCacheLimitSet(levelDynamic, c.Attr.L3PercentDynamic, c.Attr.MemBandPercentDynamic) + + for _, p := range c.listOnlinePods() { + cacheMiss, llcMiss := getPodCacheMiss(p, c.Config.PerfDuration) + if cacheMiss >= c.Attr.MaxMiss || llcMiss >= c.Attr.MaxMiss { + log.Infof("online pod %v cache miss: %v LLC miss: %v exceeds maxmiss, lower offline cache limit", + p.UID, cacheMiss, llcMiss) + + if err := c.flush(limiter, stepLess); err != nil { + log.Errorf(err.Error()) + } + return + } + if cacheMiss >= c.Attr.MinMiss || llcMiss >= c.Attr.MinMiss { + needMore = false + } + } + + if needMore { + if err := c.flush(limiter, stepMore); err != nil { + log.Errorf(err.Error()) + } + } +} + +func getPodCacheMiss(pod *typedef.PodInfo, perfDu int) (int, int) { + cgroupPath := cgroup.AbsoluteCgroupPath("perf_event", pod.CgroupPath, "") + if !util.PathExist(cgroupPath) { + return 0, 0 + } + + stat, err := perf.CgroupStat(cgroupPath, time.Duration(perfDu)*time.Millisecond) + if err != nil { + return 0, 0 + } + + return int(100.0 * float64(stat.CacheMisses) / (1.0 + float64(stat.CacheReferences))), + int(100.0 * float64(stat.LLCMiss) / (1.0 + float64(stat.LLCAccess))) +} + +func (c *CacheLimit) dynamicExist() bool { + for _, pod := range c.listOfflinePods() { + if err := c.syncLevel(pod); err != nil { + continue + } + if pod.Annotations[constant.CacheLimitAnnotationKey] == levelDynamic { + return true + } + } + return false +} + +func (c *CacheLimit) flush(limitSet *limitSet, step int) error { + var nextPercent = func(value, min, max, step int) int { + value += step + if value < min { + return min + } + if value > max { + return max + } + return value + + } + l3 := nextPercent(c.Attr.L3PercentDynamic, c.Config.L3Percent.Low, c.Config.L3Percent.High, step) + mb := nextPercent(c.Attr.MemBandPercentDynamic, c.Config.MemBandPercent.Low, c.Config.MemBandPercent.High, step) + if c.Attr.L3PercentDynamic == l3 && c.Attr.MemBandPercentDynamic == mb { + return nil + } + log.Infof("flush L3 from %v to %v, Mb from %v to %v", limitSet.l3Percent, l3, limitSet.mbPercent, mb) + limitSet.l3Percent, limitSet.mbPercent = l3, mb + return c.doFlush(limitSet) +} + +func (c *CacheLimit) doFlush(limitSet *limitSet) error { + if err := limitSet.writeResctrlSchemata(c.Attr.NumaNum); err != nil { + return fmt.Errorf("adjust dynamic cache limit to l3:%v mb:%v error: %v", + limitSet.l3Percent, limitSet.mbPercent, err) + } + c.Attr.L3PercentDynamic = limitSet.l3Percent + c.Attr.MemBandPercentDynamic = limitSet.mbPercent + + return nil +} + +func (c *CacheLimit) listOnlinePods() map[string]*typedef.PodInfo { + onlineValue := "false" + return c.Viewer.ListPodsWithOptions(func(pi *typedef.PodInfo) bool { + return pi.Annotations[constant.PriorityAnnotationKey] == onlineValue + }) +} diff --git a/pkg/services/cachelimit/sync.go b/pkg/services/cachelimit/sync.go new file mode 100644 index 0000000..1ab5536 --- /dev/null +++ b/pkg/services/cachelimit/sync.go @@ -0,0 +1,116 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file is used for cache limit sync setting + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "fmt" + "path/filepath" + "strings" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +const ( + levelLow = "low" + levelMiddle = "middle" + levelHigh = "high" + levelMax = "max" + levelDynamic = "dynamic" + + resctrlDirPrefix = "rubik_" + schemataFileName = "schemata" +) + +var validLevel = map[string]bool{ + levelLow: true, + levelMiddle: true, + levelHigh: true, + levelMax: true, + levelDynamic: true, +} + +// SyncCacheLimit will continuously set cache limit with corresponding offline pods +func (c *CacheLimit) SyncCacheLimit() { + for _, p := range c.listOfflinePods() { + if err := c.syncLevel(p); err != nil { + log.Errorf("sync cache limit level err: %v", err) + continue + } + if err := c.writeTasksToResctrl(p); err != nil { + log.Errorf("set cache limit for pod %v err: %v", p.UID, err) + continue + } + } +} + +// writeTasksToResctrl will write tasks running in containers into resctrl group +func (c *CacheLimit) writeTasksToResctrl(pod *typedef.PodInfo) error { + if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", pod.CgroupPath, "")) { + // just return since pod maybe deleted + return nil + } + var taskList []string + cgroupKey := &cgroup.Key{SubSys: "cpu", FileName: "cgroup.procs"} + for _, container := range pod.IDContainersMap { + key := container.GetCgroupAttr(cgroupKey) + if key.Err != nil { + return key.Err + } + taskList = append(taskList, strings.Split(key.Value, "\n")...) + } + if len(taskList) == 0 { + return nil + } + + resctrlTaskFile := filepath.Join(c.Config.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks") + for _, task := range taskList { + if err := util.WriteFile(resctrlTaskFile, task); err != nil { + if strings.Contains(err.Error(), "no such process") { + log.Errorf("pod %s task %s not exist", pod.UID, task) + continue + } + return fmt.Errorf("add task %v to file %v error: %v", task, resctrlTaskFile, err) + } + } + + return nil +} + +// syncLevel sync cache limit level +func (c *CacheLimit) syncLevel(pod *typedef.PodInfo) error { + if pod.Annotations[constant.CacheLimitAnnotationKey] == "" { + if c.Config.DefaultLimitMode == modeStatic { + pod.Annotations[constant.CacheLimitAnnotationKey] = levelMax + } else { + pod.Annotations[constant.CacheLimitAnnotationKey] = levelDynamic + } + } + + level := pod.Annotations[constant.CacheLimitAnnotationKey] + if isValid, ok := validLevel[level]; !ok || !isValid { + return fmt.Errorf("invalid cache limit level %v for pod: %v", level, pod.UID) + } + return nil +} + +func (c *CacheLimit) listOfflinePods() map[string]*typedef.PodInfo { + offlineValue := "true" + return c.Viewer.ListPodsWithOptions(func(pi *typedef.PodInfo) bool { + return pi.Annotations[constant.PriorityAnnotationKey] == offlineValue + }) +} -- Gitee From 7927df2b84f873bbe2b1b18925767c1e8e543445 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 23 Feb 2023 15:14:49 +0800 Subject: [PATCH 35/73] test: add testcase for cacheLimit package 1. package coverage: 87.7% 2. try package: - exporse method genFakePodInfo and genFakeContainerInfo - set and get global test cgroup root path Signed-off-by: DCCooper <1866858@gmail.com> --- .../cachelimit/cachelimit_init_test.go | 223 +++++++++++++ pkg/services/cachelimit/cachelimit_test.go | 310 ++++++++++++++++++ pkg/services/cachelimit/dynamic_test.go | 273 +++++++++++++++ pkg/services/cachelimit/sync_test.go | 251 ++++++++++++++ pkg/services/qos/qos_test.go | 2 +- test/try/pod.go | 23 +- test/try/try.go | 12 + 7 files changed, 1085 insertions(+), 9 deletions(-) create mode 100644 pkg/services/cachelimit/cachelimit_init_test.go create mode 100644 pkg/services/cachelimit/cachelimit_test.go create mode 100644 pkg/services/cachelimit/dynamic_test.go create mode 100644 pkg/services/cachelimit/sync_test.go diff --git a/pkg/services/cachelimit/cachelimit_init_test.go b/pkg/services/cachelimit/cachelimit_init_test.go new file mode 100644 index 0000000..fbdede5 --- /dev/null +++ b/pkg/services/cachelimit/cachelimit_init_test.go @@ -0,0 +1,223 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file will init cache limit directories before services running + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/perf" + "isula.org/rubik/test/try" +) + +func setMaskFile(t *testing.T, resctrlDir string, data string) { + maskDir := filepath.Join(resctrlDir, "info", "L3") + maskFile := filepath.Join(maskDir, "cbm_mask") + try.MkdirAll(maskDir, constant.DefaultDirMode).OrDie() + try.WriteFile(maskFile, data).OrDie() + try.WriteFile(filepath.Join(resctrlDir, schemataFileName), "L3:0=7fff;1=7fff;2=7fff;3=7fff\nMB:0=100;1=100;2=100;3=100").OrDie() +} + +func genNumaNodes(path string, num int) { + for i := 0; i < num; i++ { + try.MkdirAll(filepath.Join(path, fmt.Sprintf("node%d", i)), constant.DefaultDirMode).OrDie() + } +} + +func TestCacheLimit_InitCacheLimitDir(t *testing.T) { + if !perf.Support() { + t.Skipf("%s only run on physical machine", t.Name()) + } + type fields struct { + Config *Config + Attr *Attr + Name string + } + // defaultConfig := genDefaultConfig() + tests := []struct { + name string + fields fields + wantErr bool + preHook func(t *testing.T, c *CacheLimit) + postHook func(t *testing.T, c *CacheLimit) + }{ + { + name: "TC1-normal cache limit dir setting", + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Config.DefaultResctrlDir = try.GenTestDir().String() + c.Config.DefaultLimitMode = modeStatic + setMaskFile(t, c.Config.DefaultResctrlDir, "7fff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + }, + postHook: func(t *testing.T, c *CacheLimit) { + resctrlLevelMap := map[string]string{ + "rubik_max": "L3:0=7fff;1=7fff;2=7fff;3=7fff\nMB:0=100;1=100;2=100;3=100\n", + "rubik_high": "L3:0=7f;1=7f;2=7f;3=7f\nMB:0=50;1=50;2=50;3=50\n", + "rubik_middle": "L3:0=f;1=f;2=f;3=f\nMB:0=30;1=30;2=30;3=30\n", + "rubik_low": "L3:0=7;1=7;2=7;3=7\nMB:0=10;1=10;2=10;3=10\n", + "rubik_dynamic": "L3:0=7;1=7;2=7;3=7\nMB:0=10;1=10;2=10;3=10\n", + } + for level, expect := range resctrlLevelMap { + schemataFile := filepath.Join(c.DefaultResctrlDir, level, schemataFileName) + content := try.ReadFile(schemataFile).String() + assert.Equal(t, expect, content) + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC2-not share with host pid namespace", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + pidNameSpaceDir := try.GenTestDir().String() + pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") + pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") + + os.Symlink(pidNameSpaceFileOri.String(), pidNameSpace) + c.Config.DefaultPidNameSpace = pidNameSpace + }, + postHook: func(t *testing.T, c *CacheLimit) { + try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) + }, + }, + { + name: "TC3-pid namespace file is not link file", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + pidNameSpaceDir := try.GenTestDir().String() + pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") + pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") + + os.Link(pidNameSpaceFileOri.String(), pidNameSpace) + c.Config.DefaultPidNameSpace = pidNameSpace + }, + postHook: func(t *testing.T, c *CacheLimit) { + try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) + }, + }, + { + name: "TC4-resctrl path not exist", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Config.DefaultResctrlDir = "/resctrl/path/is/not/exist" + }, + }, + { + name: "TC5-resctrl schemata file not exist", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Config.DefaultResctrlDir = try.GenTestDir().String() + }, + postHook: func(t *testing.T, c *CacheLimit) { + try.RemoveAll(c.DefaultResctrlDir) + }, + }, + { + name: "TC6-no numa path", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Attr.NumaNodeDir = "/numa/node/path/is/not/exist" + }, + }, + { + name: "TC7-empty cbm mask file", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Config.DefaultResctrlDir = try.GenTestDir().String() + c.Config.DefaultLimitMode = modeStatic + setMaskFile(t, c.Config.DefaultResctrlDir, "") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 0) + }, + postHook: func(t *testing.T, c *CacheLimit) { + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC8-low cmb mask value", + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit) { + c.Config.DefaultResctrlDir = try.GenTestDir().String() + c.Config.DefaultLimitMode = modeStatic + setMaskFile(t, c.Config.DefaultResctrlDir, "1") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 0) + }, + postHook: func(t *testing.T, c *CacheLimit) { + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Name: tt.fields.Name, + } + if tt.preHook != nil { + tt.preHook(t, c) + } + if err := c.InitCacheLimitDir(); (err != nil) != tt.wantErr { + t.Errorf("CacheLimit.InitCacheLimitDir() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.postHook != nil { + tt.postHook(t, c) + } + }) + } +} diff --git a/pkg/services/cachelimit/cachelimit_test.go b/pkg/services/cachelimit/cachelimit_test.go new file mode 100644 index 0000000..35e6783 --- /dev/null +++ b/pkg/services/cachelimit/cachelimit_test.go @@ -0,0 +1,310 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file is testcase for cache limit service + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "reflect" + "strconv" + "strings" + "testing" + + "isula.org/rubik/pkg/api" +) + +func TestCacheLimit_Validate(t *testing.T) { + type fields struct { + Config *Config + Attr *Attr + Name string + } + tests := []struct { + name string + fields fields + wantErr bool + wantMsg string + }{ + { + name: "TC-static mode config", + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: minAdjustInterval + 1, + PerfDuration: minPerfDur + 1, + L3Percent: MultiLvlPercent{ + Low: minPercent + 1, + Mid: maxPercent/2 + 1, + High: maxPercent - 1, + }, + MemBandPercent: MultiLvlPercent{ + Low: minPercent + 1, + Mid: maxPercent/2 + 1, + High: maxPercent - 1, + }, + }, + }, + }, + { + name: "TC-invalid mode config", + wantErr: true, + wantMsg: modeDynamic, + fields: fields{ + Config: &Config{ + DefaultLimitMode: "invalid mode", + }, + }, + }, + { + name: "TC-invalid adjust interval less than min value", + wantErr: true, + wantMsg: strconv.Itoa(minAdjustInterval), + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: minAdjustInterval - 1, + }, + }, + }, + { + name: "TC-invalid adjust interval greater than max value", + wantErr: true, + wantMsg: strconv.Itoa(maxAdjustInterval), + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval + 1, + }, + }, + }, + { + name: "TC-invalid perf duration less than min value", + wantErr: true, + wantMsg: strconv.Itoa(minPercent), + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval/2 + 1, + PerfDuration: minPerfDur - 1, + }, + }, + }, + { + name: "TC-invalid perf duration greater than max value", + wantErr: true, + wantMsg: strconv.Itoa(maxPerfDur), + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval/2 + 1, + PerfDuration: maxPerfDur + 1, + }, + }, + }, + { + name: "TC-invalid percent value", + wantErr: true, + wantMsg: strconv.Itoa(minPercent), + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval/2 + 1, + PerfDuration: maxPerfDur/2 + 1, + L3Percent: MultiLvlPercent{ + Low: minPerfDur - 1, + }, + }, + }, + }, + { + name: "TC-invalid l3 percent low value larger than mid value", + wantErr: true, + wantMsg: "low<=mid<=high", + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval/2 + 1, + PerfDuration: maxPerfDur/2 + 1, + L3Percent: MultiLvlPercent{ + Low: minPercent + 2, + Mid: minPercent + 1, + High: minPercent + 1, + }, + MemBandPercent: MultiLvlPercent{ + Low: minPercent, + Mid: minPercent + 1, + High: minPercent + 2, + }, + }, + }, + }, + { + name: "TC-invalid memband percent mid value larger than high value", + wantErr: true, + wantMsg: "low<=mid<=high", + fields: fields{ + Config: &Config{ + DefaultLimitMode: modeStatic, + AdjustInterval: maxAdjustInterval/2 + 1, + PerfDuration: maxPerfDur/2 + 1, + L3Percent: MultiLvlPercent{ + Low: minPercent, + Mid: minPercent + 1, + High: minPercent + 2, + }, + MemBandPercent: MultiLvlPercent{ + Low: minPercent, + Mid: maxPercent/2 + 1, + High: maxPercent / 2, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Name: tt.fields.Name, + } + err := c.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("CacheLimit.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil && !strings.Contains(err.Error(), tt.wantMsg) { + t.Errorf("CacheLimit.Validate() error = %v, wantMsg %v", err, tt.wantMsg) + } + }) + } +} + +func TestNewCacheLimit(t *testing.T) { + tests := []struct { + name string + want *CacheLimit + }{ + { + name: "TC-do nothing", + want: &CacheLimit{ + Name: moduleName, + Attr: &Attr{ + NumaNodeDir: defaultNumaNodeDir, + MaxMiss: defaultMaxMiss, + MinMiss: defaultMinMiss, + }, + Config: &Config{ + DefaultLimitMode: modeStatic, + DefaultResctrlDir: defaultResctrlDir, + DefaultPidNameSpace: defaultPidNameSpace, + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{ + Low: defaultLowL3, + Mid: defaultMidL3, + High: defaultHighL3, + }, + MemBandPercent: MultiLvlPercent{ + Low: defaultLowMB, + Mid: defaultMidMB, + High: defaultHighMB, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewCacheLimit(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewCacheLimit() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCacheLimit_PreStart(t *testing.T) { + type fields struct { + Config *Config + Attr *Attr + Viewer api.Viewer + Name string + } + type args struct { + viewer api.Viewer + } + tests := []struct { + name string + fields fields + args args + wantErr bool + preHook func(t *testing.T, c *CacheLimit) + postHook func(t *testing.T, c *CacheLimit) + }{ + { + name: "TC-just call function", + wantErr: true, + fields: fields{ + Config: genDefaultConfig(), + Attr: &Attr{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Viewer: tt.fields.Viewer, + Name: tt.fields.Name, + } + if err := c.PreStart(tt.args.viewer); (err != nil) != tt.wantErr { + t.Errorf("CacheLimit.PreStart() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestCacheLimit_ID(t *testing.T) { + type fields struct { + Config *Config + Attr *Attr + Viewer api.Viewer + Name string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "TC-return service's name", + fields: fields{ + Name: "cacheLimit", + }, + want: "cacheLimit", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Viewer: tt.fields.Viewer, + Name: tt.fields.Name, + } + if got := c.ID(); got != tt.want { + t.Errorf("CacheLimit.ID() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/services/cachelimit/dynamic_test.go b/pkg/services/cachelimit/dynamic_test.go new file mode 100644 index 0000000..2f1dde2 --- /dev/null +++ b/pkg/services/cachelimit/dynamic_test.go @@ -0,0 +1,273 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file is testcases for dynamic cache limit level setting + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "fmt" + "math" + "os" + "testing" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/perf" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +func TestCacheLimit_StartDynamic(t *testing.T) { + if !perf.Support() { + t.Skipf("%s only run on physical machine", t.Name()) + } + try.InitTestCGRoot(constant.DefaultCgroupRoot) + type fields struct { + Config *Config + Attr *Attr + Name string + FakePods []*try.FakePod + } + tests := []struct { + name string + fields fields + preHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) + postHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) + }{ + { + name: "TC-normal dynamic setting", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + }, + Config: genDefaultConfig(), + Attr: &Attr{ + MaxMiss: 20, + MinMiss: 10, + }, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + resctrlDir := try.GenTestDir().String() + setMaskFile(t, resctrlDir, "3ff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + c.Config.DefaultResctrlDir = resctrlDir + c.Config.DefaultLimitMode = modeDynamic + c.Config.PerfDuration = 10 + for _, pod := range fakePods { + if pod.Annotations[constant.PriorityAnnotationKey] == "true" { + pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" + } + } + manager := genPodManager(fakePods) + c.Viewer = manager + }, + postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath() + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC-max and min miss both 0", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + }, + Config: genDefaultConfig(), + Attr: &Attr{ + MaxMiss: 0, + MinMiss: 0, + }, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + resctrlDir := try.GenTestDir().String() + setMaskFile(t, resctrlDir, "3ff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + c.Config.DefaultResctrlDir = resctrlDir + c.Config.DefaultLimitMode = modeDynamic + c.Config.PerfDuration = 10 + for _, pod := range fakePods { + if pod.Annotations[constant.PriorityAnnotationKey] == "true" { + pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" + } + } + manager := genPodManager(fakePods) + c.Viewer = manager + }, + postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath() + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC-start dynamic with very high water line", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + }, + Config: genDefaultConfig(), + Attr: &Attr{ + MaxMiss: math.MaxInt64, + MinMiss: math.MaxInt64, + }, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + resctrlDir := try.GenTestDir().String() + setMaskFile(t, resctrlDir, "3ff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + c.Config.DefaultResctrlDir = resctrlDir + c.Config.DefaultLimitMode = modeDynamic + c.Config.PerfDuration = 10 + for _, pod := range fakePods { + if pod.Annotations[constant.PriorityAnnotationKey] == "true" { + pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" + } + } + manager := genPodManager(fakePods) + c.Viewer = manager + }, + postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath() + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC-start dynamic with low min water line", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + }, + Config: genDefaultConfig(), + Attr: &Attr{ + MaxMiss: math.MaxInt64, + MinMiss: 0, + }, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + resctrlDir := try.GenTestDir().String() + setMaskFile(t, resctrlDir, "3ff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + c.Config.DefaultResctrlDir = resctrlDir + c.Config.DefaultLimitMode = modeDynamic + c.Config.PerfDuration = 10 + for _, pod := range fakePods { + if pod.Annotations[constant.PriorityAnnotationKey] == "true" { + pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" + } + } + manager := genPodManager(fakePods) + c.Viewer = manager + }, + postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath() + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + { + name: "TC-dynamic not exist", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + try.GenFakeOnlinePod(map[*cgroup.Key]string{ + {SubSys: "perf_event", FileName: "tasks"}: fmt.Sprintf("%d", os.Getegid()), + }), + }, + Config: genDefaultConfig(), + Attr: &Attr{ + MaxMiss: 20, + MinMiss: 10, + }, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + resctrlDir := try.GenTestDir().String() + setMaskFile(t, resctrlDir, "3ff") + numaNodeDir := try.GenTestDir().String() + c.Attr.NumaNodeDir = numaNodeDir + genNumaNodes(c.Attr.NumaNodeDir, 4) + c.Config.DefaultResctrlDir = resctrlDir + c.Config.DefaultLimitMode = modeDynamic + c.Config.PerfDuration = 10 + for _, pod := range fakePods { + if pod.Annotations[constant.PriorityAnnotationKey] == "true" { + pod.Annotations[constant.CacheLimitAnnotationKey] = "static" + } + } + manager := genPodManager(fakePods) + c.Viewer = manager + }, + postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath() + } + try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.Attr.NumaNodeDir) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Name: tt.fields.Name, + } + if tt.preHook != nil { + tt.preHook(t, c, tt.fields.FakePods) + } + c.StartDynamic() + if tt.postHook != nil { + tt.postHook(t, c, tt.fields.FakePods) + } + }) + } +} diff --git a/pkg/services/cachelimit/sync_test.go b/pkg/services/cachelimit/sync_test.go new file mode 100644 index 0000000..061394c --- /dev/null +++ b/pkg/services/cachelimit/sync_test.go @@ -0,0 +1,251 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Xiang Li +// Create: 2023-02-21 +// Description: This file is testcase for cache limit sync function + +// Package cachelimit is the service used for cache limit setting +package cachelimit + +import ( + "path/filepath" + "testing" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/podmanager" + "isula.org/rubik/test/try" +) + +func genDefaultConfig() *Config { + return &Config{ + DefaultLimitMode: modeStatic, + DefaultResctrlDir: defaultResctrlDir, + DefaultPidNameSpace: defaultPidNameSpace, + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{Low: defaultLowL3, Mid: defaultMidL3, High: defaultHighL3}, + MemBandPercent: MultiLvlPercent{Low: defaultLowMB, Mid: defaultMidMB, High: defaultHighMB}, + } +} + +func genPodManager(fakePods []*try.FakePod) *podmanager.PodManager { + pm := &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: make(map[string]*typedef.PodInfo, 0), + }, + } + for _, pod := range fakePods { + pm.Pods.Pods[pod.UID] = pod.PodInfo + } + return pm +} + +func cleanFakePods(fakePods []*try.FakePod) { + for _, pod := range fakePods { + pod.CleanPath().OrDie() + } +} + +func TestCacheLimit_SyncCacheLimit(t *testing.T) { + resctrlDir := try.GenTestDir().String() + defer try.RemoveAll(resctrlDir) + try.InitTestCGRoot(try.TestRoot) + defaultConfig := genDefaultConfig() + defaultConfig.DefaultResctrlDir = resctrlDir + type fields struct { + Config *Config + Attr *Attr + Name string + FakePods []*try.FakePod + } + + tests := []struct { + name string + fields fields + preHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) + }{ + { + name: "TC1-normal case", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "low" + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + }, + }, + { + name: "TC2-empty annotation with static mode config", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + }, + }, + { + name: "TC3-empty annotation with dynamic mode config", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + c.Config.DefaultLimitMode = levelDynamic + }, + }, + { + name: "TC4-invalid annotation", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "invalid" + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + }, + }, + { + name: "TC5-pod just deleted", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "low" + try.RemoveAll(cgroup.AbsoluteCgroupPath("cpu", pod.CgroupPath, "")) + } + c.Viewer = manager + }, + }, + { + name: "TC6-pod without cgroup.procs", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "aaa"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "low" + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + }, + }, + { + name: "TC7-pod without containers", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "low" + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + } + c.Viewer = manager + }, + }, + { + name: "TC8-invalid resctrl group path", + fields: fields{ + FakePods: []*try.FakePod{ + try.GenFakeOfflinePod(map[*cgroup.Key]string{ + {SubSys: "cpu", FileName: "cgroup.procs"}: "12345", + }).WithContainers(1), + }, + Config: defaultConfig, + Attr: &Attr{}, + }, + preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + manager := genPodManager(fakePods) + for _, pod := range manager.Pods.Pods { + pod.Annotations[constant.CacheLimitAnnotationKey] = "low" + } + c.Viewer = manager + c.Config.DefaultResctrlDir = "/dev/null" + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CacheLimit{ + Config: tt.fields.Config, + Attr: tt.fields.Attr, + Name: tt.fields.Name, + } + if tt.preHook != nil { + tt.preHook(t, c, tt.fields.FakePods) + } + c.SyncCacheLimit() + cleanFakePods(tt.fields.FakePods) + }) + } + try.RemoveAll(resctrlDir) +} diff --git a/pkg/services/qos/qos_test.go b/pkg/services/qos/qos_test.go index 92106a1..77899a1 100644 --- a/pkg/services/qos/qos_test.go +++ b/pkg/services/qos/qos_test.go @@ -26,7 +26,7 @@ import ( ) func init() { - cgroup.InitMountDir(try.TestRoot) + try.InitTestCGRoot(try.TestRoot) } type fields struct { diff --git a/test/try/pod.go b/test/try/pod.go index 6150cf2..5916a94 100644 --- a/test/try/pod.go +++ b/test/try/pod.go @@ -31,24 +31,27 @@ import ( type FakePod struct { *typedef.PodInfo // Keys is cgroup key list - Keys map[*cgroup.Key]string + Keys map[*cgroup.Key]string + CGRoot string } const idLen = 8 -func genFakeContainerInfo(parentCGPath string) *typedef.ContainerInfo { +// GenFakeContainerInfo will only generate fake container info under specific pod +func GenFakeContainerInfo(pod *FakePod) *typedef.ContainerInfo { containerID := genContainerID() var fakeContainer = &typedef.ContainerInfo{ Name: fmt.Sprintf("fakeContainer-%s", containerID[:idLen]), ID: containerID, - CgroupPath: filepath.Join(parentCGPath, containerID), + CgroupPath: filepath.Join(pod.CgroupPath, containerID), RequestResources: make(typedef.ResourceMap, 0), LimitResources: make(typedef.ResourceMap, 0), } return fakeContainer } -func genFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { +// GenFakePodInfo will only generate fake pod info but no cgroup files been +func GenFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { podID := uuid.New().String() // generate fake pod info var fakePod = &typedef.PodInfo{ @@ -65,7 +68,8 @@ func genFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { func NewFakePod(keys map[*cgroup.Key]string, qosClass corev1.PodQOSClass) *FakePod { return &FakePod{ Keys: keys, - PodInfo: genFakePodInfo(qosClass), + PodInfo: GenFakePodInfo(qosClass), + CGRoot: GetTestCGRoot(), } } @@ -73,7 +77,7 @@ func (pod *FakePod) genFakePodCgroupPath() Ret { if !util.PathExist(TestRoot) { MkdirAll(TestRoot, constant.DefaultDirMode).OrDie() } - cgroup.InitMountDir(TestRoot) + cgroup.InitMountDir(pod.CGRoot) // generate fake cgroup path for key, value := range pod.Keys { // generate pod absolute cgroup path @@ -106,7 +110,7 @@ func (pod *FakePod) genFakeContainersCgroupPath() Ret { func (pod *FakePod) WithContainers(containerNum int) *FakePod { pod.IDContainersMap = make(map[string]*typedef.ContainerInfo, containerNum) for i := 0; i < containerNum; i++ { - fakeContainer := genFakeContainerInfo(pod.CgroupPath) + fakeContainer := GenFakeContainerInfo(pod) pod.IDContainersMap[fakeContainer.ID] = fakeContainer } pod.genFakeContainersCgroupPath() @@ -120,7 +124,10 @@ func (pod *FakePod) CleanPath() Ret { } for key := range pod.Keys { path := cgroup.AbsoluteCgroupPath(key.SubSys, pod.CgroupPath, key.FileName) - if err := RemoveAll(filepath.Dir(path)); err.err != nil { + if len(key.FileName) != 0 { + path = filepath.Dir(path) + } + if err := RemoveAll(path); err.err != nil { return err } } diff --git a/test/try/try.go b/test/try/try.go index 438665f..2a80abb 100644 --- a/test/try/try.go +++ b/test/try/try.go @@ -126,6 +126,18 @@ const ( TestRoot = "/tmp/rubik-test" ) +var rootDir = TestRoot + +// InitTestCGRoot sets the directory of the cgroup file system for testcases +func InitTestCGRoot(arg string) { + rootDir = arg +} + +// GetTestCGRoot return the directory of the cgroup file system for testcases +func GetTestCGRoot() string { + return rootDir +} + // GenTestDir gen testdir func GenTestDir() Ret { path := filepath.Join(TestRoot, uuid.New().String()) -- Gitee From 408624a718d2d40000693132d6c2128e43e89cf4 Mon Sep 17 00:00:00 2001 From: wujing Date: Thu, 23 Feb 2023 19:37:12 +0800 Subject: [PATCH 36/73] license: add MulanPSL2 license and third party notice Signed-off-by: wujing --- License/LICENSE | 127 ++++++ ...Third_Party_Open_Source_Software_Notice.md | 361 ++++++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 License/LICENSE create mode 100644 License/Third_Party_Open_Source_Software_Notice.md diff --git a/License/LICENSE b/License/LICENSE new file mode 100644 index 0000000..9e32cde --- /dev/null +++ b/License/LICENSE @@ -0,0 +1,127 @@ + 木兰宽松许可证, 第2版 + + 木兰宽松许可证， 第2版 + 2020年1月 http://license.coscl.org.cn/MulanPSL2 + + + 您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第2版（“本许可证”）的如下条款的约束： + + 0. 定义 + + “软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 + + “贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 + + “贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 + + “法人实体”是指提交贡献的机构及其“关联实体”。 + + “关联实体”是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 + + 1. 授予版权许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、分发其“贡献”，不论修改与否。 + + 2. 授予专利许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求，不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“关联实体”直接或间接地，就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 + + 3. 无商标许可 + + “本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用除外。 + + 4. 分发限制 + + 您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。 + + 5. 免责声明与责任限制 + + “软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于何种法律理论，即使其曾被建议有此种损失的可能性。 + + 6. 语言 + “本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致，以中文版为准。 + + 条款结束 + + 如何将木兰宽松许可证，第2版，应用到您的软件 + + 如果您希望将木兰宽松许可证，第2版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步： + + 1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字； + + 2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中； + + 3， 请将如下声明文本放入每个源文件的头部注释中。 + + Copyright (c) [Year] [name of copyright holder] + [Software Name] is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. + + + Mulan Permissive Software License，Version 2 + + Mulan Permissive Software License，Version 2 (Mulan PSL v2) + January 2020 http://license.coscl.org.cn/MulanPSL2 + + Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions: + + 0. Definition + + Software means the program and related documents which are licensed under this License and comprise all Contribution(s). + + Contribution means the copyrightable work licensed by a particular Contributor under this License. + + Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. + + Legal Entity means the entity making a Contribution and all its Affiliates. + + Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. + + 1. Grant of Copyright License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. + + 2. Grant of Patent License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. + + 3. No Trademark License + + No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4. + + 4. Distribution Restriction + + You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. + + 5. Disclaimer of Warranty and Limitation of Liability + + THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 6. Language + + THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL. + + END OF THE TERMS AND CONDITIONS + + How to Apply the Mulan Permissive Software License，Version 2 (Mulan PSL v2) to Your Software + + To apply the Mulan PSL v2 to your work, for easy identification by recipients, you are suggested to complete following three steps: + + i Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; + + ii Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; + + iii Attach the statement to the appropriate annotated syntax at the beginning of each source file. + + + Copyright (c) [Year] [name of copyright holder] + [Software Name] is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. diff --git a/License/Third_Party_Open_Source_Software_Notice.md b/License/Third_Party_Open_Source_Software_Notice.md new file mode 100644 index 0000000..515f6f4 --- /dev/null +++ b/License/Third_Party_Open_Source_Software_Notice.md @@ -0,0 +1,361 @@ +**OPEN SOURCE SOFTWARE NOTICE** + +Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country. + +**Warranty Disclaimer** + +THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +**Copyright Notice and License Texts** + +**Software**: github.com/cyphar/filepath-securejoin v0.2.3 + +**Copyright notice:** + +Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. +Copyright (C) 2017 SUSE LLC. All rights reserved. + +**License:** BSD 3-clause + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**Software**: github.com/pkg/errors v0.9.1 + +**Copyright notice:** + +Copyright (c) 2015, Dave Cheney + +**License:** BSD-2-Clause + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**Software**: github.com/stretchr/testify v1.6.1 + +**Copyright notice:** + +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + +**License:** MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +**Software**: golang.org/x/sys v0.0.0-20201112073958-5cba982894dd + +**Copyright notice:** + +Copyright (c) 2009 The Go Authors. All rights reserved. + +**License:** + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**Software**: k8s.io/api v0.20.2 + +**Copyright notice:** + +Copyright The Kubernetes Authors. + +**License:** Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +**Software**: k8s.io/apimachinery v0.20.2 + +**Copyright notice:** + +Copyright The Kubernetes Authors. + +**License:** Apache License Version 2.0 + +Please see above. + +**Software**: k8s.io/client-go v0.20.2 + +**Copyright notice:** + +Copyright The Kubernetes Authors. + +**License:** Apache License Version 2.0 + +Please see above. -- Gitee From 6b5ba566ca4d6e560d0f84a346c8b82df2b6c530 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 2 Mar 2023 16:23:11 +0800 Subject: [PATCH 37/73] fix: fix logDriver default value Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index e8b1048..ac64fbe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -49,7 +49,7 @@ func NewConfig(pType parserType) *Config { c := &Config{ ConfigParser: defaultParserFactory.getParser(pType), Agent: &AgentConfig{ - LogDriver: constant.DefaultLogDir, + LogDriver: constant.LogDriverStdio, LogSize: constant.DefaultLogSize, LogLevel: constant.DefaultLogLevel, LogDir: constant.DefaultLogDir, -- Gitee From f04fe373b2a0860b4cb1a4fd471d27c503f6308c Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 2 Mar 2023 16:23:35 +0800 Subject: [PATCH 38/73] fix: modify default yaml config 1. adjust default log level from debug to info 2. no more offer default cache limit config since rubik will fail to run when env is not support resctrl control group Signed-off-by: DCCooper <1866858@gmail.com> --- hack/rubik-daemonset.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml index 1803349..9ae80d7 100644 --- a/hack/rubik-daemonset.yaml +++ b/hack/rubik-daemonset.yaml @@ -38,26 +38,11 @@ data: "logDriver": "stdio", "logDir": "/var/log/rubik", "logSize": 1024, - "logLevel": "debug", + "logLevel": "info", "cgroupRoot": "/sys/fs/cgroup" }, "qos": { "subSys": ["cpu", "memory"] - }, - "cacheLimit":{ - "defaultLimitMode": "static", - "adjustInterval": 1000, - "perfDuration": 1000, - "l3Percent": { - "low": 20, - "mid": 30, - "high": 50 - }, - "memBandPercent": { - "low": 10, - "mid": 30, - "high": 50 - } } } --- -- Gitee From 5ce7ded94730f05d63fc2eccd0a9f49228eff728 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 2 Mar 2023 20:13:04 +0800 Subject: [PATCH 39/73] refactor: print config before agent running Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/config/config.go | 9 +++++++++ pkg/rubik/rubik.go | 1 + 2 files changed, 10 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index ac64fbe..d2e7bb1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,6 +15,7 @@ package config import ( + "encoding/json" "fmt" "io/ioutil" @@ -95,6 +96,14 @@ func (c *Config) LoadConfig(path string) error { return nil } +func (c *Config) String() string { + data, err := json.MarshalIndent(c.Fields, "", " ") + if err != nil { + return "{}" + } + return fmt.Sprintf("%s", string(data)) +} + // filterNonServiceKeys returns true when inputting a non-service name func (c *Config) filterNonServiceKeys(name string) bool { // 1. ignore system configured key values diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 739ec8d..508f7fa 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -60,6 +60,7 @@ func NewAgent(cfg *config.Config) (*Agent, error) { // Run starts and runs the agent until receiving stop signal func (a *Agent) Run(ctx context.Context) error { + log.Infof("agent run with config:\n%s", a.config.String()) if err := a.startServiceHandler(ctx); err != nil { return err } -- Gitee From d0477896eeab88cb5f479baa78c463760c061112 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 2 Mar 2023 22:12:20 +0800 Subject: [PATCH 40/73] feature: support quota burst The quota burst feature is implemented through the internal interface of the cpu burst. The quota burst allows the business container to use an additional cpu time slice above the existing cpulimit quota. DT coverage achieves 97.0% --- pkg/rubik/import.go | 1 + pkg/services/quotaburst/quotaburst.go | 205 ++++++++++++++++ pkg/services/quotaburst/quotaburst_test.go | 261 +++++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 pkg/services/quotaburst/quotaburst.go create mode 100644 pkg/services/quotaburst/quotaburst_test.go diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index d3bf133..dbb9b19 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -19,6 +19,7 @@ import ( _ "isula.org/rubik/pkg/services/blkio" _ "isula.org/rubik/pkg/services/cachelimit" _ "isula.org/rubik/pkg/services/qos" + _ "isula.org/rubik/pkg/services/quotaburst" _ "isula.org/rubik/pkg/services/quotaturbo" _ "isula.org/rubik/pkg/version" ) diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go new file mode 100644 index 0000000..1d9258c --- /dev/null +++ b/pkg/services/quotaburst/quotaburst.go @@ -0,0 +1,205 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-01 +// Description: This file is used for quota burst + +// Package quotaburst is for Quota Burst +package quotaburst + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/services" +) + +const ( + moduleName = "quotaburst" +) + +var log api.Logger + +func init() { + log = &Log.EmptyLog{} + services.Register(moduleName, func() interface{} { + return NewBurst() + }) +} + +// Burst is used to control cpu burst +type Burst struct { + Name string `json:"-"` +} + +// NewBurst return an new Burst pointer +func NewBurst() *Burst { + return &Burst{ + Name: moduleName, + } +} + +// SetupLog initializes the log interface for the module +func (b *Burst) SetupLog(logger api.Logger) { + log = logger +} + +// ID returns the module name +func (b *Burst) ID() string { + return moduleName +} + +// AddFunc implement add function when pod is added in k8s +func (conf *Burst) AddFunc(podInfo *typedef.PodInfo) error { + return setPodQuotaBurst(podInfo) +} + +// UpdateFunc implement update function when pod info is changed +func (conf *Burst) UpdateFunc(oldPod, newPod *typedef.PodInfo) error { + if oldPod.Annotations[constant.QuotaBurstAnnotationKey] == newPod.Annotations[constant.QuotaBurstAnnotationKey] { + return nil + } + return setPodQuotaBurst(newPod) +} + +// DeleteFunc implement delete function when pod is deleted by k8s +func (conf *Burst) DeleteFunc(podInfo *typedef.PodInfo) error { + return nil +} + +// PreStart is the pre-start action +func (conf *Burst) PreStart(viewer api.Viewer) error { + pods := viewer.ListPodsWithOptions() + for _, pod := range pods { + if err := setPodQuotaBurst(pod); err != nil { + log.Errorf("error prestart cont %v: %v", pod.Name, err) + } + } + return nil +} + +func setPodQuotaBurst(podInfo *typedef.PodInfo) error { + if podInfo.Annotations[constant.QuotaBurstAnnotationKey] == "" { + return nil + } + burst, err := parseQuotaBurst(podInfo) + if err != nil { + return err + } + var podBurst int64 = 0 + const subsys = "cpu" + // 1. Try to write container burst value firstly + for _, c := range podInfo.IDContainersMap { + cgpath := cgroup.AbsoluteCgroupPath(subsys, c.CgroupPath, "") + if err := setQuotaBurst(burst, cgpath); err != nil { + log.Errorf("set container quota burst failed: %v", err) + continue + } + /* + Only when the burst value of the container is successfully set, + the burst value of the pod will be accumulated. + Ensure that Pod data must be written successfully + */ + podBurst += burst + } + // 2. Try to write pod burst value + podPath := cgroup.AbsoluteCgroupPath(subsys, podInfo.CgroupPath, "") + if err := setQuotaBurst(podBurst, podPath); err != nil { + log.Errorf("set pod quota burst failed: %v", err) + } + return nil +} + +func setQuotaBurst(burst int64, cgpath string) error { + const burstFileName = "cpu.cfs_burst_us" + fpath := filepath.Join(cgpath, burstFileName) + // check whether cgroup support cpu burst + if _, err := os.Stat(fpath); err != nil && os.IsNotExist(err) { + return fmt.Errorf("quota-burst path=%v missing", fpath) + } + if err := matchQuota(burst, cgpath); err != nil { + return err + } + // try to write cfs_burst_us + if err := ioutil.WriteFile(fpath, []byte(util.FormatInt64(burst)), constant.DefaultFileMode); err != nil { + return fmt.Errorf("quota-burst path=%v setting failed: %v", fpath, err) + } + log.Infof("quota-burst path=%v setting success", fpath) + return nil +} + +func matchQuota(burst int64, cgpath string) error { + const ( + cpuPeriodFileName = "cpu.cfs_period_us" + cpuQuotaFileName = "cpu.cfs_quota_us" + ) + quotaStr, err := util.ReadSmallFile(filepath.Join(cgpath, cpuQuotaFileName)) + if err != nil { + return fmt.Errorf("fail to read cfs.cpu_quota_us: %v", err) + } + quota, err := util.ParseInt64(strings.TrimSpace(string(quotaStr))) + if err != nil { + return fmt.Errorf("fail to parse quota as int64: %v", err) + } + + periodStr, err := util.ReadSmallFile(filepath.Join(cgpath, cpuPeriodFileName)) + if err != nil { + return fmt.Errorf("fail to read cfs.cpu_period_us: %v", err) + } + period, err := util.ParseInt64(strings.TrimSpace(string(periodStr))) + if err != nil { + return fmt.Errorf("fail to parse period as int64: %v", err) + } + + /* + The current pod has been allowed to use all cores, usually there are two situations: + 1.the pod quota is -1 (in this case, there must be a container with a quota of -1) + 2.the pod quota exceeds the maximum value (the cumulative quota value of all containers + exceeds the maximum value) + */ + maxQuota := period * int64(runtime.NumCPU()) + if quota >= maxQuota { + return fmt.Errorf("burst fail when quota exceed the maxQuota") + } + /* + All containers under the pod have set cpulimit, and the cumulative value is less than the maximum core. + At this time, the quota of the pod should be the accumulated value of the quota of all pods. + If the burst value of the container is set successfully, then the burst value of the Pod + must be set successfully + */ + if quota < burst { + return fmt.Errorf("burst should be less than or equal to quota") + } + return nil +} + +// parseQuotaBurst checks CPU quota burst annotation value. +func parseQuotaBurst(pod *typedef.PodInfo) (int64, error) { + const invalidVal int64 = -1 + val, err := util.ParseInt64(pod.Annotations[constant.QuotaBurstAnnotationKey]) + if err != nil { + return invalidVal, err + } + + if val < 0 { + return invalidVal, fmt.Errorf("quota burst value should be positive") + } + return val, nil +} diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go new file mode 100644 index 0000000..d311c33 --- /dev/null +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -0,0 +1,261 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-02 +// Description: This file is used for testing quota burst + +// Package quotaburst is for Quota Burst +package quotaburst + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/podmanager" + "isula.org/rubik/test/try" +) + +var ( + cfsBurstUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_burst_us"} + cfsQuotaUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} + cfsPeriodUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} +) + +// TestBurst_AddFunc tests AddFunc +func TestBurst_AddFunc(t *testing.T) { + type args struct { + pod *try.FakePod + burst string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "TC-1: set burst successfully", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsPeriodUs: "100000", + cfsQuotaUs: "100000", + }).WithContainers(1), + burst: "1000", + }, + wantErr: false, + }, + { + name: "TC-2.1: parseQuotaBurst invalid burst < 0", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{}), + burst: "-100", + }, + wantErr: true, + }, + { + name: "TC-2.2: parseQuotaBurst invalid burst non int64", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{}), + burst: "abc", + }, + wantErr: true, + }, + { + name: "TC-3.1: matchQuota quota file not existed", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + }).WithContainers(1), + burst: "10000", + }, + wantErr: false, + }, + { + name: "TC-3.2: matchQuota quota value invalid", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsQuotaUs: "abc", + }), + burst: "10000", + }, + wantErr: false, + }, + { + name: "TC-3.3: matchQuota period file not existed", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsQuotaUs: "10000", + }), + burst: "10000", + }, + wantErr: false, + }, + { + name: "TC-3.4: matchQuota period value invalid", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsPeriodUs: "abc", + cfsQuotaUs: "10000", + }), + burst: "10000", + }, + wantErr: false, + }, + { + name: "TC-3.5: matchQuota quota > max", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsPeriodUs: "0", + cfsQuotaUs: "10000", + }), + burst: "10000", + }, + wantErr: false, + }, + { + name: "TC-3.6: matchQuota quota < burst", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{ + cfsBurstUs: "0", + cfsPeriodUs: "10000", + cfsQuotaUs: "10000", + }).WithContainers(1), + burst: "200000000", + }, + wantErr: false, + }, + { + name: "Tc-4: burst file not existed", + args: args{ + pod: try.GenFakeGuaranteedPod(map[*cgroup.Key]string{}), + burst: "1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf := NewBurst() + if tt.args.burst != "" { + tt.args.pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.args.burst + } + if err := conf.AddFunc(tt.args.pod.PodInfo); (err != nil) != tt.wantErr { + t.Errorf("Burst.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + } + tt.args.pod.CleanPath().OrDie() + }) + } + cgroup.InitMountDir(constant.DefaultCgroupRoot) +} + +// TestOther tests other function +func TestOther(t *testing.T) { + const tcName = "TC1-test Other" + t.Run(tcName, func(t *testing.T) { + got := NewBurst() + got.SetupLog(&Log.EmptyLog{}) + assert.NoError(t, got.DeleteFunc(&typedef.PodInfo{})) + assert.Equal(t, moduleName, got.ID()) + }) +} + +// TestBurst_UpdateFunc tests UpdateFunc +func TestBurst_UpdateFunc(t *testing.T) { + type args struct { + oldPod *typedef.PodInfo + newPod *typedef.PodInfo + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "TC1-same burst", + args: args{ + oldPod: &typedef.PodInfo{ + Annotations: map[string]string{ + constant.QuotaBurstAnnotationKey: "10", + }, + }, + newPod: &typedef.PodInfo{ + Annotations: map[string]string{ + constant.QuotaBurstAnnotationKey: "10", + }, + }, + }, + }, + { + name: "TC2-different burst", + args: args{ + oldPod: &typedef.PodInfo{ + Annotations: make(map[string]string), + }, + newPod: &typedef.PodInfo{ + Annotations: map[string]string{ + constant.QuotaBurstAnnotationKey: "10", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf := NewBurst() + if err := conf.UpdateFunc(tt.args.oldPod, tt.args.newPod); (err != nil) != tt.wantErr { + t.Errorf("Burst.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestBurst_PreStart tests PreStart +func TestBurst_PreStart(t *testing.T) { + type args struct { + viewer api.Viewer + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "TC1-set pod", + args: args{ + viewer: &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + "testPod1": {}, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf := NewBurst() + if err := conf.PreStart(tt.args.viewer); (err != nil) != tt.wantErr { + t.Errorf("Burst.PreStart() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} -- Gitee From 88c4c59e38eefa32e94ed6336c5bb568f6d4698a Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 3 Mar 2023 11:02:23 +0800 Subject: [PATCH 41/73] bugfix: exit when fail to parse agent config --- pkg/config/config.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index d2e7bb1..6795a05 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -70,12 +70,16 @@ func loadConfigFile(config string) ([]byte, error) { } // parseAgentConfig parses config as AgentConfig -func (c *Config) parseAgentConfig() { +func (c *Config) parseAgentConfig() error { content, ok := c.Fields[agentKey] if !ok { - return + // not setting agent means using the default configuration file + return nil } - c.UnmarshalSubConfig(content, c.Agent) + if err := c.UnmarshalSubConfig(content, c.Agent); err != nil { + return err + } + return nil } // LoadConfig loads and parses configuration data from the file, and save it to the Config @@ -89,10 +93,12 @@ func (c *Config) LoadConfig(path string) error { } fields, err := c.ParseConfig(data) if err != nil { - return fmt.Errorf("error parsing data: %s", err) + return fmt.Errorf("error parsing config: %v", err) } c.Fields = fields - c.parseAgentConfig() + if err := c.parseAgentConfig(); err != nil { + return fmt.Errorf("error parsing agent config: %v", err) + } return nil } -- Gitee From 4681586dc0ad2a760551654cf5004ab3f927773d Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Fri, 3 Mar 2023 11:01:57 +0800 Subject: [PATCH 42/73] bugfix: limit read file function use ReadSmallFile instead of normal read to limit file size Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index d2e7bb1..1ce187d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,9 @@ package config import ( "encoding/json" "fmt" - "io/ioutil" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" ) const agentKey = "agent" @@ -62,7 +62,7 @@ func NewConfig(pType parserType) *Config { // loadConfigFile loads data from configuration file func loadConfigFile(config string) ([]byte, error) { - buffer, err := ioutil.ReadFile(config) + buffer, err := util.ReadSmallFile(config) if err != nil { return nil, err } -- Gitee From 117507e14fba48e5a963cbdcb670d5f74f0e05bc Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Thu, 2 Mar 2023 16:21:42 +0800 Subject: [PATCH 43/73] feat: add retry mechanism for add event will retry five times if failed to process add event Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/services/servicemanager.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 3e2b502..37394c9 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -273,10 +273,15 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { return } + const retryCount = 5 runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) - if err := s.AddFunc(podInfo); err != nil { - log.Errorf("service %s add func failed: %v", s.ID(), err) + for i := 0; i < retryCount; i++ { + if err := s.AddFunc(podInfo); err != nil { + log.Errorf("service %s add func failed: %v", s.ID(), err) + } else { + break + } } wg.Done() } -- Gitee From c7799e1010d9528c4c943c30ee5d172946cfbaa0 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 3 Mar 2023 11:29:16 +0800 Subject: [PATCH 44/73] bugfix: optimize log level & service log 1. The registration operation is initialized before the log configuration, so it is not recommended to use the log printing capability before the log configuration is initialized. 2. Adjust the default log level to info 3. Increase the log level of service startup to info to facilitate understanding of the currently used services. --- pkg/common/log/log.go | 2 +- pkg/services/registry.go | 3 --- pkg/services/servicemanager.go | 5 ++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index 02b12ee..1fefc3b 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -55,7 +55,7 @@ const ( var ( logDriver = stdio logFname = filepath.Join(constant.DefaultLogDir, "rubik.log") - logLevel = 0 + logLevel = logInfo logSize int64 = 1024 logFileMaxSize int64 logFileSize int64 diff --git a/pkg/services/registry.go b/pkg/services/registry.go index 8588865..28f2d99 100644 --- a/pkg/services/registry.go +++ b/pkg/services/registry.go @@ -16,8 +16,6 @@ package services import ( "sync" - - "isula.org/rubik/pkg/common/log" ) type ( @@ -41,7 +39,6 @@ func Register(name string, creator Creator) { servicesRegistry.Lock() servicesRegistry.services[name] = creator servicesRegistry.Unlock() - log.Debugf("func register (%s)", name) } // GetServiceCreator returns the service creator based on the incoming service name diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 3e2b502..ff01ed0 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -97,7 +97,6 @@ func (manager *ServiceManager) AddRunningService(name string, service interface{ addService := manager.tryAddService(name, service) addPersistentService := manager.tryAddPersistentService(name, service) if addPersistentService || addService { - log.Debugf("pre-start service %s", name) return nil } return fmt.Errorf("invalid service %s (type %T)", name, service) @@ -134,7 +133,7 @@ func (manager *ServiceManager) tryAddService(name string, service interface{}) b manager.Lock() manager.RunningServices[name] = s manager.Unlock() - log.Debugf("service %s will run", name) + log.Infof("service %s will run", name) } return ok } @@ -146,7 +145,7 @@ func (manager *ServiceManager) tryAddPersistentService(name string, service inte manager.Lock() manager.RunningPersistentServices[name] = s manager.Unlock() - log.Debugf("persistent service %s will run", name) + log.Infof("persistent service %s will run", name) } return ok } -- Gitee From b2e97505e85fd5ced0e541900da0b6a67eb64d2f Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 3 Mar 2023 15:11:54 +0800 Subject: [PATCH 45/73] bugfix: service prestart handles pods that run before rubik The current serviceManager starts before the informer, which causes the pod that runs before the rubik cannot be obtained when the service service starts, so adjust the startup sequence of the serviceManager and the informer add qos's Prestart to handle pods running before rubik Signed-off-by: vegbir --- pkg/rubik/rubik.go | 4 ++-- pkg/services/qos/qos.go | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 508f7fa..62fa40f 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -61,10 +61,10 @@ func NewAgent(cfg *config.Config) (*Agent, error) { // Run starts and runs the agent until receiving stop signal func (a *Agent) Run(ctx context.Context) error { log.Infof("agent run with config:\n%s", a.config.String()) - if err := a.startServiceHandler(ctx); err != nil { + if err := a.startInformer(ctx); err != nil { return err } - if err := a.startInformer(ctx); err != nil { + if err := a.startServiceHandler(ctx); err != nil { return err } <-ctx.Done() diff --git a/pkg/services/qos/qos.go b/pkg/services/qos/qos.go index 2868acc..4bf9d81 100644 --- a/pkg/services/qos/qos.go +++ b/pkg/services/qos/qos.go @@ -60,6 +60,16 @@ func (q *QoS) ID() string { return q.Name } +// PreStart is the pre-start action +func (q *QoS) PreStart(viewer api.Viewer) error { + for _, pod := range viewer.ListPodsWithOptions() { + if err := q.SetQoS(pod); err != nil { + q.Log.Errorf("error prestart pod %v: %v", pod.Name, err) + } + } + return nil +} + // AddFunc implement add function when pod is added in k8s func (q *QoS) AddFunc(pod *typedef.PodInfo) error { if err := q.SetQoS(pod); err != nil { -- Gitee From 90a1b5759961a3879755264305ae6d3ee8ffd6db Mon Sep 17 00:00:00 2001 From: hanchao Date: Wed, 8 Mar 2023 13:21:59 +0800 Subject: [PATCH 46/73] feature: support iocost IOCOST is supported in the rubik refactor. --- pkg/core/typedef/cgroup/common.go | 16 +- pkg/core/typedef/containerinfo.go | 2 +- pkg/core/typedef/podinfo.go | 2 +- pkg/rubik/import.go | 2 +- .../blkio.go => blkcg/blkiothrottle.go} | 28 +-- pkg/services/blkcg/iocost.go | 210 ++++++++++++++++++ pkg/services/blkcg/iocost_origin.go | 78 +++++++ pkg/services/blkcg/utils.go | 47 ++++ pkg/services/quotaturbo/data.go | 2 +- 9 files changed, 360 insertions(+), 27 deletions(-) rename pkg/services/{blkio/blkio.go => blkcg/blkiothrottle.go} (54%) create mode 100644 pkg/services/blkcg/iocost.go create mode 100644 pkg/services/blkcg/iocost_origin.go create mode 100644 pkg/services/blkcg/utils.go diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 9d0e6a9..9cc2aa6 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -25,16 +25,14 @@ import ( var rootDir = constant.DefaultCgroupRoot -func AbsoluteCgroupPath(subsys, cgroupParent, cgroupFileName string) string { - if subsys == "" { - return "" - } - return filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) +func AbsoluteCgroupPath(elem ...string) string { + elem = append([]string{rootDir}, elem...) + return filepath.Join(elem...) } // ReadCgroupFile reads data from cgroup files -func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) { - cgfile := filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) +func ReadCgroupFile(elem ...string) ([]byte, error) { + cgfile := AbsoluteCgroupPath(elem...) if !util.PathExist(cgfile) { return nil, fmt.Errorf("%v: no such file or diretory", cgfile) } @@ -42,8 +40,8 @@ func ReadCgroupFile(subsys, cgroupParent, cgroupFileName string) ([]byte, error) } // WriteCgroupFile writes data to cgroup file -func WriteCgroupFile(subsys, cgroupParent, cgroupFileName string, content string) error { - cgfile := filepath.Join(rootDir, subsys, cgroupParent, cgroupFileName) +func WriteCgroupFile(content string, elem ...string) error { + cgfile := AbsoluteCgroupPath(elem...) if !util.PathExist(cgfile) { return fmt.Errorf("%v: no such file or diretory", cgfile) } diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index a43b0eb..7ab9d3c 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -105,7 +105,7 @@ func (cont *ContainerInfo) SetCgroupAttr(key *cgroup.Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return cgroup.WriteCgroupFile(key.SubSys, cont.CgroupPath, key.FileName, value) + return cgroup.WriteCgroupFile(value, key.SubSys, cont.CgroupPath, key.FileName) } // GetCgroupAttr gets container cgroup file content diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index b622715..7f725f3 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -80,7 +80,7 @@ func (pod *PodInfo) SetCgroupAttr(key *cgroup.Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return cgroup.WriteCgroupFile(key.SubSys, pod.CgroupPath, key.FileName, value) + return cgroup.WriteCgroupFile(value, key.SubSys, pod.CgroupPath, key.FileName) } // GetCgroupAttr gets container cgroup file content diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index dbb9b19..31f4dbc 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -16,7 +16,7 @@ package rubik import ( // introduce packages to auto register service - _ "isula.org/rubik/pkg/services/blkio" + _ "isula.org/rubik/pkg/services/blkcg" _ "isula.org/rubik/pkg/services/cachelimit" _ "isula.org/rubik/pkg/services/qos" _ "isula.org/rubik/pkg/services/quotaburst" diff --git a/pkg/services/blkio/blkio.go b/pkg/services/blkcg/blkiothrottle.go similarity index 54% rename from pkg/services/blkio/blkio.go rename to pkg/services/blkcg/blkiothrottle.go index 522284a..f564c74 100644 --- a/pkg/services/blkio/blkio.go +++ b/pkg/services/blkcg/blkiothrottle.go @@ -1,4 +1,4 @@ -package blkio +package blkcg import ( "isula.org/rubik/pkg/api" @@ -12,50 +12,50 @@ type DeviceConfig struct { DeviceValue string `json:"value,omitempty"` } -type BlkioConfig struct { +type BlkioThrottleConfig struct { DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` } -type Blkio struct { +type BlkioThrottle struct { Name string `json:"-"` Log api.Logger } func init() { services.Register("blkio", func() interface{} { - return NewBlkio() + return NewBlkioThrottle() }) } -func NewBlkio() *Blkio { - return &Blkio{Name: "blkio"} +func NewBlkioThrottle() *BlkioThrottle { + return &BlkioThrottle{Name: "blkiothrottle"} } -func (b *Blkio) PreStart(viewer api.Viewer) error { - b.Log.Debugf("blkio prestart") +func (b *BlkioThrottle) PreStart(viewer api.Viewer) error { + b.Log.Debugf("blkiothrottle prestart") return nil } -func (b *Blkio) Terminate(viewer api.Viewer) error { - b.Log.Infof("blkio Terminate") +func (b *BlkioThrottle) Terminate(viewer api.Viewer) error { + b.Log.Infof("blkiothrottle Terminate") return nil } -func (b *Blkio) ID() string { +func (b *BlkioThrottle) ID() string { return b.Name } -func (b *Blkio) AddFunc(podInfo *typedef.PodInfo) error { +func (b *BlkioThrottle) AddFunc(podInfo *typedef.PodInfo) error { return nil } -func (b *Blkio) UpdateFunc(old, new *typedef.PodInfo) error { +func (b *BlkioThrottle) UpdateFunc(old, new *typedef.PodInfo) error { return nil } -func (b *Blkio) DeleteFunc(podInfo *typedef.PodInfo) error { +func (b *BlkioThrottle) DeleteFunc(podInfo *typedef.PodInfo) error { return nil } diff --git a/pkg/services/blkcg/iocost.go b/pkg/services/blkcg/iocost.go new file mode 100644 index 0000000..c392b9d --- /dev/null +++ b/pkg/services/blkcg/iocost.go @@ -0,0 +1,210 @@ +package blkcg + +import ( + "os" + "strings" + "unicode" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/services" +) + +var log api.Logger + +const ( + blkcgRootDir = "blkio" + memcgRootDir = "memory" + offlineWeight = 10 + onlineWeight = 1000 + scale = 10 +) + +// LinearParam for linear model +type LinearParam struct { + Rbps int64 `json:"rbps,omitempty"` + Rseqiops int64 `json:"rseqiops,omitempty"` + Rrandiops int64 `json:"rrandiops,omitempty"` + Wbps int64 `json:"wbps,omitempty"` + Wseqiops int64 `json:"wseqiops,omitempty"` + Wrandiops int64 `json:"wrandiops,omitempty"` +} + +// IOCostConfig define iocost for node +type IOCostConfig struct { + Dev string `json:"dev,omitempty"` + Enable bool `json:"enable,omitempty"` + Model string `json:"model,omitempty"` + Param LinearParam `json:"param,omitempty"` +} + +// NodeConfig define the config of node, include iocost +type NodeConfig struct { + NodeName string `json:"nodeName,omitempty"` + IOCostEnable bool `json:"iocostEnable,omitempty"` + IOCostConfig []IOCostConfig `json:"iocostConfig,omitempty"` +} + +// IOCost for iocost class +type IOCost struct { + name string + nodeConfig []NodeConfig +} + +var ( + nodeName string +) + +// init iocost: register service and ensure this platform support iocost. +func init() { + services.Register("iocost", func() interface{} { + return &IOCost{name: "iocost"} + }) +} + +// IOCostSupport tell if the os support iocost. +func IOCostSupport() bool { + qosFile := cgroup.AbsoluteCgroupPath(blkcgRootDir, iocostQosFile) + modelFile := cgroup.AbsoluteCgroupPath(blkcgRootDir, iocostModelFile) + return util.PathExist(qosFile) && util.PathExist(modelFile) +} + +func SetLogger(l api.Logger) { + log = l +} + +// ID for get the name of iocost +func (b *IOCost) ID() string { + return b.name +} + +func (b *IOCost) PreStart(viewer api.Viewer) error { + nodeName = os.Getenv(constant.NodeNameEnvKey) + if err := b.loadConfig(); err != nil { + return err + } + return b.dealExistedPods(viewer) +} + +func (b *IOCost) loadConfig() error { + var nodeConfig *NodeConfig + // global will set all node + for _, config := range b.nodeConfig { + if config.NodeName == nodeName { + nodeConfig = &config + break + } + if config.NodeName == "global" { + nodeConfig = &config + } + } + + // no config, return + if nodeConfig == nil { + log.Warnf("no matching node exist:%v", nodeName) + return nil + } + + // ensure that previous configuration is cleared. + if err := b.clearIOCost(); err != nil { + log.Errorf("clear iocost err:%v", err) + return err + } + + if !nodeConfig.IOCostEnable { + // clear iocost before + return nil + } + + b.configIOCost(nodeConfig.IOCostConfig) + return nil +} + +func (b *IOCost) Terminate(viewer api.Viewer) error { + if err := b.clearIOCost(); err != nil { + return err + } + return nil +} + +func (b *IOCost) dealExistedPods(viewer api.Viewer) error { + pods := viewer.ListPodsWithOptions() + for _, pod := range pods { + b.configPodIOCostWeight(pod) + } + return nil +} + +func (b *IOCost) AddFunc(podInfo *typedef.PodInfo) error { + return b.configPodIOCostWeight(podInfo) +} + +func (b *IOCost) UpdateFunc(old, new *typedef.PodInfo) error { + return b.configPodIOCostWeight(new) +} + +// deal with deleted pod. +func (b *IOCost) DeleteFunc(podInfo *typedef.PodInfo) error { + return nil +} + +func (b *IOCost) configIOCost(configs []IOCostConfig) { + for _, config := range configs { + devno, err := getBlkDeviceNo(config.Dev) + if err != nil { + log.Errorf("this device not found:%v", config.Dev) + continue + } + if config.Model == "linear" { + if err := ConfigIOCostModel(devno, config.Param); err != nil { + log.Errorf("this device not found:%v", err) + continue + } + } else { + log.Errorf("non-linear models are not supported") + continue + } + if err := ConfigIOCostQoS(devno, config.Enable); err != nil { + log.Errorf("Config iocost qos failed:%v", err) + } + } +} + +// clearIOCost used to disable all iocost +func (b *IOCost) clearIOCost() error { + qosbytes, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostQosFile) + if err != nil { + return err + } + + if len(qosbytes) == 0 { + return nil + } + + qosParams := strings.Split(string(qosbytes), "\n") + for _, qosParam := range qosParams { + words := strings.FieldsFunc(qosParam, unicode.IsSpace) + if len(words) != 0 { + if err := ConfigIOCostQoS(words[0], false); err != nil { + return err + } + } + } + return nil +} + +func (b *IOCost) configPodIOCostWeight(podInfo *typedef.PodInfo) error { + var weight uint64 = offlineWeight + if podInfo.Annotations[constant.PriorityAnnotationKey] == "true" { + weight = onlineWeight + } + for _, container := range podInfo.IDContainersMap { + if err := ConfigContainerIOCostWeight(container.CgroupPath, weight); err != nil { + return err + } + } + return nil +} diff --git a/pkg/services/blkcg/iocost_origin.go b/pkg/services/blkcg/iocost_origin.go new file mode 100644 index 0000000..cd66e11 --- /dev/null +++ b/pkg/services/blkcg/iocost_origin.go @@ -0,0 +1,78 @@ +package blkcg + +import ( + "fmt" + "strconv" + + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +const ( + // iocost model file + iocostModelFile = "blkio.cost.model" + // iocost weight file + iocostWeightFile = "blkio.cost.weight" + // iocost weight qos file + iocostQosFile = "blkio.cost.qos" + // cgroup writeback file + wbBlkioinoFile = "memory.wb_blkio_ino" +) + +// ConfigIOCostQoS for config iocost qos. +func ConfigIOCostQoS(devno string, enable bool) error { + t := 0 + if enable { + t = 1 + } + qosParam := fmt.Sprintf("%v enable=%v ctrl=user min=100.00 max=100.00", devno, t) + return cgroup.WriteCgroupFile(qosParam, blkcgRootDir, iocostQosFile) +} + +// ConfigIOCostModel for config iocost model +func ConfigIOCostModel(devno string, p interface{}) error { + var paramStr string + switch param := p.(type) { + case LinearParam: + if param.Rbps <= 0 || param.Rseqiops <= 0 || param.Rrandiops <= 0 || + param.Wbps <= 0 || param.Wseqiops <= 0 || param.Wrandiops <= 0 { + return fmt.Errorf("invalid params, linear params must be greater than 0") + } + + paramStr = fmt.Sprintf("%v rbps=%v rseqiops=%v rrandiops=%v wbps=%v wseqiops=%v wrandiops=%v", + devno, + param.Rbps, param.Rseqiops, param.Rrandiops, + param.Wbps, param.Wseqiops, param.Wrandiops, + ) + default: + return fmt.Errorf("model param is errror") + } + return cgroup.WriteCgroupFile(paramStr, blkcgRootDir, iocostModelFile) +} + +// ConfigContainerIOCostWeight for config iocost weight +// cgroup v1 iocost cannot be inherited. Therefore, only the container level can be configured. +func ConfigContainerIOCostWeight(containerRelativePath string, weight uint64) error { + if err := cgroup.WriteCgroupFile(strconv.FormatUint(weight, scale), blkcgRootDir, + containerRelativePath, iocostWeightFile); err != nil { + return err + } + if err := bindMemcgBlkcg(containerRelativePath); err != nil { + return err + } + return nil +} + +// bindMemcgBlkcg for bind memcg and blkcg +func bindMemcgBlkcg(containerRelativePath string) error { + blkcgPath := cgroup.AbsoluteCgroupPath(blkcgRootDir, containerRelativePath) + ino, err := getDirInode(blkcgPath) + if err != nil { + return err + } + + if err := cgroup.WriteCgroupFile(strconv.FormatUint(ino, scale), + memcgRootDir, containerRelativePath, wbBlkioinoFile); err != nil { + return err + } + return nil +} diff --git a/pkg/services/blkcg/utils.go b/pkg/services/blkcg/utils.go new file mode 100644 index 0000000..ac1f5a4 --- /dev/null +++ b/pkg/services/blkcg/utils.go @@ -0,0 +1,47 @@ +package blkcg + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + + "github.com/pkg/errors" +) + +const ( + devNoMax = 256 +) + +func getBlkDeviceNo(devName string) (string, error) { + devPath := filepath.Join("/dev", devName) + fi, err := os.Stat(devPath) + if err != nil { + return "", fmt.Errorf("stat %s failed with error: %v", devName, err) + } + + if fi.Mode()&os.ModeDevice == 0 { + return "", fmt.Errorf("%s is not a device", devName) + } + + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.Errorf("failed to get Sys(), %v has type %v", devName, st) + } + + devno := st.Rdev + major, minor := devno/devNoMax, devno%devNoMax + return fmt.Sprintf("%v:%v", major, minor), nil +} + +func getDirInode(file string) (uint64, error) { + fi, err := os.Stat(file) + if err != nil { + return 0, err + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, errors.Errorf("failed to get Sys(), %v has type %v", file, st) + } + return st.Ino, nil +} diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go index d4589f8..796aab7 100644 --- a/pkg/services/quotaturbo/data.go +++ b/pkg/services/quotaturbo/data.go @@ -98,7 +98,7 @@ func (d *NodeData) removeContainer(id string) error { return nil } - if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath, "")) { + if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath)) { return safeDel(id) } // cq.Period ranges from 1000(us) to 1000000(us) and does not overflow. -- Gitee From 33a041adb012c70bdbbf643b5e6cd1ba5b4edd34 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 9 Mar 2023 20:43:50 +0800 Subject: [PATCH 47/73] optimize: Introduced hashicorp/errwrap package for error handling, increased hierarchy for handling cgroup objects --- go.mod | 1 + go.sum | 10 + pkg/common/util/conversion.go | 2 +- pkg/core/typedef/cgroup/common.go | 68 +++- vendor/github.com/hashicorp/errwrap/LICENSE | 354 ++++++++++++++++++ vendor/github.com/hashicorp/errwrap/README.md | 89 +++++ .../github.com/hashicorp/errwrap/errwrap.go | 169 +++++++++ vendor/github.com/hashicorp/errwrap/go.mod | 1 + .../hashicorp/go-multierror/.travis.yml | 12 + .../hashicorp/go-multierror/LICENSE | 353 +++++++++++++++++ .../hashicorp/go-multierror/Makefile | 31 ++ .../hashicorp/go-multierror/README.md | 97 +++++ .../hashicorp/go-multierror/append.go | 41 ++ .../hashicorp/go-multierror/flatten.go | 26 ++ .../hashicorp/go-multierror/format.go | 27 ++ .../github.com/hashicorp/go-multierror/go.mod | 3 + .../github.com/hashicorp/go-multierror/go.sum | 4 + .../hashicorp/go-multierror/multierror.go | 51 +++ .../hashicorp/go-multierror/prefix.go | 37 ++ .../hashicorp/go-multierror/sort.go | 16 + vendor/modules.txt | 5 + 21 files changed, 1387 insertions(+), 10 deletions(-) create mode 100644 vendor/github.com/hashicorp/errwrap/LICENSE create mode 100644 vendor/github.com/hashicorp/errwrap/README.md create mode 100644 vendor/github.com/hashicorp/errwrap/errwrap.go create mode 100644 vendor/github.com/hashicorp/errwrap/go.mod create mode 100644 vendor/github.com/hashicorp/go-multierror/.travis.yml create mode 100644 vendor/github.com/hashicorp/go-multierror/LICENSE create mode 100644 vendor/github.com/hashicorp/go-multierror/Makefile create mode 100644 vendor/github.com/hashicorp/go-multierror/README.md create mode 100644 vendor/github.com/hashicorp/go-multierror/append.go create mode 100644 vendor/github.com/hashicorp/go-multierror/flatten.go create mode 100644 vendor/github.com/hashicorp/go-multierror/format.go create mode 100644 vendor/github.com/hashicorp/go-multierror/go.mod create mode 100644 vendor/github.com/hashicorp/go-multierror/go.sum create mode 100644 vendor/github.com/hashicorp/go-multierror/multierror.go create mode 100644 vendor/github.com/hashicorp/go-multierror/prefix.go create mode 100644 vendor/github.com/hashicorp/go-multierror/sort.go diff --git a/go.mod b/go.mod index d14af5e..6297396 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/cyphar/filepath-securejoin v0.0.0-00010101000000-000000000000 github.com/google/uuid v1.1.2 + github.com/hashicorp/go-multierror v1.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.6.1 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd diff --git a/go.sum b/go.sum index 6e9ecd9..770db0b 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -124,6 +125,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -138,9 +143,11 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -167,6 +174,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -322,6 +330,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -377,6 +386,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go index b686872..4cbc496 100644 --- a/pkg/common/util/conversion.go +++ b/pkg/common/util/conversion.go @@ -53,7 +53,7 @@ func ParseInt64Map(data string) (map[string]int64, error) { arr := strings.Fields(scanner.Text()) const defaultLength = 2 if len(arr) != defaultLength { - return nil, fmt.Errorf(" fail to parse a single line into two strings") + return nil, fmt.Errorf("fail to parse a single line into two strings %v", arr) } value, err := ParseInt64(arr[1]) if err != nil { diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 9cc2aa6..b750615 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -18,6 +18,7 @@ import ( "fmt" "path/filepath" "strconv" + "strings" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/util" @@ -32,20 +33,26 @@ func AbsoluteCgroupPath(elem ...string) string { // ReadCgroupFile reads data from cgroup files func ReadCgroupFile(elem ...string) ([]byte, error) { - cgfile := AbsoluteCgroupPath(elem...) - if !util.PathExist(cgfile) { - return nil, fmt.Errorf("%v: no such file or diretory", cgfile) - } - return util.ReadFile(cgfile) + return readCgroupFile(filepath.Join(AbsoluteCgroupPath(elem...))) } // WriteCgroupFile writes data to cgroup file func WriteCgroupFile(content string, elem ...string) error { - cgfile := AbsoluteCgroupPath(elem...) - if !util.PathExist(cgfile) { - return fmt.Errorf("%v: no such file or diretory", cgfile) + return writeCgroupFile(AbsoluteCgroupPath(elem...), content) +} + +func readCgroupFile(cgPath string) ([]byte, error) { + if !util.PathExist(cgPath) { + return nil, fmt.Errorf("%v: no such file or diretory", cgPath) + } + return util.ReadFile(cgPath) +} + +func writeCgroupFile(cgPath, content string) error { + if !util.PathExist(cgPath) { + return fmt.Errorf("%v: no such file or diretory", cgPath) } - return util.WriteFile(cgfile, content) + return util.WriteFile(cgPath, content) } // InitMountDir sets the mount directory of the cgroup file system @@ -137,3 +144,46 @@ func (attr *Attr) CPUStat() (*CPUStat, error) { } return NewCPUStat(attr.Value) } + +type Hierarchy struct { + MountPoint string + Path string +} + +func NewHierarchy(mountPoint, path string) *Hierarchy { + return &Hierarchy{ + MountPoint: mountPoint, + Path: path, + } +} + +// SetCgroupAttr sets value to the cgroup file +func (h *Hierarchy) SetCgroupAttr(key *Key, value string) error { + if err := validateCgroupKey(key); err != nil { + return err + } + return writeCgroupFile(filepath.Join(h.MountPoint, key.SubSys, h.Path, key.FileName), value) +} + +// GetCgroupAttr gets cgroup file content +func (h *Hierarchy) GetCgroupAttr(key *Key) *Attr { + if err := validateCgroupKey(key); err != nil { + return &Attr{Err: err} + } + data, err := readCgroupFile(filepath.Join(h.MountPoint, key.SubSys, h.Path, key.FileName)) + if err != nil { + return &Attr{Err: err} + } + return &Attr{Value: strings.TrimSpace(string(data)), Err: nil} +} + +// validateCgroupKey is used to verify the validity of the cgroup key +func validateCgroupKey(key *Key) error { + if key == nil { + return fmt.Errorf("key cannot be empty") + } + if len(key.SubSys) == 0 || len(key.FileName) == 0 { + return fmt.Errorf("invalid key") + } + return nil +} diff --git a/vendor/github.com/hashicorp/errwrap/LICENSE b/vendor/github.com/hashicorp/errwrap/LICENSE new file mode 100644 index 0000000..c33dcc7 --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/LICENSE @@ -0,0 +1,354 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/hashicorp/errwrap/README.md b/vendor/github.com/hashicorp/errwrap/README.md new file mode 100644 index 0000000..444df08 --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/README.md @@ -0,0 +1,89 @@ +# errwrap + +`errwrap` is a package for Go that formalizes the pattern of wrapping errors +and checking if an error contains another error. + +There is a common pattern in Go of taking a returned `error` value and +then wrapping it (such as with `fmt.Errorf`) before returning it. The problem +with this pattern is that you completely lose the original `error` structure. + +Arguably the _correct_ approach is that you should make a custom structure +implementing the `error` interface, and have the original error as a field +on that structure, such [as this example](http://golang.org/pkg/os/#PathError). +This is a good approach, but you have to know the entire chain of possible +rewrapping that happens, when you might just care about one. + +`errwrap` formalizes this pattern (it doesn't matter what approach you use +above) by giving a single interface for wrapping errors, checking if a specific +error is wrapped, and extracting that error. + +## Installation and Docs + +Install using `go get github.com/hashicorp/errwrap`. + +Full documentation is available at +http://godoc.org/github.com/hashicorp/errwrap + +## Usage + +#### Basic Usage + +Below is a very basic example of its usage: + +```go +// A function that always returns an error, but wraps it, like a real +// function might. +func tryOpen() error { + _, err := os.Open("/i/dont/exist") + if err != nil { + return errwrap.Wrapf("Doesn't exist: {{err}}", err) + } + + return nil +} + +func main() { + err := tryOpen() + + // We can use the Contains helpers to check if an error contains + // another error. It is safe to do this with a nil error, or with + // an error that doesn't even use the errwrap package. + if errwrap.Contains(err, "does not exist") { + // Do something + } + if errwrap.ContainsType(err, new(os.PathError)) { + // Do something + } + + // Or we can use the associated `Get` functions to just extract + // a specific error. This would return nil if that specific error doesn't + // exist. + perr := errwrap.GetType(err, new(os.PathError)) +} +``` + +#### Custom Types + +If you're already making custom types that properly wrap errors, then +you can get all the functionality of `errwraps.Contains` and such by +implementing the `Wrapper` interface with just one function. Example: + +```go +type AppError { + Code ErrorCode + Err error +} + +func (e *AppError) WrappedErrors() []error { + return []error{e.Err} +} +``` + +Now this works: + +```go +err := &AppError{Err: fmt.Errorf("an error")} +if errwrap.ContainsType(err, fmt.Errorf("")) { + // This will work! +} +``` diff --git a/vendor/github.com/hashicorp/errwrap/errwrap.go b/vendor/github.com/hashicorp/errwrap/errwrap.go new file mode 100644 index 0000000..a733bef --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/errwrap.go @@ -0,0 +1,169 @@ +// Package errwrap implements methods to formalize error wrapping in Go. +// +// All of the top-level functions that take an `error` are built to be able +// to take any error, not just wrapped errors. This allows you to use errwrap +// without having to type-check and type-cast everywhere. +package errwrap + +import ( + "errors" + "reflect" + "strings" +) + +// WalkFunc is the callback called for Walk. +type WalkFunc func(error) + +// Wrapper is an interface that can be implemented by custom types to +// have all the Contains, Get, etc. functions in errwrap work. +// +// When Walk reaches a Wrapper, it will call the callback for every +// wrapped error in addition to the wrapper itself. Since all the top-level +// functions in errwrap use Walk, this means that all those functions work +// with your custom type. +type Wrapper interface { + WrappedErrors() []error +} + +// Wrap defines that outer wraps inner, returning an error type that +// can be cleanly used with the other methods in this package, such as +// Contains, GetAll, etc. +// +// This function won't modify the error message at all (the outer message +// will be used). +func Wrap(outer, inner error) error { + return &wrappedError{ + Outer: outer, + Inner: inner, + } +} + +// Wrapf wraps an error with a formatting message. This is similar to using +// `fmt.Errorf` to wrap an error. If you're using `fmt.Errorf` to wrap +// errors, you should replace it with this. +// +// format is the format of the error message. The string '{{err}}' will +// be replaced with the original error message. +func Wrapf(format string, err error) error { + outerMsg := "" + if err != nil { + outerMsg = err.Error() + } + + outer := errors.New(strings.Replace( + format, "{{err}}", outerMsg, -1)) + + return Wrap(outer, err) +} + +// Contains checks if the given error contains an error with the +// message msg. If err is not a wrapped error, this will always return +// false unless the error itself happens to match this msg. +func Contains(err error, msg string) bool { + return len(GetAll(err, msg)) > 0 +} + +// ContainsType checks if the given error contains an error with +// the same concrete type as v. If err is not a wrapped error, this will +// check the err itself. +func ContainsType(err error, v interface{}) bool { + return len(GetAllType(err, v)) > 0 +} + +// Get is the same as GetAll but returns the deepest matching error. +func Get(err error, msg string) error { + es := GetAll(err, msg) + if len(es) > 0 { + return es[len(es)-1] + } + + return nil +} + +// GetType is the same as GetAllType but returns the deepest matching error. +func GetType(err error, v interface{}) error { + es := GetAllType(err, v) + if len(es) > 0 { + return es[len(es)-1] + } + + return nil +} + +// GetAll gets all the errors that might be wrapped in err with the +// given message. The order of the errors is such that the outermost +// matching error (the most recent wrap) is index zero, and so on. +func GetAll(err error, msg string) []error { + var result []error + + Walk(err, func(err error) { + if err.Error() == msg { + result = append(result, err) + } + }) + + return result +} + +// GetAllType gets all the errors that are the same type as v. +// +// The order of the return value is the same as described in GetAll. +func GetAllType(err error, v interface{}) []error { + var result []error + + var search string + if v != nil { + search = reflect.TypeOf(v).String() + } + Walk(err, func(err error) { + var needle string + if err != nil { + needle = reflect.TypeOf(err).String() + } + + if needle == search { + result = append(result, err) + } + }) + + return result +} + +// Walk walks all the wrapped errors in err and calls the callback. If +// err isn't a wrapped error, this will be called once for err. If err +// is a wrapped error, the callback will be called for both the wrapper +// that implements error as well as the wrapped error itself. +func Walk(err error, cb WalkFunc) { + if err == nil { + return + } + + switch e := err.(type) { + case *wrappedError: + cb(e.Outer) + Walk(e.Inner, cb) + case Wrapper: + cb(err) + + for _, err := range e.WrappedErrors() { + Walk(err, cb) + } + default: + cb(err) + } +} + +// wrappedError is an implementation of error that has both the +// outer and inner errors. +type wrappedError struct { + Outer error + Inner error +} + +func (w *wrappedError) Error() string { + return w.Outer.Error() +} + +func (w *wrappedError) WrappedErrors() []error { + return []error{w.Outer, w.Inner} +} diff --git a/vendor/github.com/hashicorp/errwrap/go.mod b/vendor/github.com/hashicorp/errwrap/go.mod new file mode 100644 index 0000000..c9b8402 --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/go.mod @@ -0,0 +1 @@ +module github.com/hashicorp/errwrap diff --git a/vendor/github.com/hashicorp/go-multierror/.travis.yml b/vendor/github.com/hashicorp/go-multierror/.travis.yml new file mode 100644 index 0000000..304a835 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/.travis.yml @@ -0,0 +1,12 @@ +sudo: false + +language: go + +go: + - 1.x + +branches: + only: + - master + +script: make test testrace diff --git a/vendor/github.com/hashicorp/go-multierror/LICENSE b/vendor/github.com/hashicorp/go-multierror/LICENSE new file mode 100644 index 0000000..82b4de9 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/LICENSE @@ -0,0 +1,353 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/hashicorp/go-multierror/Makefile b/vendor/github.com/hashicorp/go-multierror/Makefile new file mode 100644 index 0000000..b97cd6e --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/Makefile @@ -0,0 +1,31 @@ +TEST?=./... + +default: test + +# test runs the test suite and vets the code. +test: generate + @echo "==> Running tests..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS} + +# testrace runs the race checker +testrace: generate + @echo "==> Running tests (race)..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -race ${TESTARGS} + +# updatedeps installs all the dependencies needed to run and build. +updatedeps: + @sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'" + +# generate runs `go generate` to build the dynamically generated source files. +generate: + @echo "==> Generating..." + @find . -type f -name '.DS_Store' -delete + @go list ./... \ + | grep -v "/vendor/" \ + | xargs -n1 go generate + +.PHONY: default test testrace updatedeps generate diff --git a/vendor/github.com/hashicorp/go-multierror/README.md b/vendor/github.com/hashicorp/go-multierror/README.md new file mode 100644 index 0000000..ead5830 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/README.md @@ -0,0 +1,97 @@ +# go-multierror + +[![Build Status](http://img.shields.io/travis/hashicorp/go-multierror.svg?style=flat-square)][travis] +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[travis]: https://travis-ci.org/hashicorp/go-multierror +[godocs]: https://godoc.org/github.com/hashicorp/go-multierror + +`go-multierror` is a package for Go that provides a mechanism for +representing a list of `error` values as a single `error`. + +This allows a function in Go to return an `error` that might actually +be a list of errors. If the caller knows this, they can unwrap the +list and access the errors. If the caller doesn't know, the error +formats to a nice human-readable format. + +`go-multierror` implements the +[errwrap](https://github.com/hashicorp/errwrap) interface so that it can +be used with that library, as well. + +## Installation and Docs + +Install using `go get github.com/hashicorp/go-multierror`. + +Full documentation is available at +http://godoc.org/github.com/hashicorp/go-multierror + +## Usage + +go-multierror is easy to use and purposely built to be unobtrusive in +existing Go applications/libraries that may not be aware of it. + +**Building a list of errors** + +The `Append` function is used to create a list of errors. This function +behaves a lot like the Go built-in `append` function: it doesn't matter +if the first argument is nil, a `multierror.Error`, or any other `error`, +the function behaves as you would expect. + +```go +var result error + +if err := step1(); err != nil { + result = multierror.Append(result, err) +} +if err := step2(); err != nil { + result = multierror.Append(result, err) +} + +return result +``` + +**Customizing the formatting of the errors** + +By specifying a custom `ErrorFormat`, you can customize the format +of the `Error() string` function: + +```go +var result *multierror.Error + +// ... accumulate errors here, maybe using Append + +if result != nil { + result.ErrorFormat = func([]error) string { + return "errors!" + } +} +``` + +**Accessing the list of errors** + +`multierror.Error` implements `error` so if the caller doesn't know about +multierror, it will work just fine. But if you're aware a multierror might +be returned, you can use type switches to access the list of errors: + +```go +if err := something(); err != nil { + if merr, ok := err.(*multierror.Error); ok { + // Use merr.Errors + } +} +``` + +**Returning a multierror only if there are errors** + +If you build a `multierror.Error`, you can use the `ErrorOrNil` function +to return an `error` implementation only if there are errors to return: + +```go +var result *multierror.Error + +// ... accumulate errors here + +// Return the `error` only if errors were added to the multierror, otherwise +// return nil since there are no errors. +return result.ErrorOrNil() +``` diff --git a/vendor/github.com/hashicorp/go-multierror/append.go b/vendor/github.com/hashicorp/go-multierror/append.go new file mode 100644 index 0000000..775b6e7 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/append.go @@ -0,0 +1,41 @@ +package multierror + +// Append is a helper function that will append more errors +// onto an Error in order to create a larger multi-error. +// +// If err is not a multierror.Error, then it will be turned into +// one. If any of the errs are multierr.Error, they will be flattened +// one level into err. +func Append(err error, errs ...error) *Error { + switch err := err.(type) { + case *Error: + // Typed nils can reach here, so initialize if we are nil + if err == nil { + err = new(Error) + } + + // Go through each error and flatten + for _, e := range errs { + switch e := e.(type) { + case *Error: + if e != nil { + err.Errors = append(err.Errors, e.Errors...) + } + default: + if e != nil { + err.Errors = append(err.Errors, e) + } + } + } + + return err + default: + newErrs := make([]error, 0, len(errs)+1) + if err != nil { + newErrs = append(newErrs, err) + } + newErrs = append(newErrs, errs...) + + return Append(&Error{}, newErrs...) + } +} diff --git a/vendor/github.com/hashicorp/go-multierror/flatten.go b/vendor/github.com/hashicorp/go-multierror/flatten.go new file mode 100644 index 0000000..aab8e9a --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/flatten.go @@ -0,0 +1,26 @@ +package multierror + +// Flatten flattens the given error, merging any *Errors together into +// a single *Error. +func Flatten(err error) error { + // If it isn't an *Error, just return the error as-is + if _, ok := err.(*Error); !ok { + return err + } + + // Otherwise, make the result and flatten away! + flatErr := new(Error) + flatten(err, flatErr) + return flatErr +} + +func flatten(err error, flatErr *Error) { + switch err := err.(type) { + case *Error: + for _, e := range err.Errors { + flatten(e, flatErr) + } + default: + flatErr.Errors = append(flatErr.Errors, err) + } +} diff --git a/vendor/github.com/hashicorp/go-multierror/format.go b/vendor/github.com/hashicorp/go-multierror/format.go new file mode 100644 index 0000000..47f13c4 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/format.go @@ -0,0 +1,27 @@ +package multierror + +import ( + "fmt" + "strings" +) + +// ErrorFormatFunc is a function callback that is called by Error to +// turn the list of errors into a string. +type ErrorFormatFunc func([]error) string + +// ListFormatFunc is a basic formatter that outputs the number of errors +// that occurred along with a bullet point list of the errors. +func ListFormatFunc(es []error) string { + if len(es) == 1 { + return fmt.Sprintf("1 error occurred:\n\t* %s\n\n", es[0]) + } + + points := make([]string, len(es)) + for i, err := range es { + points[i] = fmt.Sprintf("* %s", err) + } + + return fmt.Sprintf( + "%d errors occurred:\n\t%s\n\n", + len(es), strings.Join(points, "\n\t")) +} diff --git a/vendor/github.com/hashicorp/go-multierror/go.mod b/vendor/github.com/hashicorp/go-multierror/go.mod new file mode 100644 index 0000000..2534331 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/go.mod @@ -0,0 +1,3 @@ +module github.com/hashicorp/go-multierror + +require github.com/hashicorp/errwrap v1.0.0 diff --git a/vendor/github.com/hashicorp/go-multierror/go.sum b/vendor/github.com/hashicorp/go-multierror/go.sum new file mode 100644 index 0000000..85b1f8f --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/go.sum @@ -0,0 +1,4 @@ +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/vendor/github.com/hashicorp/go-multierror/multierror.go b/vendor/github.com/hashicorp/go-multierror/multierror.go new file mode 100644 index 0000000..89b1422 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/multierror.go @@ -0,0 +1,51 @@ +package multierror + +import ( + "fmt" +) + +// Error is an error type to track multiple errors. This is used to +// accumulate errors in cases and return them as a single "error". +type Error struct { + Errors []error + ErrorFormat ErrorFormatFunc +} + +func (e *Error) Error() string { + fn := e.ErrorFormat + if fn == nil { + fn = ListFormatFunc + } + + return fn(e.Errors) +} + +// ErrorOrNil returns an error interface if this Error represents +// a list of errors, or returns nil if the list of errors is empty. This +// function is useful at the end of accumulation to make sure that the value +// returned represents the existence of errors. +func (e *Error) ErrorOrNil() error { + if e == nil { + return nil + } + if len(e.Errors) == 0 { + return nil + } + + return e +} + +func (e *Error) GoString() string { + return fmt.Sprintf("*%#v", *e) +} + +// WrappedErrors returns the list of errors that this Error is wrapping. +// It is an implementation of the errwrap.Wrapper interface so that +// multierror.Error can be used with that library. +// +// This method is not safe to be called concurrently and is no different +// than accessing the Errors field directly. It is implemented only to +// satisfy the errwrap.Wrapper interface. +func (e *Error) WrappedErrors() []error { + return e.Errors +} diff --git a/vendor/github.com/hashicorp/go-multierror/prefix.go b/vendor/github.com/hashicorp/go-multierror/prefix.go new file mode 100644 index 0000000..5c477ab --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/prefix.go @@ -0,0 +1,37 @@ +package multierror + +import ( + "fmt" + + "github.com/hashicorp/errwrap" +) + +// Prefix is a helper function that will prefix some text +// to the given error. If the error is a multierror.Error, then +// it will be prefixed to each wrapped error. +// +// This is useful to use when appending multiple multierrors +// together in order to give better scoping. +func Prefix(err error, prefix string) error { + if err == nil { + return nil + } + + format := fmt.Sprintf("%s {{err}}", prefix) + switch err := err.(type) { + case *Error: + // Typed nils can reach here, so initialize if we are nil + if err == nil { + err = new(Error) + } + + // Wrap each of the errors + for i, e := range err.Errors { + err.Errors[i] = errwrap.Wrapf(format, e) + } + + return err + default: + return errwrap.Wrapf(format, err) + } +} diff --git a/vendor/github.com/hashicorp/go-multierror/sort.go b/vendor/github.com/hashicorp/go-multierror/sort.go new file mode 100644 index 0000000..fecb14e --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/sort.go @@ -0,0 +1,16 @@ +package multierror + +// Len implements sort.Interface function for length +func (err Error) Len() int { + return len(err.Errors) +} + +// Swap implements sort.Interface function for swapping elements +func (err Error) Swap(i, j int) { + err.Errors[i], err.Errors[j] = err.Errors[j], err.Errors[i] +} + +// Less implements sort.Interface function for determining order +func (err Error) Less(i, j int) bool { + return err.Errors[i].Error() < err.Errors[j].Error() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2f895a3..209b861 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -32,6 +32,11 @@ github.com/google/uuid github.com/googleapis/gnostic/compiler github.com/googleapis/gnostic/extensions github.com/googleapis/gnostic/openapiv2 +# github.com/hashicorp/errwrap v1.0.0 +github.com/hashicorp/errwrap +# github.com/hashicorp/go-multierror v1.0.0 +## explicit +github.com/hashicorp/go-multierror # github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru -- Gitee From 846b8f4574c98db197e7b1b8036416dfb4ab93db Mon Sep 17 00:00:00 2001 From: hanchao Date: Fri, 10 Mar 2023 11:51:48 +0800 Subject: [PATCH 48/73] refactor: service module remove self logger service module use global logger and remove self logger --- pkg/services/blkcg/blkiothrottle.go | 6 ++--- pkg/services/blkcg/iocost.go | 7 +----- pkg/services/blkcg/utils.go | 6 ++--- pkg/services/cachelimit/cachelimit.go | 10 --------- pkg/services/cachelimit/cachelimit_init.go | 1 + pkg/services/cachelimit/dynamic.go | 1 + pkg/services/cachelimit/sync.go | 1 + pkg/services/qos/qos.go | 8 +++---- pkg/services/qos/qos_test.go | 10 --------- pkg/services/quotaburst/quotaburst.go | 10 +-------- pkg/services/quotaburst/quotaburst_test.go | 2 -- pkg/services/quotaturbo/cpuquota.go | 1 + pkg/services/quotaturbo/data.go | 1 + pkg/services/quotaturbo/driverevent.go | 1 + pkg/services/quotaturbo/quotaturbo.go | 9 +------- pkg/services/quotaturbo/quotaturbo_test.go | 2 -- pkg/services/servicemanager.go | 26 ---------------------- 17 files changed, 18 insertions(+), 84 deletions(-) diff --git a/pkg/services/blkcg/blkiothrottle.go b/pkg/services/blkcg/blkiothrottle.go index f564c74..4c23cf2 100644 --- a/pkg/services/blkcg/blkiothrottle.go +++ b/pkg/services/blkcg/blkiothrottle.go @@ -2,6 +2,7 @@ package blkcg import ( "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" ) @@ -21,7 +22,6 @@ type BlkioThrottleConfig struct { type BlkioThrottle struct { Name string `json:"-"` - Log api.Logger } func init() { @@ -35,12 +35,12 @@ func NewBlkioThrottle() *BlkioThrottle { } func (b *BlkioThrottle) PreStart(viewer api.Viewer) error { - b.Log.Debugf("blkiothrottle prestart") + log.Infof("blkiothrottle prestart") return nil } func (b *BlkioThrottle) Terminate(viewer api.Viewer) error { - b.Log.Infof("blkiothrottle Terminate") + log.Infof("blkiothrottle Terminate") return nil } diff --git a/pkg/services/blkcg/iocost.go b/pkg/services/blkcg/iocost.go index c392b9d..e45ab7a 100644 --- a/pkg/services/blkcg/iocost.go +++ b/pkg/services/blkcg/iocost.go @@ -7,14 +7,13 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/services" ) -var log api.Logger - const ( blkcgRootDir = "blkio" memcgRootDir = "memory" @@ -72,10 +71,6 @@ func IOCostSupport() bool { return util.PathExist(qosFile) && util.PathExist(modelFile) } -func SetLogger(l api.Logger) { - log = l -} - // ID for get the name of iocost func (b *IOCost) ID() string { return b.name diff --git a/pkg/services/blkcg/utils.go b/pkg/services/blkcg/utils.go index ac1f5a4..9086c19 100644 --- a/pkg/services/blkcg/utils.go +++ b/pkg/services/blkcg/utils.go @@ -5,8 +5,6 @@ import ( "os" "path/filepath" "syscall" - - "github.com/pkg/errors" ) const ( @@ -26,7 +24,7 @@ func getBlkDeviceNo(devName string) (string, error) { st, ok := fi.Sys().(*syscall.Stat_t) if !ok { - return "", errors.Errorf("failed to get Sys(), %v has type %v", devName, st) + return "", fmt.Errorf("failed to get Sys(), %v has type %v", devName, st) } devno := st.Rdev @@ -41,7 +39,7 @@ func getDirInode(file string) (uint64, error) { } st, ok := fi.Sys().(*syscall.Stat_t) if !ok { - return 0, errors.Errorf("failed to get Sys(), %v has type %v", file, st) + return 0, fmt.Errorf("failed to get Sys(), %v has type %v", file, st) } return st.Ino, nil } diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/cachelimit/cachelimit.go index a66c6e9..afe68a0 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/cachelimit/cachelimit.go @@ -22,7 +22,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "isula.org/rubik/pkg/api" - Log "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/services" ) @@ -113,21 +112,12 @@ type Attr struct { MinMiss int } -// log is global logger for cache limit service -var log api.Logger - func init() { - log = &Log.EmptyLog{} services.Register(moduleName, func() interface{} { return NewCacheLimit() }) } -// SetupLog initializes the log interface for the module -func (c *CacheLimit) SetupLog(logger api.Logger) { - log = logger -} - // NewCacheLimit return cache limit instance with default settings func NewCacheLimit() *CacheLimit { return &CacheLimit{ diff --git a/pkg/services/cachelimit/cachelimit_init.go b/pkg/services/cachelimit/cachelimit_init.go index eb6d3bb..ea766f2 100644 --- a/pkg/services/cachelimit/cachelimit_init.go +++ b/pkg/services/cachelimit/cachelimit_init.go @@ -22,6 +22,7 @@ import ( "strings" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/perf" "isula.org/rubik/pkg/common/util" ) diff --git a/pkg/services/cachelimit/dynamic.go b/pkg/services/cachelimit/dynamic.go index f5839da..6a3b771 100644 --- a/pkg/services/cachelimit/dynamic.go +++ b/pkg/services/cachelimit/dynamic.go @@ -19,6 +19,7 @@ import ( "time" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/perf" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" diff --git a/pkg/services/cachelimit/sync.go b/pkg/services/cachelimit/sync.go index 1ab5536..18f8679 100644 --- a/pkg/services/cachelimit/sync.go +++ b/pkg/services/cachelimit/sync.go @@ -20,6 +20,7 @@ import ( "strings" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" diff --git a/pkg/services/qos/qos.go b/pkg/services/qos/qos.go index 4bf9d81..23b0a3b 100644 --- a/pkg/services/qos/qos.go +++ b/pkg/services/qos/qos.go @@ -20,6 +20,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/services" @@ -33,7 +34,6 @@ var supportCgroupTypes = map[string]*cgroup.Key{ // QoS define service which related to qos level setting type QoS struct { Name string `json:"-"` - Log api.Logger Config } @@ -64,7 +64,7 @@ func (q *QoS) ID() string { func (q *QoS) PreStart(viewer api.Viewer) error { for _, pod := range viewer.ListPodsWithOptions() { if err := q.SetQoS(pod); err != nil { - q.Log.Errorf("error prestart pod %v: %v", pod.Name, err) + log.Errorf("error prestart pod %v: %v", pod.Name, err) } } return nil @@ -128,7 +128,7 @@ func (q *QoS) SetQoS(pod *typedef.PodInfo) error { } qosLevel := getQoSLevel(pod) if qosLevel == constant.Online { - q.Log.Debugf("pod %s already online", pod.Name) + log.Debugf("pod %s already online", pod.Name) return nil } @@ -142,7 +142,7 @@ func (q *QoS) SetQoS(pod *typedef.PodInfo) error { } } } - q.Log.Debugf("set pod %s(%s) qos level %d ok", pod.Name, pod.UID, qosLevel) + log.Debugf("set pod %s(%s) qos level %d ok", pod.Name, pod.UID, qosLevel) return nil } diff --git a/pkg/services/qos/qos_test.go b/pkg/services/qos/qos_test.go index 77899a1..f4dd395 100644 --- a/pkg/services/qos/qos_test.go +++ b/pkg/services/qos/qos_test.go @@ -15,12 +15,9 @@ package qos import ( - "context" "testing" - "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/test/try" ) @@ -31,7 +28,6 @@ func init() { type fields struct { Name string - Log api.Logger Config Config } type args struct { @@ -50,7 +46,6 @@ type test struct { var getCommonField = func(subSys []string) fields { return fields{ Name: "qos", - Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), Config: Config{SubSys: subSys}, } } @@ -112,7 +107,6 @@ func TestQoS_AddFunc(t *testing.T) { t.Run(tt.name, func(t *testing.T) { q := &QoS{ Name: tt.fields.Name, - Log: tt.fields.Log, Config: tt.fields.Config, } if tt.preHook != nil { @@ -171,7 +165,6 @@ func TestQoS_UpdateFunc(t *testing.T) { t.Run(tt.name, func(t *testing.T) { q := &QoS{ Name: tt.fields.Name, - Log: tt.fields.Log, Config: tt.fields.Config, } if tt.preHook != nil { @@ -192,7 +185,6 @@ func TestQoS_Validate(t *testing.T) { name: "TC1-normal config", fields: fields{ Name: "qos", - Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), Config: Config{SubSys: []string{"cpu", "memory"}}, }, }, @@ -200,7 +192,6 @@ func TestQoS_Validate(t *testing.T) { name: "TC2-abnormal config", fields: fields{ Name: "undefine", - Log: log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), "qos")), Config: Config{SubSys: []string{"undefine"}}, }, wantErr: true, @@ -215,7 +206,6 @@ func TestQoS_Validate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { q := &QoS{ Name: tt.fields.Name, - Log: tt.fields.Log, Config: tt.fields.Config, } if err := q.Validate(); (err != nil) != tt.wantErr { diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index 1d9258c..db6bbb7 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -24,7 +24,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" @@ -35,10 +35,7 @@ const ( moduleName = "quotaburst" ) -var log api.Logger - func init() { - log = &Log.EmptyLog{} services.Register(moduleName, func() interface{} { return NewBurst() }) @@ -56,11 +53,6 @@ func NewBurst() *Burst { } } -// SetupLog initializes the log interface for the module -func (b *Burst) SetupLog(logger api.Logger) { - log = logger -} - // ID returns the module name func (b *Burst) ID() string { return moduleName diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go index d311c33..6a2442e 100644 --- a/pkg/services/quotaburst/quotaburst_test.go +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -21,7 +21,6 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - Log "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/podmanager" @@ -170,7 +169,6 @@ func TestOther(t *testing.T) { const tcName = "TC1-test Other" t.Run(tcName, func(t *testing.T) { got := NewBurst() - got.SetupLog(&Log.EmptyLog{}) assert.NoError(t, got.DeleteFunc(&typedef.PodInfo{})) assert.Equal(t, moduleName, got.ID()) }) diff --git a/pkg/services/quotaturbo/cpuquota.go b/pkg/services/quotaturbo/cpuquota.go index dcb079f..86c837a 100644 --- a/pkg/services/quotaturbo/cpuquota.go +++ b/pkg/services/quotaturbo/cpuquota.go @@ -19,6 +19,7 @@ import ( "path" "time" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go index 796aab7..c777490 100644 --- a/pkg/services/quotaturbo/data.go +++ b/pkg/services/quotaturbo/data.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" diff --git a/pkg/services/quotaturbo/driverevent.go b/pkg/services/quotaturbo/driverevent.go index abba6a8..75025ca 100644 --- a/pkg/services/quotaturbo/driverevent.go +++ b/pkg/services/quotaturbo/driverevent.go @@ -18,6 +18,7 @@ import ( "math" "runtime" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" ) diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index fcb2702..f217b76 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -23,7 +23,7 @@ import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/constant" - Log "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services" @@ -31,10 +31,8 @@ import ( const moduleName = "quotaturbo" -var log api.Logger func init() { - log = &Log.EmptyLog{} services.Register(moduleName, func() interface{} { return NewQuotaTurbo() }) @@ -58,11 +56,6 @@ func NewQuotaTurbo() *QuotaTurbo { } } -// SetupLog initializes the log interface for the module -func (qt *QuotaTurbo) SetupLog(logger api.Logger) { - log = logger -} - // ID returns the module name func (qt *QuotaTurbo) ID() string { return moduleName diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index 52ca780..aa9d3c9 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/assert" "isula.org/rubik/pkg/common/constant" - Log "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/podmanager" @@ -339,7 +338,6 @@ func TestNewQuotaTurbo(t *testing.T) { testName := "TC1-test otherv functions" t.Run(testName, func(t *testing.T) { got := NewQuotaTurbo() - got.SetupLog(&Log.EmptyLog{}) assert.Equal(t, moduleName, got.ID()) got.Viewer = &podmanager.PodManager{ Pods: &podmanager.PodCache{ diff --git a/pkg/services/servicemanager.go b/pkg/services/servicemanager.go index 09bf007..8130842 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/services/servicemanager.go @@ -17,14 +17,12 @@ package services import ( "context" "fmt" - "reflect" "sync" "time" "k8s.io/apimachinery/pkg/util/wait" "isula.org/rubik/pkg/api" - "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/subscriber" @@ -66,11 +64,6 @@ func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{} return fmt.Errorf("error unmarshaling %s config: %v", name, err) } - if SetLoggerOnService(service, - log.WithCtx(context.WithValue(context.Background(), log.CtxKey(constant.LogEntryKey), name))) { - log.Debugf("set logger for service: %s", name) - } - // try to verify configuration if validator, ok := service.(Validator); ok { if err := validator.Validate(); err != nil { @@ -163,25 +156,6 @@ func (manager *ServiceManager) terminatingRunningServices(err error) error { return err } -// SetLoggerOnService assigns a value to the variable Log member if there is a Log field -func SetLoggerOnService(value interface{}, logger api.Logger) bool { - // 1. call the SetupLog function to set up the log - method := reflect.ValueOf(value).MethodByName("SetupLog") - if method.IsValid() && !method.IsZero() && !method.IsNil() { - method.Call([]reflect.Value{reflect.ValueOf(logger)}) - return true - } - - // 2. look for a member variable named Log - field := reflect.ValueOf(value).Elem().FieldByName("Log") - if field.IsValid() && field.CanSet() && field.Type().String() == "api.Logger" { - field.Set(reflect.ValueOf(logger)) - return true - } - - return false -} - // Setup pre-starts services, such as preparing the environment, etc. func (manager *ServiceManager) Setup(v api.Viewer) error { // only when viewer is prepared -- Gitee From e2d7fdca5497ef3cd00bbd8bba7170e3d604e582 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 10 Mar 2023 15:57:42 +0800 Subject: [PATCH 49/73] optimize: extract the capabilities of the quotaTurbo feature as an independent library Currently, the underlying capabilities of quotaTurbo are coupled with the service itself. This decoupling helps simplify quotaTurbo maintenance, that is, the underlying layer provides the service with a constant interface, and the service implements functions such as reliability assurance Service adaptation is left to follow Signed-off-by: vegbir --- pkg/lib/cpu/quotaturbo/client.go | 54 ++++++ pkg/lib/cpu/quotaturbo/config.go | 144 +++++++++++++++ pkg/lib/cpu/quotaturbo/cpu.go | 111 +++++++++++ pkg/lib/cpu/quotaturbo/cpuquota.go | 253 ++++++++++++++++++++++++++ pkg/lib/cpu/quotaturbo/driverevent.go | 181 ++++++++++++++++++ pkg/lib/cpu/quotaturbo/statusstore.go | 189 +++++++++++++++++++ 6 files changed, 932 insertions(+) create mode 100644 pkg/lib/cpu/quotaturbo/client.go create mode 100644 pkg/lib/cpu/quotaturbo/config.go create mode 100644 pkg/lib/cpu/quotaturbo/cpu.go create mode 100644 pkg/lib/cpu/quotaturbo/cpuquota.go create mode 100644 pkg/lib/cpu/quotaturbo/driverevent.go create mode 100644 pkg/lib/cpu/quotaturbo/statusstore.go diff --git a/pkg/lib/cpu/quotaturbo/client.go b/pkg/lib/cpu/quotaturbo/client.go new file mode 100644 index 0000000..a9e40ba --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/client.go @@ -0,0 +1,54 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-09 +// Description: This file is used for quota turbo client + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" +) + +// Client is quotaTurbo client +type Client struct { + *StatusStore + Driver +} + +// NewClient returns a quotaTurbo client instance +func NewClient() *Client { + return &Client{ + StatusStore: NewStatusStore(), + Driver: &EventDriver{}, + } +} + +// AdjustQuota is used to update status and adjust cgroup quota value +func (c *Client) AdjustQuota() error { + if err := c.updateCPUUtils(); err != nil { + return fmt.Errorf("fail to get current cpu utilization: %v", err) + } + if len(c.cpuQuotas) == 0 { + return nil + } + var errs error + if err := c.updateCPUQuotas(); err != nil { + errs = multierror.Append(errs, err) + } + c.adjustQuota(c.StatusStore) + if err := c.writeQuota(); err != nil { + errs = multierror.Append(errs, err) + } + return errs +} diff --git a/pkg/lib/cpu/quotaturbo/config.go b/pkg/lib/cpu/quotaturbo/config.go new file mode 100644 index 0000000..919501c --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/config.go @@ -0,0 +1,144 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-07 +// Description: This file is used for quota turbo config + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + + "isula.org/rubik/pkg/common/constant" +) + +const ( + defaultHighWaterMark = 60 + defaultAlarmWaterMark = 80 + defaultElevateLimit = 1.0 + defaultSlowFallbackRatio = 0.1 + defaultCPUFloatingLimit = 10.0 +) + +// Config defines configuration of QuotaTurbo +type Config struct { + /* + If the CPU utilization exceeds HighWaterMark, it will trigger a slow fall, + */ + HighWaterMark int `json:"highWaterMark,omitempty"` + /* + If the CPU utilization exceeds the Alarm WaterMark, it will trigger a fast fallback; + otherwise it will trigger a slow increase + */ + AlarmWaterMark int `json:"alarmWaterMark,omitempty"` + /* + Cgroup Root indicates the mount point of the system Cgroup, the default is /sys/fs/cgroup + */ + CgroupRoot string `json:"cgroupRoot,omitempty"` + /* + ElevateLimit is the maximum percentage(%) of the total amount of + a single promotion to the total amount of nodes + Default is 1.0 + */ + ElevateLimit float64 `json:"elevateLimit,omitempty"` + /* + Slow Fallback Ratio is used to control the rate of slow fallback. Default is 0.1 + */ + SlowFallbackRatio float64 `json:"slowFallbackRatio,omitempty"` + /* + CPUFloatingLimit indicates the Upper Percentage Change of the CPU utilization of the node + within the specified time period. + Only when the floating rate is lower than the upper limit can the quota be increased, + and the decrease is not limited + Default is 10.0 + */ + CPUFloatingLimit float64 `json:"cpuFloatingLimit,omitempty"` +} + +// NewConfig returns a quota Turbo config instance with default values +func NewConfig() *Config { + return &Config{ + HighWaterMark: defaultHighWaterMark, + AlarmWaterMark: defaultAlarmWaterMark, + CgroupRoot: constant.DefaultCgroupRoot, + ElevateLimit: defaultElevateLimit, + SlowFallbackRatio: defaultSlowFallbackRatio, + CPUFloatingLimit: defaultCPUFloatingLimit, + } +} + +// validateWaterMark verifies that the WaterMark is set correctly +func (c *Config) validateWaterMark() error { + const minQuotaTurboWaterMark, maxQuotaTurboWaterMark = 0, 100 + outOfRange := func(num int) bool { + return num < minQuotaTurboWaterMark || num > maxQuotaTurboWaterMark + } + if c.AlarmWaterMark <= c.HighWaterMark || outOfRange(c.HighWaterMark) || outOfRange(c.AlarmWaterMark) { + return fmt.Errorf("alarmWaterMark >= highWaterMark, both of which ranges from 0 to 100") + } + return nil +} + +// SetAlarmWaterMark sets AlarmWaterMark of QuotaTurbo +func (c *Config) SetAlarmWaterMark(arg int) error { + tmp := c.AlarmWaterMark + c.AlarmWaterMark = arg + if err := c.validateWaterMark(); err != nil { + c.AlarmWaterMark = tmp + return err + } + return nil +} + +// SetHighWaterMark sets HighWaterMark of QuotaTurbo +func (c *Config) SetHighWaterMark(arg int) error { + tmp := c.HighWaterMark + c.HighWaterMark = arg + if err := c.validateWaterMark(); err != nil { + c.HighWaterMark = tmp + return err + } + return nil +} + +// SetCgroupRoot sets CgroupRoot of QuotaTurbo +func (c *Config) SetCgroupRoot(arg string) { + c.CgroupRoot = arg +} + +// SetlEvateLimit sets ElevateLimit of QuotaTurbo +func (c *Config) SetlEvateLimit(arg float64) error { + if arg < minimumUtilization || arg > maximumUtilization { + return fmt.Errorf("the size range of SingleTotalIncreaseLimit is [0,100]") + } + c.ElevateLimit = arg + return nil +} + +// SetSlowFallbackRatio sets SlowFallbackRatio of QuotaTurbo +func (c *Config) SetSlowFallbackRatio(arg float64) { + c.SlowFallbackRatio = arg +} + +// SetCPUFloatingLimit sets CPUFloatingLimit of QuotaTurbo +func (c *Config) SetCPUFloatingLimit(arg float64) error { + if arg < minimumUtilization || arg > maximumUtilization { + return fmt.Errorf("the size range of SingleTotalIncreaseLimit is [0,100]") + } + c.CPUFloatingLimit = arg + return nil +} + +// GetConfig returns a copy of the QuotaTurbo configuration +func (c *Config) GetConfig() *Config { + copyConf := *c + return ©Conf +} diff --git a/pkg/lib/cpu/quotaturbo/cpu.go b/pkg/lib/cpu/quotaturbo/cpu.go new file mode 100644 index 0000000..72415b7 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/cpu.go @@ -0,0 +1,111 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-08 +// Description: This file is used for computing cpu utilization + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "io/ioutil" + "math" + "strings" + + "isula.org/rubik/pkg/common/util" +) + +const ( + maximumUtilization float64 = 100 + minimumUtilization float64 = 0 +) + +// ProcStat store /proc/stat data +type ProcStat struct { + name string + user float64 + nice float64 + system float64 + idle float64 + iowait float64 + irq float64 + softirq float64 + steal float64 + guest float64 + guestNice float64 + total float64 + busy float64 +} + +// getProcStat create a proc stat object +func getProcStat() (ProcStat, error) { + const ( + procStatFilePath = "/proc/stat" + nameLineNum = 0 + userIndex = 0 + niceIndex = 1 + systemIndex = 2 + idleIndex = 3 + iowaitIndex = 4 + irqIndex = 5 + softirqIndex = 6 + stealIndex = 7 + guestIndex = 8 + guestNiceIndex = 9 + statsFieldsCount = 10 + supportFieldNumber = 11 + ) + data, err := ioutil.ReadFile(procStatFilePath) + if err != nil { + return ProcStat{}, err + } + // format of the first line of the file /proc/stat : + // name user nice system idle iowait irq softirq steal guest guest_nice + line := strings.Fields(strings.Split(string(data), "\n")[0]) + if len(line) < supportFieldNumber { + return ProcStat{}, fmt.Errorf("too few fields and check the kernel version") + } + var fields [statsFieldsCount]float64 + for i := 0; i < statsFieldsCount; i++ { + fields[i], err = util.ParseFloat64(line[i+1]) + if err != nil { + return ProcStat{}, err + } + } + ps := ProcStat{ + name: line[nameLineNum], + user: fields[userIndex], + nice: fields[niceIndex], + system: fields[systemIndex], + idle: fields[idleIndex], + iowait: fields[iowaitIndex], + irq: fields[irqIndex], + softirq: fields[softirqIndex], + steal: fields[stealIndex], + guest: fields[guestIndex], + guestNice: fields[guestNiceIndex], + } + ps.busy = ps.user + ps.system + ps.nice + ps.iowait + ps.irq + ps.softirq + ps.steal + ps.total = ps.busy + ps.idle + return ps, nil +} + +// calculateUtils calculate the CPU utilization rate based on the two interval /proc/stat +func calculateUtils(t1, t2 ProcStat) float64 { + if t2.busy <= t1.busy { + return minimumUtilization + } + if t2.total <= t1.total { + return maximumUtilization + } + return math.Min(maximumUtilization, + math.Max(minimumUtilization, util.Div(t2.busy-t1.busy, t2.total-t1.total)*maximumUtilization)) +} diff --git a/pkg/lib/cpu/quotaturbo/cpuquota.go b/pkg/lib/cpu/quotaturbo/cpuquota.go new file mode 100644 index 0000000..4d89d73 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/cpuquota.go @@ -0,0 +1,253 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: cpu container cpu quota data and methods + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "path" + "time" + + "github.com/hashicorp/go-multierror" + + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +const ( + // numberOfRestrictedCycles is the number of periods in which the quota limits the CPU usage. + numberOfRestrictedCycles = 60 + // The default value of the cfs_period_us file is 100ms + defaultCFSPeriodUs int64 = 100000 +) + +var ( + cpuPeriodKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} + cpuQuotaKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} + cpuAcctUsageKey = &cgroup.Key{SubSys: "cpuacct", FileName: "cpuacct.usage"} + cpuStatKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.stat"} +) + +// cpuUsage cpu time used by the container at timestamp +type cpuUsage struct { + timestamp int64 + usage int64 +} + +// CPUQuota stores the CPU quota information of a single container. +type CPUQuota struct { + // basic container information + *cgroup.Hierarchy + // expect cpu limit + cpuLimit float64 + // current throttling data for the container + curThrottle *cgroup.CPUStat + // previous throttling data for container + preThrottle *cgroup.CPUStat + // container cfs_period_us + period int64 + // current cpu quota of the container + curQuota int64 + // cpu quota of the container in the next period + nextQuota int64 + // the delta of the cpu quota to be adjusted based on the decision. + quotaDelta float64 + // the upper limit of the container cpu quota + heightLimit float64 + // maximum quota that can be used by a container in the next period, + // calculated based on the total usage in the past N-1 cycles + maxQuotaNextPeriod float64 + // container cpu usage sequence + cpuUsages []cpuUsage +} + +// NewCPUQuota create a cpu quota object +func NewCPUQuota(h *cgroup.Hierarchy, cpuLimit float64) (*CPUQuota, error) { + var defaultQuota = cpuLimit * float64(defaultCFSPeriodUs) + cq := &CPUQuota{ + Hierarchy: h, + cpuLimit: cpuLimit, + cpuUsages: make([]cpuUsage, 0), + quotaDelta: 0, + curThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, + preThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, + period: defaultCFSPeriodUs, + curQuota: int64(defaultQuota), + nextQuota: int64(defaultQuota), + heightLimit: defaultQuota, + maxQuotaNextPeriod: defaultQuota, + } + if err := cq.update(); err != nil { + return cq, err + } + // The throttle data before and after the initialization is the same. + cq.preThrottle = cq.curThrottle + return cq, nil +} + +func (c *CPUQuota) update() error { + var errs error + if err := c.updatePeriod(); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.updateThrottle(); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.updateQuota(); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.updateUsage(); err != nil { + errs = multierror.Append(errs, err) + } + if errs != nil { + return errs + } + return nil +} + +func (c *CPUQuota) updatePeriod() error { + us, err := c.GetCgroupAttr(cpuPeriodKey).Int64() + // If an error occurs, the period remains unchanged or the default value is used. + if err != nil { + return err + } + c.period = us + return nil +} + +func (c *CPUQuota) updateThrottle() error { + // update suppression times and duration + // if data cannot be obtained from cpu.stat, the value remains unchanged. + c.preThrottle = c.curThrottle + cs, err := c.GetCgroupAttr(cpuStatKey).CPUStat() + if err != nil { + return err + } + c.curThrottle = cs + return nil +} + +func (c *CPUQuota) updateQuota() error { + c.quotaDelta = 0 + curQuota, err := c.GetCgroupAttr(cpuQuotaKey).Int64() + if err != nil { + return err + } + c.curQuota = curQuota + return nil +} + +func (c *CPUQuota) updateUsage() error { + latest, err := c.GetCgroupAttr(cpuAcctUsageKey).Int64() + if err != nil { + return err + } + c.cpuUsages = append(c.cpuUsages, cpuUsage{timestamp: time.Now().UnixNano(), usage: latest}) + // ensure that the CPU usage of the container does not exceed the upper limit. + if len(c.cpuUsages) >= numberOfRestrictedCycles { + c.cpuUsages = c.cpuUsages[1:] + } + return nil +} + +func writeQuota(mountPoint string, paths []string, delta int64) error { + type cgroupQuotaPair struct { + h *cgroup.Hierarchy + value int64 + } + var ( + writed []cgroupQuotaPair + save = func(mountPoint, path string, delta int64) error { + h := cgroup.NewHierarchy(mountPoint, path) + curQuota, err := h.GetCgroupAttr(cpuQuotaKey).Int64() + if err != nil { + return fmt.Errorf("error getting cgroup %v quota: %v", path, err) + } + if curQuota == -1 { + return nil + } + + nextQuota := curQuota + delta + if err := h.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(nextQuota)); err != nil { + return fmt.Errorf("error setting cgroup %v quota (%v to %v): %v", path, curQuota, nextQuota, err) + } + writed = append(writed, cgroupQuotaPair{h: h, value: curQuota}) + return nil + } + + fallback = func() { + for _, w := range writed { + if err := w.h.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(w.value)); err != nil { + fmt.Printf("error recovering cgroup %v quota %v\n", w.h.Path, w.value) + } + } + } + ) + + if delta > 0 { + // update the parent cgroup first, then update the child cgroup + for i, j := 0, len(paths)-1; i < j; i, j = i+1, j-1 { + paths[i], paths[j] = paths[j], paths[i] + } + } + + for _, path := range paths { + if err := save(mountPoint, path, delta); err != nil { + fallback() + return err + } + } + return nil +} + +// writeQuota use to modify quota for cgroup +func (c *CPUQuota) writeQuota() error { + var ( + delta = c.nextQuota - c.curQuota + paths []string + fullPath = c.Path + ) + if delta == 0 { + return nil + } + // the upper cgroup needs to be updated synchronously + if len(fullPath) == 0 { + return fmt.Errorf("invalid cgroup path: %v", fullPath) + } + for { + /* + a non-slash start will end up with . + start with a slash and end up with slash + */ + if fullPath == "." || fullPath == "/" || fullPath == "kubepods" || fullPath == "/kubepods" { + break + } + paths = append(paths, fullPath) + fullPath = path.Dir(fullPath) + } + if len(paths) == 0 { + return fmt.Errorf("empty cgroup path") + } + if err := writeQuota(c.MountPoint, paths, delta); err != nil { + return err + } + c.curQuota = c.nextQuota + return nil +} + +func (c *CPUQuota) recoverQuota() error { + // period ranges from 1000(us) to 1000000(us) and does not overflow. + c.nextQuota = int64(c.cpuLimit * float64(c.period)) + return c.writeQuota() +} diff --git a/pkg/lib/cpu/quotaturbo/driverevent.go b/pkg/lib/cpu/quotaturbo/driverevent.go new file mode 100644 index 0000000..bd81973 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/driverevent.go @@ -0,0 +1,181 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: event driver method for quota turbo + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "math" + "runtime" + + "isula.org/rubik/pkg/common/util" +) + +// Driver uses different methods based on different policies. +type Driver interface { + // adjustQuota calculate the quota in the next period based on the customized policy, upper limit, and quota. + adjustQuota(status *StatusStore) +} + +// EventDriver event based quota adjustment driver. +type EventDriver struct{} + +// adjustQuota calculates quota delta based on events +func (e *EventDriver) adjustQuota(status *StatusStore) { + e.slowFallback(status) + e.fastFallback(status) + // Ensure that the CPU usage does not change by more than 10% within one minute. + // Otherwise, the available quota rollback continues but does not increase. + if !sharpFluctuates(status) { + e.elevate(status) + } else { + fmt.Printf("CPU utilization fluctuates by more than %.2f\n", status.CPUFloatingLimit) + } + for _, c := range status.cpuQuotas { + // get height limit + const easingMultiple = 2.0 + // c.Period + c.heightLimit = easingMultiple * c.cpuLimit * float64(c.period) + // get the maximum available ensuring that the overall utilization does not exceed the limit. + c.maxQuotaNextPeriod = getMaxQuota(c) + // c.Period ranges from 1000(us) to 1000000(us) and does not overflow. + c.nextQuota = int64(math.Max(math.Min(float64(c.curQuota)+c.quotaDelta, c.maxQuotaNextPeriod), + c.cpuLimit*float64(c.period))) + } +} + +// elevate boosts when cpu is suppressed +func (e *EventDriver) elevate(status *StatusStore) { + // the CPU usage of the current node is lower than the warning watermark. + // U + R <= a & a > U ======> a - U >= R && a - U > 0 =====> a - U >= R + if float64(status.AlarmWaterMark)-status.getLastCPUUtil() < status.ElevateLimit { + return + } + // sumDelta : total number of cores to be adjusted + var sumDelta float64 = 0 + delta := make(map[string]float64, 0) + for id, c := range status.cpuQuotas { + if c.curThrottle.NrThrottled > c.preThrottle.NrThrottled { + delta[id] = NsToUs(c.curThrottle.ThrottledTime-c.preThrottle.ThrottledTime) / + float64(c.curThrottle.NrThrottled-c.preThrottle.NrThrottled) / float64(c.period) + sumDelta += delta[id] + } + } + // the container quota does not need to be increased in this round. + if sumDelta == 0 { + return + } + // the total increase cannot exceed ( status.SingleTotalIncreaseLimit% ) of the total available CPUs of the node. + A := math.Min(sumDelta, util.PercentageToDecimal(status.ElevateLimit)*float64(runtime.NumCPU())) + coefficient := A / sumDelta + for id, quotaDelta := range delta { + status.cpuQuotas[id].quotaDelta += coefficient * quotaDelta * float64(status.cpuQuotas[id].period) + } +} + +// fastFallback decreases the quota to ensure that the CPU utilization of the node is below the warning water level +// when the water level exceeds the warning water level. +func (e *EventDriver) fastFallback(status *StatusStore) { + // The CPU usage of the current node is greater than the warning watermark, triggering a fast rollback. + if float64(status.AlarmWaterMark) > status.getLastCPUUtil() { + return + } + // sub: the total number of CPU quotas to be reduced on a node. + sub := util.PercentageToDecimal(float64(status.AlarmWaterMark)-status.getLastCPUUtil()) * float64(runtime.NumCPU()) + // sumDelta ：total number of cpu cores that can be decreased. + var sumDelta float64 = 0 + delta := make(map[string]float64, 0) + for id, c := range status.cpuQuotas { + delta[id] = float64(c.curQuota)/float64(c.period) - c.cpuLimit + sumDelta += delta[id] + } + if sumDelta <= 0 { + return + } + // proportional adjustment of each business quota. + for id, quotaDelta := range delta { + status.cpuQuotas[id].quotaDelta += (quotaDelta / sumDelta) * sub * float64(status.cpuQuotas[id].period) + } +} + +// slowFallback triggers quota callback of unpressed containers when the CPU utilization exceeds the control watermark. +func (e *EventDriver) slowFallback(status *StatusStore) { + // The CPU usage of the current node is greater than the high watermark, triggering a slow rollback. + if float64(status.HighWaterMark) > status.getLastCPUUtil() { + return + } + coefficient := (status.getLastCPUUtil() - float64(status.HighWaterMark)) / + float64(status.AlarmWaterMark-status.HighWaterMark) * status.SlowFallbackRatio + for id, c := range status.cpuQuotas { + originQuota := int64(c.cpuLimit * float64(c.period)) + if c.curQuota > originQuota && c.curThrottle.NrThrottled == c.preThrottle.NrThrottled { + status.cpuQuotas[id].quotaDelta += coefficient * float64(originQuota-c.curQuota) + } + } +} + +// sharpFluctuates checks whether the node CPU utilization exceeds the specified value within one minute. +func sharpFluctuates(status *StatusStore) bool { + var ( + min float64 = maximumUtilization + max float64 = minimumUtilization + ) + for _, u := range status.cpuUtils { + min = math.Min(min, u.util) + max = math.Max(max, u.util) + } + return max-min > status.CPUFloatingLimit +} + +// getMaxQuota calculate the maximum available quota in the next period based on the container CPU usage in N-1 periods. +func getMaxQuota(c *CPUQuota) float64 { + if len(c.cpuUsages) <= 1 { + return c.heightLimit + } + // the time unit is nanosecond + first := c.cpuUsages[0] + last := c.cpuUsages[len(c.cpuUsages)-1] + timeDelta := NsToUs(last.timestamp - first.timestamp) + coefficient := float64(len(c.cpuUsages)) / float64(len(c.cpuUsages)-1) + maxAvailable := c.cpuLimit * timeDelta * coefficient + used := NsToUs(last.usage - first.usage) + remainingUsage := maxAvailable - used + origin := c.cpuLimit * float64(c.period) + const ( + // To prevent sharp service jitters, the Rubik proactively decreases the traffic in advance + // when the available balance reaches a certain threshold. + // The limitMultiplier is used to control the relationship between the upper limit and the threshold. + // Experiments show that the value 3 is efficient and secure. + limitMultiplier = 3 + precision = 1e-10 + ) + var threshold = limitMultiplier * c.heightLimit + remainingQuota := util.Div(remainingUsage, timeDelta, math.MaxFloat64, precision) * + float64(len(c.cpuUsages)-1) * float64(c.period) + + // gradually decrease beyond the threshold to prevent sudden dips. + res := remainingQuota + if remainingQuota <= threshold { + res = origin + util.Div((c.heightLimit-origin)*remainingQuota, threshold, threshold, precision) + } + // The utilization must not exceed the height limit and must not be less than the cpuLimit. + return math.Max(math.Min(res, c.heightLimit), origin) +} + +// NsToUs converts nanoseconds into microseconds +func NsToUs(ns int64) float64 { + // number of nanoseconds contained in 1 microsecond + const nanoSecPerMicroSec float64 = 1000 + return util.Div(float64(ns), nanoSecPerMicroSec) +} diff --git a/pkg/lib/cpu/quotaturbo/statusstore.go b/pkg/lib/cpu/quotaturbo/statusstore.go new file mode 100644 index 0000000..0b7e067 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/statusstore.go @@ -0,0 +1,189 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: QuotaTurbo Status Store + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "fmt" + "path/filepath" + "runtime" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +// cpuUtil is used to store the cpu usage at a specific time +type cpuUtil struct { + timestamp int64 + util float64 +} + +// StatusStore is the information of node/containers obtained for quotaTurbo +type StatusStore struct { + // configuration of the QuotaTurbo + *Config + // ensuring Concurrent Sequential Consistency + sync.RWMutex + // map between container IDs and container CPU quota + cpuQuotas map[string]*CPUQuota + // cpu utilization sequence for N consecutive cycles + cpuUtils []cpuUtil + // /proc/stat of the previous period + lastProcStat ProcStat +} + +// NewStatusStore returns a pointer to StatusStore +func NewStatusStore() *StatusStore { + return &StatusStore{ + Config: NewConfig(), + lastProcStat: ProcStat{ + total: -1, + busy: -1, + }, + cpuQuotas: make(map[string]*CPUQuota, 0), + cpuUtils: make([]cpuUtil, 0), + } +} + +// AddCgroup adds cgroup need to be adjusted +func (store *StatusStore) AddCgroup(cgroupPath string, cpuLimit float64) error { + if len(cgroupPath) == 0 { + return fmt.Errorf("cgroup path should not be empty") + } + if store.CgroupRoot == "" { + return fmt.Errorf("undefined cgroup mount point, please set it firstly") + } + h := cgroup.NewHierarchy(store.CgroupRoot, cgroupPath) + if !isAdjustmentAllowed(h, cpuLimit) { + return fmt.Errorf("cgroup not allow to adjust") + } + c, err := NewCPUQuota(h, cpuLimit) + if err != nil { + return fmt.Errorf("error creating cpu quota: %v", err) + } + store.Lock() + store.cpuQuotas[cgroupPath] = c + store.Unlock() + return nil +} + +// RemoveCgroup deletes cgroup that do not need to be adjusted. +func (store *StatusStore) RemoveCgroup(cgroupPath string) error { + store.RLock() + cq, ok := store.cpuQuotas[cgroupPath] + store.RUnlock() + if !ok { + return nil + } + safeDel := func(id string) error { + store.Lock() + delete(store.cpuQuotas, id) + store.Unlock() + return nil + } + + if !util.PathExist(filepath.Join(cq.MountPoint, "cpu", cq.Path)) { + return safeDel(cgroupPath) + } + if err := cq.recoverQuota(); err != nil { + return fmt.Errorf("fail to recover cpu.cfs_quota_us for cgroup %s : %v", cq.Path, err) + } + return safeDel(cgroupPath) +} + +// getLastCPUUtil obtain the latest cpu utilization +func (store *StatusStore) getLastCPUUtil() float64 { + if len(store.cpuUtils) == 0 { + return 0 + } + return store.cpuUtils[len(store.cpuUtils)-1].util +} + +// updateCPUUtils updates the cpu usage of a node +func (store *StatusStore) updateCPUUtils() error { + var ( + curUtil float64 = 0 + index = 0 + t cpuUtil + ) + ps, err := getProcStat() + if err != nil { + return err + } + if store.lastProcStat.total >= 0 { + curUtil = calculateUtils(store.lastProcStat, ps) + } + store.lastProcStat = ps + cur := time.Now().UnixNano() + store.cpuUtils = append(store.cpuUtils, cpuUtil{ + timestamp: cur, + util: curUtil, + }) + // retain utilization data for only one minute + const minuteTimeDelta = int64(time.Minute) + for index, t = range store.cpuUtils { + if cur-t.timestamp <= minuteTimeDelta { + break + } + } + if index > 0 { + store.cpuUtils = store.cpuUtils[index:] + } + return nil +} + +func (store *StatusStore) updateCPUQuotas() error { + var errs error + for id, c := range store.cpuQuotas { + if err := c.update(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error updating cpu quota %v: %v", id, err)) + } + } + return errs +} + +// writeQuota writes the calculated quota value into the cgroup file and takes effect +func (store *StatusStore) writeQuota() error { + var errs error + for id, c := range store.cpuQuotas { + if err := c.writeQuota(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("error writing cgroup quota %v: %v", id, err)) + } + } + return errs +} + +// isAdjustmentAllowed judges whether quota adjustment is allowed +func isAdjustmentAllowed(h *cgroup.Hierarchy, cpuLimit float64) bool { + // 1. containers whose cgroup path does not exist are not considered. + if !util.PathExist(filepath.Join(h.MountPoint, "cpu", h.Path)) { + return false + } + + /* + 2. abnormal CPULimit + a). containers that do not limit the quota => cpuLimit = 0 + b). cpuLimit = 0 : k8s allows the CPULimit to be 0, but the quota is not limited. + c). cpuLimit >= all cores + */ + if cpuLimit <= 0 || + cpuLimit >= float64(runtime.NumCPU()) { + return false + } + return true +} -- Gitee From dd83a91234dba91c245cc7ae662f9533c7a2de16 Mon Sep 17 00:00:00 2001 From: wujing Date: Fri, 10 Mar 2023 16:25:04 +0800 Subject: [PATCH 50/73] DT: add testcase for client and cpuquota submodule of quotaturbo Signed-off-by: wujing --- pkg/lib/cpu/quotaturbo/client_test.go | 111 +++++++++ pkg/lib/cpu/quotaturbo/cpu_test.go | 57 +++++ pkg/lib/cpu/quotaturbo/cpuquota_test.go | 305 ++++++++++++++++++++++++ 3 files changed, 473 insertions(+) create mode 100644 pkg/lib/cpu/quotaturbo/client_test.go create mode 100644 pkg/lib/cpu/quotaturbo/cpu_test.go create mode 100644 pkg/lib/cpu/quotaturbo/cpuquota_test.go diff --git a/pkg/lib/cpu/quotaturbo/client_test.go b/pkg/lib/cpu/quotaturbo/client_test.go new file mode 100644 index 0000000..9389fa1 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/client_test.go @@ -0,0 +1,111 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-09 +// Description: This file is used for testing quota turbo client + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +// TestClient_AdjustQuota tests AdjustQuota of Client +func TestClient_AdjustQuota(t *testing.T) { + const ( + contPath = "kubepods/testPod1/testCon1" + podPath = "kubepods/testPod1" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + stat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + quota = "200000" + period = "100000" + usage = "1234567" + ) + tests := []struct { + name string + wantErr bool + pre func(t *testing.T, c *Client) + post func(t *testing.T) + }{ + { + name: "TC1-empty CPUQuotas", + wantErr: false, + }, + { + name: "TC2-fail to updateCPUQuota causing absent of path", + pre: func(t *testing.T, c *Client) { + c.SetCgroupRoot(constant.TmpTestDir) + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath)) + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath), constant.DefaultDirMode) + + assert.Equal(t, 0, len(c.cpuQuotas)) + c.AddCgroup(contPath, float64(runtime.NumCPU())) + c.cpuQuotas[contPath] = &CPUQuota{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: c.CgroupRoot, + Path: contPath, + }, + cpuLimit: float64(runtime.NumCPU()) - 1, + curThrottle: &cgroup.CPUStat{}, + preThrottle: &cgroup.CPUStat{}, + } + assert.Equal(t, 1, len(c.cpuQuotas)) + }, + post: func(t *testing.T) { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath)) + }, + wantErr: true, + }, + { + name: "TC3-success", + pre: func(t *testing.T, c *Client) { + c.SetCgroupRoot(constant.TmpTestDir) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuPeriodFile), period) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuQuotaFile), quota) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpuacct", contPath, cpuUsageFile), usage) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuStatFile), stat) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", podPath, cpuPeriodFile), period) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", podPath, cpuQuotaFile), quota) + assert.NoError(t, c.AddCgroup(contPath, float64(runtime.NumCPU())-1)) + assert.Equal(t, 1, len(c.cpuQuotas)) + }, + post: func(t *testing.T) { + try.RemoveAll(constant.TmpTestDir) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewClient() + if tt.pre != nil { + tt.pre(t, c) + } + if err := c.AdjustQuota(); (err != nil) != tt.wantErr { + t.Errorf("Client.AdjustQuota() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/lib/cpu/quotaturbo/cpu_test.go b/pkg/lib/cpu/quotaturbo/cpu_test.go new file mode 100644 index 0000000..ead2d05 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/cpu_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for testing cpu.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestCalculateUtils tests calculateUtils +func TestCalculateUtils(t *testing.T) { + var ( + n1 float64 = 1 + n2 float64 = 2 + n3 float64 = 3 + n4 float64 = 4 + ) + + var ( + t1 = ProcStat{ + total: n2, + busy: n1, + } + t2 = ProcStat{ + total: n4, + busy: n2, + } + t3 = ProcStat{ + total: n3, + busy: n3, + } + ) + // normal return result + const ( + util float64 = 50 + minimumUtilization float64 = 0 + maximumUtilization float64 = 100 + ) + assert.Equal(t, util, calculateUtils(t1, t2)) + // busy errors + assert.Equal(t, minimumUtilization, calculateUtils(t2, t1)) + // total errors + assert.Equal(t, maximumUtilization, calculateUtils(t2, t3)) +} diff --git a/pkg/lib/cpu/quotaturbo/cpuquota_test.go b/pkg/lib/cpu/quotaturbo/cpuquota_test.go new file mode 100644 index 0000000..feba814 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/cpuquota_test.go @@ -0,0 +1,305 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for testing cpu_quota + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "path" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +// TestNewCPUQuota tests NewCPUQuota +func TestNewCPUQuota(t *testing.T) { + const ( + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + validStat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + throttleTime int64 = 1 + quota = "200000" + quotaValue int64 = 200000 + period = "100000" + periodValue int64 = 100000 + usage = "1234567" + usageValue int64 = 1234567 + ) + + var ( + cgPath = "kubepods/testPod1/testCon1" + h = &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: cgPath, + } + + contPath = filepath.Join(constant.TmpTestDir, "cpu", cgPath, "") + contPeriodPath = filepath.Join(contPath, cpuPeriodFile) + contQuotaPath = filepath.Join(contPath, cpuQuotaFile) + contUsagePath = filepath.Join(constant.TmpTestDir, "cpuacct", cgPath, cpuUsageFile) + contStatPath = filepath.Join(contPath, cpuStatFile) + ) + + try.RemoveAll(constant.TmpTestDir) + try.MkdirAll(contPath, constant.DefaultDirMode) + try.MkdirAll(path.Dir(contUsagePath), constant.DefaultDirMode) + defer try.RemoveAll(constant.TmpTestDir) + + const cpuLimit = 2.0 + // absent of period file + try.RemoveAll(contPeriodPath) + _, err := NewCPUQuota(h, cpuLimit) + assert.Error(t, err, "should lacking of period file") + try.WriteFile(contPeriodPath, period) + + // absent of throttle file + try.RemoveAll(contStatPath) + _, err = NewCPUQuota(h, cpuLimit) + assert.Error(t, err, "should lacking of throttle file") + try.WriteFile(contStatPath, validStat) + + // absent of quota file + try.RemoveAll(contQuotaPath) + _, err = NewCPUQuota(h, cpuLimit) + assert.Error(t, err, "should lacking of quota file") + try.WriteFile(contQuotaPath, quota) + + // absent of usage file + try.RemoveAll(contUsagePath) + _, err = NewCPUQuota(h, cpuLimit) + assert.Error(t, err, "should lacking of usage file") + try.WriteFile(contUsagePath, usage) + + cq, err := NewCPUQuota(h, cpuLimit) + assert.NoError(t, err) + assert.Equal(t, usageValue, cq.cpuUsages[0].usage) + assert.Equal(t, quotaValue, cq.curQuota) + assert.Equal(t, periodValue, cq.period) + + cu := make([]cpuUsage, numberOfRestrictedCycles) + for i := 0; i < numberOfRestrictedCycles; i++ { + cu[i] = cpuUsage{} + } + cq.cpuUsages = cu + assert.NoError(t, cq.updateUsage()) +} + +// TestCPUQuota_WriteQuota tests WriteQuota of CPUQuota +func TestCPUQuota_WriteQuota(t *testing.T) { + const ( + largerQuota = "200000" + largerQuotaVal int64 = 200000 + smallerQuota = "100000" + smallerQuotaVal int64 = 100000 + unlimitedQuota = "-1" + unlimitedQuotaVal int64 = -1 + periodUs = "100000" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + ) + + var ( + cgPath = "kubepods/testPod1/testCon1" + contPath = filepath.Join(constant.TmpTestDir, "cpu", cgPath, "") + podPeriodPath = filepath.Join(filepath.Dir(contPath), cpuPeriodFile) + podQuotaPath = filepath.Join(filepath.Dir(contPath), cpuQuotaFile) + contPeriodPath = filepath.Join(contPath, cpuPeriodFile) + contQuotaPath = filepath.Join(contPath, cpuQuotaFile) + assertValue = func(t *testing.T, paths []string, value string) { + for _, p := range paths { + data, err := util.ReadFile(p) + assert.NoError(t, err) + assert.Equal(t, value, strings.TrimSpace(string(data))) + } + } + ) + + try.RemoveAll(constant.TmpTestDir) + defer try.RemoveAll(constant.TmpTestDir) + + type fields struct { + Hierarchy *cgroup.Hierarchy + curQuota int64 + nextQuota int64 + } + tests := []struct { + name string + pre func() + fields fields + post func(t *testing.T, cq *CPUQuota) + wantErr bool + }{ + { + name: "TC1-empty cgroup path", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{}, + nextQuota: largerQuotaVal, + curQuota: smallerQuotaVal, + }, + wantErr: true, + }, + { + name: "TC2-fail to get paths", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + Path: "/", + }, + nextQuota: largerQuotaVal, + curQuota: smallerQuotaVal, + }, + wantErr: true, + }, + { + name: "TC3-None of the paths exist", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: "kubepods/testPod1/testCon1", + }, + nextQuota: largerQuotaVal, + curQuota: smallerQuotaVal, + }, + wantErr: true, + }, + { + name: "TC4-Only pod path existed", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: "kubepods/testPod1/testCon1", + }, + // write the pod first and then write the container + nextQuota: largerQuotaVal, + curQuota: smallerQuotaVal, + }, + pre: func() { + try.WriteFile(podQuotaPath, smallerQuota) + try.WriteFile(podPeriodPath, periodUs) + try.RemoveAll(contQuotaPath) + try.RemoveAll(contPeriodPath) + }, + post: func(t *testing.T, cq *CPUQuota) { + // Unable to write to container, so restore pod as it is + assertValue(t, []string{podQuotaPath}, smallerQuota) + assert.Equal(t, smallerQuotaVal, cq.curQuota) + }, + wantErr: true, + }, + { + name: "TC5-success delta > 0", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: "kubepods/testPod1/testCon1", + }, + nextQuota: largerQuotaVal, + curQuota: smallerQuotaVal, + }, + pre: func() { + try.WriteFile(podQuotaPath, smallerQuota) + try.WriteFile(podPeriodPath, periodUs) + try.WriteFile(contQuotaPath, smallerQuota) + try.WriteFile(contPeriodPath, periodUs) + }, + post: func(t *testing.T, cq *CPUQuota) { + assertValue(t, []string{podQuotaPath, contQuotaPath}, largerQuota) + assert.Equal(t, largerQuotaVal, cq.curQuota) + }, + wantErr: false, + }, + { + name: "TC6-success delta < 0", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: "kubepods/testPod1/testCon1", + }, + nextQuota: smallerQuotaVal, + curQuota: largerQuotaVal, + }, + pre: func() { + try.WriteFile(podQuotaPath, largerQuota) + try.WriteFile(podPeriodPath, periodUs) + try.WriteFile(contQuotaPath, largerQuota) + try.WriteFile(contPeriodPath, periodUs) + }, + post: func(t *testing.T, cq *CPUQuota) { + assertValue(t, []string{podQuotaPath, contQuotaPath}, smallerQuota) + assert.Equal(t, smallerQuotaVal, cq.curQuota) + }, + wantErr: false, + }, + { + name: "TC6.1-success delta < 0 unlimited pod", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: "kubepods/testPod1/testCon1", + }, + nextQuota: smallerQuotaVal, + curQuota: largerQuotaVal, + }, + pre: func() { + try.WriteFile(podQuotaPath, unlimitedQuota) + try.WriteFile(podPeriodPath, periodUs) + try.WriteFile(contQuotaPath, largerQuota) + try.WriteFile(contPeriodPath, periodUs) + }, + post: func(t *testing.T, cq *CPUQuota) { + assertValue(t, []string{contQuotaPath}, smallerQuota) + assertValue(t, []string{podQuotaPath}, unlimitedQuota) + assert.Equal(t, smallerQuotaVal, cq.curQuota) + }, + wantErr: false, + }, + { + name: "TC7-success delta = 0", + fields: fields{ + Hierarchy: &cgroup.Hierarchy{}, + nextQuota: smallerQuotaVal, + curQuota: smallerQuotaVal, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &CPUQuota{ + Hierarchy: tt.fields.Hierarchy, + curQuota: tt.fields.curQuota, + nextQuota: tt.fields.nextQuota, + } + if tt.pre != nil { + tt.pre() + } + if err := c.writeQuota(); (err != nil) != tt.wantErr { + t.Errorf("CPUQuota.WriteQuota() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post(t, c) + } + }) + } +} -- Gitee From 4775944262d615bab19545f8914a245e7da54ff8 Mon Sep 17 00:00:00 2001 From: wujing Date: Fri, 10 Mar 2023 16:35:13 +0800 Subject: [PATCH 51/73] DT: add testcase for the config submodeule of quotaTurbo Signed-off-by: wujing --- pkg/lib/cpu/quotaturbo/config_test.go | 282 ++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 pkg/lib/cpu/quotaturbo/config_test.go diff --git a/pkg/lib/cpu/quotaturbo/config_test.go b/pkg/lib/cpu/quotaturbo/config_test.go new file mode 100644 index 0000000..e41c487 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/config_test.go @@ -0,0 +1,282 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-07 +// Description: This file is used for testing quota turbo config + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" +) + +func TestConfig_SetAlarmWaterMark(t *testing.T) { + type fields struct { + HighWaterMark int + } + type args struct { + arg int + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "TC1-set alarmWaterMark successfully", + fields: fields{ + HighWaterMark: 60, + }, + args: args{ + arg: 100, + }, + wantErr: false, + }, + { + name: "TC2-alarmWaterMark = highwatermark", + fields: fields{ + HighWaterMark: 60, + }, + args: args{ + arg: 60, + }, + wantErr: true, + }, + { + name: "TC2.1-alarmWaterMark < highwatermark", + fields: fields{ + HighWaterMark: 60, + }, + args: args{ + arg: 59, + }, + wantErr: true, + }, + { + name: "TC3-alarmWaterMark > 100", + fields: fields{ + HighWaterMark: 60, + }, + args: args{ + arg: 101, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Config{ + HighWaterMark: tt.fields.HighWaterMark, + } + if err := c.SetAlarmWaterMark(tt.args.arg); (err != nil) != tt.wantErr { + t.Errorf("Config.SetAlarmWaterMark() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestConfig_SetHighWaterMark tests SetHighWaterMark of Config +func TestConfig_SetHighWaterMark(t *testing.T) { + type fields struct { + AlarmWaterMark int + } + type args struct { + arg int + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "TC1-set highWaterMark successfully", + fields: fields{ + AlarmWaterMark: 80, + }, + args: args{ + arg: 10, + }, + wantErr: false, + }, + { + name: "TC2-alarmWaterMark = highwatermark", + fields: fields{ + AlarmWaterMark: 80, + }, + args: args{ + arg: 80, + }, + wantErr: true, + }, + { + name: "TC2.1-alarmWaterMark < highwatermark", + fields: fields{ + AlarmWaterMark: 80, + }, + args: args{ + arg: 81, + }, + wantErr: true, + }, + { + name: "TC3-highWaterMark < 0", + fields: fields{ + AlarmWaterMark: 60, + }, + args: args{ + arg: -1, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Config{ + AlarmWaterMark: tt.fields.AlarmWaterMark, + } + if err := c.SetHighWaterMark(tt.args.arg); (err != nil) != tt.wantErr { + t.Errorf("Config.SetHighWaterMark() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestConfig_SetlEvateLimit tests SetlEvateLimit of Config +func TestConfig_SetlEvateLimit(t *testing.T) { + const ( + normal = 2.0 + larger = 100.01 + negative = -0.01 + ) + tests := []struct { + name string + arg float64 + wantErr bool + }{ + { + name: "TC1-set EvateLimit successfully", + arg: normal, + wantErr: false, + }, + { + name: "TC2-too large EvateLimit", + arg: larger, + wantErr: true, + }, + { + name: "TC3-negative EvateLimit", + arg: negative, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfig() + err := c.SetlEvateLimit(tt.arg) + if (err != nil) != tt.wantErr { + t.Errorf("Config.SetlEvateLimit() error = %v, wantErr %v", err, tt.wantErr) + } + if err == nil { + assert.Equal(t, tt.arg, c.ElevateLimit) + } else { + assert.Equal(t, defaultElevateLimit, c.ElevateLimit) + } + }) + } +} + +// TestConfig_SetCPUFloatingLimit tests SetCPUFloatingLimit of Config +func TestConfig_SetCPUFloatingLimit(t *testing.T) { + const ( + normal = 20.0 + larger = 100.01 + negative = -0.01 + ) + tests := []struct { + name string + arg float64 + wantErr bool + }{ + { + name: "TC1-set CPUFloatingLimit successfully", + arg: normal, + wantErr: false, + }, + { + name: "TC2-too large CPUFloatingLimit", + arg: larger, + wantErr: true, + }, + { + name: "TC3-negative CPUFloatingLimit", + arg: negative, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewConfig() + err := c.SetCPUFloatingLimit(tt.arg) + if (err != nil) != tt.wantErr { + t.Errorf("Config.SetCPUFloatingLimit() error = %v, wantErr %v", err, tt.wantErr) + } + if err == nil { + assert.Equal(t, tt.arg, c.CPUFloatingLimit) + } else { + assert.Equal(t, defaultCPUFloatingLimit, c.CPUFloatingLimit) + } + }) + } +} + +// TestOther tests other function of Config +func TestOther(t *testing.T) { + tests := []struct { + name string + want *Config + }{ + { + name: "TC1-test other", + want: &Config{ + HighWaterMark: defaultHighWaterMark, + AlarmWaterMark: defaultAlarmWaterMark, + CgroupRoot: constant.DefaultCgroupRoot, + ElevateLimit: defaultElevateLimit, + SlowFallbackRatio: defaultSlowFallbackRatio, + CPUFloatingLimit: defaultCPUFloatingLimit, + }, + }, + } + const slowFallback = 3.0 + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewConfig() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewConfig() = %v, want %v", got, tt.want) + } + got.SetCgroupRoot(constant.TmpTestDir) + assert.Equal(t, got.CgroupRoot, constant.TmpTestDir) + got.SetSlowFallbackRatio(slowFallback) + assert.Equal(t, got.SlowFallbackRatio, slowFallback) + copyConf := got.GetConfig() + if !reflect.DeepEqual(got, copyConf) { + t.Errorf("GetConfig() = %v, want %v", got, copyConf) + } + }) + } +} -- Gitee From 5e4b05ac43689397b8f2b2b51201d8a25e3e677c Mon Sep 17 00:00:00 2001 From: wujing Date: Fri, 10 Mar 2023 16:45:28 +0800 Subject: [PATCH 52/73] DT: add testcase for driverevent and statusstore submodule of quotaTurbo Signed-off-by: wujing --- pkg/lib/cpu/quotaturbo/driverevent_test.go | 604 +++++++++++++++++++++ pkg/lib/cpu/quotaturbo/statusstore_test.go | 568 +++++++++++++++++++ 2 files changed, 1172 insertions(+) create mode 100644 pkg/lib/cpu/quotaturbo/driverevent_test.go create mode 100644 pkg/lib/cpu/quotaturbo/statusstore_test.go diff --git a/pkg/lib/cpu/quotaturbo/driverevent_test.go b/pkg/lib/cpu/quotaturbo/driverevent_test.go new file mode 100644 index 0000000..926b457 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/driverevent_test.go @@ -0,0 +1,604 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-20 +// Description: This file is used for testing driverevent.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "math" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef/cgroup" +) + +// TestEventDriverElevate tests elevate of EventDriver +func TestEventDriverElevate(t *testing.T) { + var elevateTests = []struct { + status *StatusStore + judgements func(t *testing.T, status *StatusStore) + name string + }{ + { + name: "TC1 - CPU usage >= the alarmWaterMark.", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 60, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon1": {}, + }, + cpuUtils: []cpuUtil{ + { + util: 90, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + var delta float64 = 0 + conID := "testCon1" + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC2 - the container is not suppressed.", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 70, + ElevateLimit: defaultElevateLimit, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon2": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod2/testCon2", + }, + // currently not suppressed + curThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 10, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 10, + }, + period: 100000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 60, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + var delta float64 = 0 + conID := "testCon2" + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC3 - increase the quota of the suppressed container", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 60, + ElevateLimit: defaultElevateLimit, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon3": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod3/testCon3", + }, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 50, + ThrottledTime: 200000, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 40, + ThrottledTime: 100000, + }, + period: 100000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 40, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon3" + c := status.cpuQuotas[conID] + coefficient := math.Min(float64(0.0001), + util.PercentageToDecimal(status.ElevateLimit)*float64(runtime.NumCPU())) / + float64(0.0001) + delta := coefficient * float64(0.0001) * float64(c.period) + assert.True(t, status.cpuQuotas[conID].quotaDelta == delta) + }, + }, + } + + e := &EventDriver{} + for _, tt := range elevateTests { + t.Run(tt.name, func(t *testing.T) { + e.elevate(tt.status) + tt.judgements(t, tt.status) + }) + } +} + +// TestSlowFallback tests slowFallback of EventDriver +func TestSlowFallback(t *testing.T) { + var slowFallBackTests = []struct { + status *StatusStore + judgements func(t *testing.T, status *StatusStore) + name string + }{ + { + name: "TC1-CPU usage <= the highWaterMark.", + status: &StatusStore{ + Config: &Config{ + HighWaterMark: 60, + SlowFallbackRatio: defaultSlowFallbackRatio, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon4": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod4/testCon4", + }, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 40, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon4" + var delta float64 = 0 + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC2-the container is suppressed.", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 50, + SlowFallbackRatio: defaultSlowFallbackRatio, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon5": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod5/testCon5", + }, + cpuLimit: 1, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 0, + }, + period: 100000, + curQuota: 200000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 70, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + var delta float64 = 0 + conID := "testCon5" + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC3-decrease the quota of the uncompressed containers", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 90, + HighWaterMark: 40, + SlowFallbackRatio: defaultSlowFallbackRatio, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon6": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod6/testCon6", + }, + cpuLimit: 2, + // currently not suppressed + curThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + ThrottledTime: 100000, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 10, + ThrottledTime: 100000, + }, + period: 100000, + curQuota: 400000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 60.0, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon6" + c := status.cpuQuotas[conID] + coefficient := (status.getLastCPUUtil() - float64(status.HighWaterMark)) / + float64(status.AlarmWaterMark-status.HighWaterMark) * status.SlowFallbackRatio + delta := coefficient * + ((float64(c.cpuLimit) * float64(c.period)) - float64(c.curQuota)) + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range slowFallBackTests { + t.Run(tt.name, func(t *testing.T) { + e.slowFallback(tt.status) + tt.judgements(t, tt.status) + }) + } +} + +// TestFastFallback tests fastFallback of EventDriver +func TestFastFallback(t *testing.T) { + var fastFallBackTests = []struct { + status *StatusStore + judgements func(t *testing.T, status *StatusStore) + name string + }{ + { + name: "TC1-CPU usage <= the AlarmWaterMark.", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 30, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon7": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod7/testCon7", + }, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 10, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon7" + var delta float64 = 0 + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC2-the quota of container is not increased.", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 30, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon8": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod8/testCon8", + }, + cpuLimit: 1, + period: 100, + curQuota: 100, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 48, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + var delta float64 = 0 + conID := "testCon8" + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC3-decrease the quota of the containers", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 65, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon9": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod9/testCon9", + }, + cpuLimit: 3, + period: 10000, + curQuota: 40000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 90, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon9" + c := status.cpuQuotas[conID] + delta := util.PercentageToDecimal(float64(status.AlarmWaterMark)-status.getLastCPUUtil()) * + float64(runtime.NumCPU()) * float64(c.period) + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range fastFallBackTests { + t.Run(tt.name, func(t *testing.T) { + e.fastFallback(tt.status) + tt.judgements(t, tt.status) + }) + } +} + +// TestSharpFluctuates tests sharpFluctuates +func TestSharpFluctuates(t *testing.T) { + const cpuUtil90 = 90 + var sharpFluctuatesTests = []struct { + status *StatusStore + want bool + name string + }{ + { + name: "TC1-the cpu changes rapidly", + status: &StatusStore{ + Config: &Config{ + CPUFloatingLimit: defaultCPUFloatingLimit, + }, + cpuUtils: []cpuUtil{ + { + util: cpuUtil90, + }, + { + util: cpuUtil90 - defaultCPUFloatingLimit - 1, + }, + }, + }, + want: true, + }, + { + name: "TC2-the cpu changes steadily", + status: &StatusStore{ + Config: &Config{ + CPUFloatingLimit: defaultCPUFloatingLimit, + }, + cpuUtils: []cpuUtil{ + { + util: cpuUtil90, + }, + { + util: cpuUtil90 - defaultCPUFloatingLimit + 1, + }, + }, + }, + want: false, + }, + } + for _, tt := range sharpFluctuatesTests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, sharpFluctuates(tt.status) == tt.want) + }) + } +} + +// TestEventDriverAdjustQuota tests adjustQuota of EventDriver +func TestEventDriverAdjustQuota(t *testing.T) { + var eDriverAdjustQuotaTests = []struct { + status *StatusStore + judgements func(t *testing.T, status *StatusStore) + name string + }{ + { + name: "TC1-no promotion", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 80, + HighWaterMark: 73, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon10": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod10/testCon10", + }, + cpuLimit: 1, + period: 80, + curQuota: 100, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 1, + }, + { + util: -defaultCPUFloatingLimit, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + var delta float64 = 0 + conID := "testCon10" + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + { + name: "TC2-make a promotion", + status: &StatusStore{ + Config: &Config{ + AlarmWaterMark: 97, + HighWaterMark: 73, + }, + cpuQuotas: map[string]*CPUQuota{ + "testCon11": { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod11/testCon11", + }, + cpuLimit: 2, + curThrottle: &cgroup.CPUStat{ + NrThrottled: 1, + ThrottledTime: 200, + }, + preThrottle: &cgroup.CPUStat{ + NrThrottled: 0, + ThrottledTime: 100, + }, + period: 2000, + curQuota: 5000, + }, + }, + cpuUtils: []cpuUtil{ + { + util: 10, + }, + }, + }, + judgements: func(t *testing.T, status *StatusStore) { + conID := "testCon11" + c := status.cpuQuotas[conID] + coefficient := math.Min(float64(0.00005), util.PercentageToDecimal(status.ElevateLimit)* + float64(runtime.NumCPU())) / float64(0.00005) + delta := coefficient * float64(0.00005) * float64(c.period) + assert.Equal(t, delta, status.cpuQuotas[conID].quotaDelta) + }, + }, + } + e := &EventDriver{} + for _, tt := range eDriverAdjustQuotaTests { + t.Run(tt.name, func(t *testing.T) { + e.adjustQuota(tt.status) + tt.judgements(t, tt.status) + }) + } +} + +// TestGetMaxQuota tests getMaxQuota +func TestGetMaxQuota(t *testing.T) { + var getMaxQuotaTests = []struct { + cq *CPUQuota + judgements func(t *testing.T, cq *CPUQuota) + name string + }{ + { + name: "TC1-empty cpu usage", + cq: &CPUQuota{ + heightLimit: 100, + cpuUsages: []cpuUsage{}, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 100 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC2-The remaining value is less than 3 times the upper limit.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {100000, 100000}, + {200000, 200000}, + }, + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod1/testCon1", + }, + cpuLimit: 4, + period: 100, + heightLimit: 800, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + const res = 400 + float64(400*700)/float64(3*800) + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC3-The remaining value is greater than 3 times the limit height.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {10000, 0}, + {20000, 0}, + {30000, 0}, + {40000, 0}, + {50000, 0}, + {60000, 0}, + {70000, 0}, + {80000, 100}, + }, + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod1/testCon1", + }, + cpuLimit: 1, + period: 100, + heightLimit: 200, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 200 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + { + name: "TC4-The remaining value is less than the initial value.", + cq: &CPUQuota{ + cpuUsages: []cpuUsage{ + {100, 0}, + {200, 1000000}, + }, + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.DefaultCgroupRoot, + Path: "kubepods/testPod1/testCon1", + }, + cpuLimit: 10, + period: 10, + heightLimit: 150, + }, + judgements: func(t *testing.T, cq *CPUQuota) { + var res float64 = 100 + assert.Equal(t, res, getMaxQuota(cq)) + }, + }, + } + for _, tt := range getMaxQuotaTests { + t.Run(tt.name, func(t *testing.T) { + tt.judgements(t, tt.cq) + }) + } +} diff --git a/pkg/lib/cpu/quotaturbo/statusstore_test.go b/pkg/lib/cpu/quotaturbo/statusstore_test.go new file mode 100644 index 0000000..a69fac4 --- /dev/null +++ b/pkg/lib/cpu/quotaturbo/statusstore_test.go @@ -0,0 +1,568 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-02-16 +// Description: This file is used for testing statusstore.go + +// Package quotaturbo is for Quota Turbo feature +package quotaturbo + +import ( + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/test/try" +) + +// TestIsAdjustmentAllowed tests isAdjustmentAllowed +func TestIsAdjustmentAllowed(t *testing.T) { + const contPath1 = "kubepods/testPod1/testCon1" + + try.RemoveAll(constant.TmpTestDir) + defer try.RemoveAll(constant.TmpTestDir) + + tests := []struct { + h *cgroup.Hierarchy + cpuLimit float64 + pre func() + post func() + name string + want bool + }{ + { + name: "TC1-allow adjustment", + h: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath1, + }, + cpuLimit: float64(runtime.NumCPU()) - 1, + pre: func() { + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1, "cpu.cfs_quota_us"), + constant.DefaultFileMode) + }, + post: func() { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1)) + }, + want: true, + }, + { + name: "TC2-cgroup path is not existed", + h: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath1, + }, + cpuLimit: float64(runtime.NumCPU()) - 1, + pre: func() { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1)) + }, + want: false, + }, + { + name: "TC3-cpulimit = 0", + h: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath1, + }, + cpuLimit: 0, + pre: func() { + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1, "cpu.cfs_quota_us"), + constant.DefaultFileMode) + }, + post: func() { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1)) + }, + want: false, + }, + { + name: "TC4-cpulimit over max", + h: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath1, + }, + cpuLimit: float64(runtime.NumCPU()) + 1, + pre: func() { + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1, "cpu.cfs_quota_us"), + constant.DefaultFileMode) + }, + post: func() { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1)) + }, + want: false, + }, + { + name: "TC5-cpurequest over max", + h: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath1, + }, + cpuLimit: 0, + pre: func() { + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1, "cpu.cfs_quota_us"), + constant.DefaultFileMode) + }, + post: func() { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath1)) + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre() + } + assert.Equal(t, isAdjustmentAllowed(tt.h, tt.cpuLimit), tt.want) + if tt.post != nil { + tt.post() + } + }) + } +} + +// TestStatusStore_RemoveCgroup tests RemoveCgroup of StatusStore +func TestStatusStore_RemoveCgroup(t *testing.T) { + const ( + podPath = "kubepods/testPod1" + contPath = "kubepods/testPod1/testCon1" + ) + type fields struct { + Config *Config + cpuQuotas map[string]*CPUQuota + } + type args struct { + cgroupPath string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + pre func() + post func(t *testing.T, d *StatusStore) + }{ + { + name: "TC1-empty cgroupPath", + args: args{ + cgroupPath: "", + }, + fields: fields{ + cpuQuotas: make(map[string]*CPUQuota), + }, + wantErr: false, + }, + { + name: "TC2-cgroupPath is not existed", + args: args{ + cgroupPath: contPath, + }, + fields: fields{ + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "TC3-cgroupPath existed but can not set", + args: args{ + cgroupPath: contPath, + }, + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + curQuota: 100000, + nextQuota: 200000, + }, + }, + }, + pre: func() { + try.MkdirAll(filepath.Join(constant.TmpTestDir, "cpu", contPath), constant.DefaultDirMode) + }, + post: func(t *testing.T, d *StatusStore) { + try.RemoveAll(filepath.Join(constant.TmpTestDir, "cpu", contPath)) + }, + wantErr: true, + }, + { + name: "TC4-remove cgroupPath successfully", + args: args{ + cgroupPath: contPath, + }, + fields: fields{ + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + cpuLimit: 2, + period: 100000, + curQuota: 250000, + nextQuota: 240000, + }, + }, + }, + pre: func() { + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, "cpu.cfs_quota_us"), "250000") + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", podPath, "cpu.cfs_quota_us"), "-1") + }, + post: func(t *testing.T, d *StatusStore) { + val := strings.TrimSpace(try.ReadFile( + filepath.Join(constant.TmpTestDir, "cpu", contPath, "cpu.cfs_quota_us")).String()) + assert.Equal(t, "200000", val) + val = strings.TrimSpace(try.ReadFile( + filepath.Join(constant.TmpTestDir, "cpu", podPath, "cpu.cfs_quota_us")).String()) + assert.Equal(t, "-1", val) + assert.Equal(t, 0, len(d.cpuQuotas)) + try.RemoveAll(constant.TmpTestDir) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &StatusStore{ + Config: tt.fields.Config, + cpuQuotas: tt.fields.cpuQuotas, + } + if tt.pre != nil { + tt.pre() + } + if err := d.RemoveCgroup(tt.args.cgroupPath); (err != nil) != tt.wantErr { + t.Errorf("StatusStore.RemoveCgroup() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post(t, d) + } + }) + } +} + +// TestStatusStore_AddCgroup tests AddCgroup of StatusStore +func TestStatusStore_AddCgroup(t *testing.T) { + const ( + contPath = "kubepods/testPod1/testCon1" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + stat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + quota = "200000" + period = "100000" + usage = "1234567" + ) + type fields struct { + Config *Config + cpuQuotas map[string]*CPUQuota + } + type args struct { + cgroupPath string + cpuLimit float64 + cpuRequest float64 + } + tests := []struct { + name string + fields fields + args args + wantErr bool + pre func(t *testing.T, d *StatusStore) + post func(t *testing.T, d *StatusStore) + }{ + { + name: "TC1-empty cgroup path", + args: args{ + cgroupPath: "", + }, + fields: fields{ + cpuQuotas: make(map[string]*CPUQuota), + }, + wantErr: true, + }, + { + name: "TC2-empty cgroup mount point", + args: args{ + cgroupPath: contPath, + }, + fields: fields{ + Config: &Config{ + CgroupRoot: "", + }, + }, + wantErr: true, + }, + { + name: "TC3-cgroup not allow to adjust", + args: args{ + cgroupPath: contPath, + cpuLimit: 3, + }, + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + }, + wantErr: true, + }, + { + name: "TC4-failed to create CPUQuota", + args: args{ + cgroupPath: contPath, + cpuLimit: 3, + }, + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + cpuQuotas: make(map[string]*CPUQuota), + }, + pre: func(t *testing.T, d *StatusStore) { + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuPeriodFile), period) + }, + post: func(t *testing.T, d *StatusStore) { + try.RemoveAll(constant.TmpTestDir) + }, + wantErr: true, + }, + { + name: "TC5-add successfully", + args: args{ + cgroupPath: contPath, + cpuLimit: 2, + }, + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + cpuQuotas: make(map[string]*CPUQuota), + }, + pre: func(t *testing.T, d *StatusStore) { + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuPeriodFile), period) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuQuotaFile), quota) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpuacct", contPath, cpuUsageFile), usage) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuStatFile), stat) + }, + post: func(t *testing.T, d *StatusStore) { + assert.Equal(t, 1, len(d.cpuQuotas)) + try.RemoveAll(constant.TmpTestDir) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &StatusStore{ + Config: tt.fields.Config, + cpuQuotas: tt.fields.cpuQuotas, + } + if tt.pre != nil { + tt.pre(t, d) + } + if err := d.AddCgroup(tt.args.cgroupPath, tt.args.cpuLimit); (err != nil) != tt.wantErr { + t.Errorf("StatusStore.AddCgroup() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post(t, d) + } + }) + } +} + +// TestStatusStoreGetLastCPUUtil tests getLastCPUUtil of StatusStore +func TestStatusStore_getLastCPUUtil(t *testing.T) { + // 1. empty CPU Utils + d := &StatusStore{} + t.Run("TC1-empty CPU Util", func(t *testing.T) { + util := float64(0.0) + assert.Equal(t, util, d.getLastCPUUtil()) + }) + // 2. CPU Utils + cpuUtil20 := 20 + d = &StatusStore{cpuUtils: []cpuUtil{{ + util: float64(cpuUtil20), + }}} + t.Run("TC2-CPU Util is 20", func(t *testing.T) { + util := float64(20.0) + assert.Equal(t, util, d.getLastCPUUtil()) + }) +} + +// TestQuotaTurboUpdateCPUUtils tests updateCPUUtils of QuotaTurbo and NewProcStat +func TestStatusStore_updateCPUUtils(t *testing.T) { + status := NewStatusStore() + // 1. obtain the cpu usage for the first time + if err := status.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + num1 := 1 + assert.Equal(t, num1, len(status.cpuUtils)) + // 2. obtain the cpu usage for the second time + if err := status.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + num2 := 2 + assert.Equal(t, num2, len(status.cpuUtils)) + // 3. obtain the cpu usage after 1 minute + var minuteTimeDelta int64 = 60000000001 + status.cpuUtils[0].timestamp -= minuteTimeDelta + if err := status.updateCPUUtils(); err != nil { + assert.NoError(t, err) + } + assert.Equal(t, num2, len(status.cpuUtils)) +} + +// TestStatusStore_updateCPUQuotas tests updateCPUQuotas of StatusStore +func TestStatusStore_updateCPUQuotas(t *testing.T) { + const ( + contPath = "kubepods/testPod1/testCon1" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + stat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + quota = "200000" + period = "100000" + usage = "1234567" + ) + type fields struct { + Config *Config + cpuQuotas map[string]*CPUQuota + cpuUtils []cpuUtil + } + tests := []struct { + name string + fields fields + wantErr bool + pre func() + post func() + }{ + { + name: "TC1-fail to get CPUQuota", + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + }, + }, + cpuUtils: make([]cpuUtil, 0), + }, + wantErr: true, + }, + { + name: "TC2-update successfully", + fields: fields{ + Config: &Config{ + CgroupRoot: constant.TmpTestDir, + }, + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + }, + }, + cpuUtils: make([]cpuUtil, 0), + }, + pre: func() { + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuPeriodFile), period) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuQuotaFile), quota) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpuacct", contPath, cpuUsageFile), usage) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuStatFile), stat) + }, + post: func() { + try.RemoveAll(constant.TmpTestDir) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &StatusStore{ + Config: tt.fields.Config, + cpuQuotas: tt.fields.cpuQuotas, + cpuUtils: tt.fields.cpuUtils, + } + if tt.pre != nil { + tt.pre() + } + if err := d.updateCPUQuotas(); (err != nil) != tt.wantErr { + t.Errorf("StatusStore.update() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post() + } + }) + } +} + +// TestStatusStore_writeQuota tests writeQuota of StatusStore +func TestStatusStore_writeQuota(t *testing.T) { + const contPath = "kubepods/testPod1/testCon1" + tests := []struct { + name string + cpuQuotas map[string]*CPUQuota + wantErr bool + }{ + { + name: "TC1-empty cgroup path", + cpuQuotas: map[string]*CPUQuota{ + contPath: { + Hierarchy: &cgroup.Hierarchy{ + MountPoint: constant.TmpTestDir, + Path: contPath, + }, + curQuota: 100000, + nextQuota: 200000, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &StatusStore{ + cpuQuotas: tt.cpuQuotas, + } + if err := d.writeQuota(); (err != nil) != tt.wantErr { + t.Errorf("StatusStore.writeQuota() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} -- Gitee From 4293f6321ad5abbed772cb9a229593730fc10f37 Mon Sep 17 00:00:00 2001 From: hanchao Date: Fri, 10 Mar 2023 17:09:09 +0800 Subject: [PATCH 53/73] refactor: renamed features and related configuration names --- pkg/config/config.go | 11 ++-- pkg/rubik/import.go | 6 +- .../cachelimit.go => dynCache/dynCache.go} | 18 +++--- .../dynCache_init.go} | 6 +- .../dynCache_init_test.go} | 36 +++++------ .../dynCache_test.go} | 16 ++--- .../{cachelimit => dynCache}/dynamic.go | 12 ++-- .../{cachelimit => dynCache}/dynamic_test.go | 28 ++++----- pkg/services/{cachelimit => dynCache}/sync.go | 10 +-- .../{cachelimit => dynCache}/sync_test.go | 22 +++---- pkg/services/{blkcg => iocost}/iocost.go | 20 +++--- .../{blkcg => iocost}/iocost_origin.go | 2 +- pkg/services/{blkcg => iocost}/utils.go | 2 +- .../blkiothrottle.go => iolimit/iolimit.go} | 28 ++++----- .../{qos/qos.go => preemption/preemption.go} | 62 +++++++++---------- .../preemption_test.go} | 20 +++--- 16 files changed, 147 insertions(+), 152 deletions(-) rename pkg/services/{cachelimit/cachelimit.go => dynCache/dynCache.go} (94%) rename pkg/services/{cachelimit/cachelimit_init.go => dynCache/dynCache_init.go} (97%) rename pkg/services/{cachelimit/cachelimit_init_test.go => dynCache/dynCache_init_test.go} (88%) rename pkg/services/{cachelimit/cachelimit_test.go => dynCache/dynCache_test.go} (97%) rename pkg/services/{cachelimit => dynCache}/dynamic.go (92%) rename pkg/services/{cachelimit => dynCache}/dynamic_test.go (89%) rename pkg/services/{cachelimit => dynCache}/sync.go (92%) rename pkg/services/{cachelimit => dynCache}/sync_test.go (91%) rename pkg/services/{blkcg => iocost}/iocost.go (95%) rename pkg/services/{blkcg => iocost}/iocost_origin.go (99%) rename pkg/services/{blkcg => iocost}/utils.go (98%) rename pkg/services/{blkcg/blkiothrottle.go => iolimit/iolimit.go} (53%) rename pkg/services/{qos/qos.go => preemption/preemption.go} (67%) rename pkg/services/{qos/qos_test.go => preemption/preemption_test.go} (93%) diff --git a/pkg/config/config.go b/pkg/config/config.go index c828f12..4d4b3de 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,11 +38,12 @@ type Config struct { // AgentConfig is the configuration of rubik, including important basic configurations such as logs type AgentConfig struct { - LogDriver string `json:"logDriver,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - LogSize int64 `json:"logSize,omitempty"` - LogDir string `json:"logDir,omitempty"` - CgroupRoot string `json:"cgroupRoot,omitempty"` + LogDriver string `json:"logDriver,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + LogSize int64 `json:"logSize,omitempty"` + LogDir string `json:"logDir,omitempty"` + CgroupRoot string `json:"cgroupRoot,omitempty"` + EnabledFeatures []string `json:"enabledFeatures,omitempty"` } // NewConfig returns an config object pointer diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index 31f4dbc..a35ada8 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -16,9 +16,9 @@ package rubik import ( // introduce packages to auto register service - _ "isula.org/rubik/pkg/services/blkcg" - _ "isula.org/rubik/pkg/services/cachelimit" - _ "isula.org/rubik/pkg/services/qos" + _ "isula.org/rubik/pkg/services/dynCache" + _ "isula.org/rubik/pkg/services/iocost" + _ "isula.org/rubik/pkg/services/preemption" _ "isula.org/rubik/pkg/services/quotaburst" _ "isula.org/rubik/pkg/services/quotaturbo" _ "isula.org/rubik/pkg/version" diff --git a/pkg/services/cachelimit/cachelimit.go b/pkg/services/dynCache/dynCache.go similarity index 94% rename from pkg/services/cachelimit/cachelimit.go rename to pkg/services/dynCache/dynCache.go index afe68a0..5dbf63b 100644 --- a/pkg/services/cachelimit/cachelimit.go +++ b/pkg/services/dynCache/dynCache.go @@ -12,7 +12,7 @@ // Description: This file is cache limit service // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "context" @@ -86,8 +86,8 @@ type Config struct { MemBandPercent MultiLvlPercent `json:"memBandPercent,omitempty"` } -// CacheLimit is cache limit service structure -type CacheLimit struct { +// DynCache is cache limit service structure +type DynCache struct { *Config Attr *Attr Viewer api.Viewer @@ -119,8 +119,8 @@ func init() { } // NewCacheLimit return cache limit instance with default settings -func NewCacheLimit() *CacheLimit { - return &CacheLimit{ +func NewCacheLimit() *DynCache { + return &DynCache{ Name: moduleName, Attr: &Attr{ NumaNodeDir: defaultNumaNodeDir, @@ -148,7 +148,7 @@ func NewCacheLimit() *CacheLimit { } // PreStart will do some pre-setting actions -func (c *CacheLimit) PreStart(viewer api.Viewer) error { +func (c *DynCache) PreStart(viewer api.Viewer) error { c.Viewer = viewer if err := c.InitCacheLimitDir(); err != nil { @@ -158,18 +158,18 @@ func (c *CacheLimit) PreStart(viewer api.Viewer) error { } // ID returns service's name -func (c *CacheLimit) ID() string { +func (c *DynCache) ID() string { return c.Name } // Run implement service run function -func (c *CacheLimit) Run(ctx context.Context) { +func (c *DynCache) Run(ctx context.Context) { go wait.Until(c.SyncCacheLimit, time.Second, ctx.Done()) wait.Until(c.StartDynamic, time.Millisecond*time.Duration(c.Config.AdjustInterval), ctx.Done()) } // Validate validate service's config -func (c *CacheLimit) Validate() error { +func (c *DynCache) Validate() error { defaultLimitMode := c.Config.DefaultLimitMode if defaultLimitMode != modeStatic && defaultLimitMode != modeDynamic { return fmt.Errorf("invalid cache limit mode: %s, should be %s or %s", diff --git a/pkg/services/cachelimit/cachelimit_init.go b/pkg/services/dynCache/dynCache_init.go similarity index 97% rename from pkg/services/cachelimit/cachelimit_init.go rename to pkg/services/dynCache/dynCache_init.go index ea766f2..198be96 100644 --- a/pkg/services/cachelimit/cachelimit_init.go +++ b/pkg/services/dynCache/dynCache_init.go @@ -12,7 +12,7 @@ // Description: This file will init cache limit directories before services running // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "fmt" @@ -39,7 +39,7 @@ type limitSet struct { } // InitCacheLimitDir init multi-level cache limit directories -func (c *CacheLimit) InitCacheLimitDir() error { +func (c *DynCache) InitCacheLimitDir() error { log.Debugf("init cache limit directory") const ( defaultL3PercentMax = 100 @@ -80,7 +80,7 @@ func (c *CacheLimit) InitCacheLimitDir() error { return nil } -func (c *CacheLimit) newCacheLimitSet(level string, l3Per, mbPer int) *limitSet { +func (c *DynCache) newCacheLimitSet(level string, l3Per, mbPer int) *limitSet { return &limitSet{ level: level, l3Percent: l3Per, diff --git a/pkg/services/cachelimit/cachelimit_init_test.go b/pkg/services/dynCache/dynCache_init_test.go similarity index 88% rename from pkg/services/cachelimit/cachelimit_init_test.go rename to pkg/services/dynCache/dynCache_init_test.go index fbdede5..fe07a01 100644 --- a/pkg/services/cachelimit/cachelimit_init_test.go +++ b/pkg/services/dynCache/dynCache_init_test.go @@ -12,7 +12,7 @@ // Description: This file will init cache limit directories before services running // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "fmt" @@ -54,8 +54,8 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { name string fields fields wantErr bool - preHook func(t *testing.T, c *CacheLimit) - postHook func(t *testing.T, c *CacheLimit) + preHook func(t *testing.T, c *DynCache) + postHook func(t *testing.T, c *DynCache) }{ { name: "TC1-normal cache limit dir setting", @@ -63,7 +63,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Config.DefaultResctrlDir = try.GenTestDir().String() c.Config.DefaultLimitMode = modeStatic setMaskFile(t, c.Config.DefaultResctrlDir, "7fff") @@ -71,7 +71,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { resctrlLevelMap := map[string]string{ "rubik_max": "L3:0=7fff;1=7fff;2=7fff;3=7fff\nMB:0=100;1=100;2=100;3=100\n", "rubik_high": "L3:0=7f;1=7f;2=7f;3=7f\nMB:0=50;1=50;2=50;3=50\n", @@ -95,7 +95,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { pidNameSpaceDir := try.GenTestDir().String() pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") @@ -103,7 +103,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { os.Symlink(pidNameSpaceFileOri.String(), pidNameSpace) c.Config.DefaultPidNameSpace = pidNameSpace }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) }, }, @@ -114,7 +114,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { pidNameSpaceDir := try.GenTestDir().String() pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") @@ -122,7 +122,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { os.Link(pidNameSpaceFileOri.String(), pidNameSpace) c.Config.DefaultPidNameSpace = pidNameSpace }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) }, }, @@ -133,7 +133,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Config.DefaultResctrlDir = "/resctrl/path/is/not/exist" }, }, @@ -144,10 +144,10 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Config.DefaultResctrlDir = try.GenTestDir().String() }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { try.RemoveAll(c.DefaultResctrlDir) }, }, @@ -158,7 +158,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Attr.NumaNodeDir = "/numa/node/path/is/not/exist" }, }, @@ -169,7 +169,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Config.DefaultResctrlDir = try.GenTestDir().String() c.Config.DefaultLimitMode = modeStatic setMaskFile(t, c.Config.DefaultResctrlDir, "") @@ -177,7 +177,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 0) }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { try.RemoveAll(c.Config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, @@ -188,7 +188,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Config: genDefaultConfig(), Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit) { + preHook: func(t *testing.T, c *DynCache) { c.Config.DefaultResctrlDir = try.GenTestDir().String() c.Config.DefaultLimitMode = modeStatic setMaskFile(t, c.Config.DefaultResctrlDir, "1") @@ -196,7 +196,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 0) }, - postHook: func(t *testing.T, c *CacheLimit) { + postHook: func(t *testing.T, c *DynCache) { try.RemoveAll(c.Config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, @@ -204,7 +204,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, diff --git a/pkg/services/cachelimit/cachelimit_test.go b/pkg/services/dynCache/dynCache_test.go similarity index 97% rename from pkg/services/cachelimit/cachelimit_test.go rename to pkg/services/dynCache/dynCache_test.go index 35e6783..3a43bd9 100644 --- a/pkg/services/cachelimit/cachelimit_test.go +++ b/pkg/services/dynCache/dynCache_test.go @@ -12,7 +12,7 @@ // Description: This file is testcase for cache limit service // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "reflect" @@ -173,7 +173,7 @@ func TestCacheLimit_Validate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, @@ -192,11 +192,11 @@ func TestCacheLimit_Validate(t *testing.T) { func TestNewCacheLimit(t *testing.T) { tests := []struct { name string - want *CacheLimit + want *DynCache }{ { name: "TC-do nothing", - want: &CacheLimit{ + want: &DynCache{ Name: moduleName, Attr: &Attr{ NumaNodeDir: defaultNumaNodeDir, @@ -247,8 +247,8 @@ func TestCacheLimit_PreStart(t *testing.T) { fields fields args args wantErr bool - preHook func(t *testing.T, c *CacheLimit) - postHook func(t *testing.T, c *CacheLimit) + preHook func(t *testing.T, c *DynCache) + postHook func(t *testing.T, c *DynCache) }{ { name: "TC-just call function", @@ -261,7 +261,7 @@ func TestCacheLimit_PreStart(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, @@ -296,7 +296,7 @@ func TestCacheLimit_ID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, diff --git a/pkg/services/cachelimit/dynamic.go b/pkg/services/dynCache/dynamic.go similarity index 92% rename from pkg/services/cachelimit/dynamic.go rename to pkg/services/dynCache/dynamic.go index 6a3b771..147efde 100644 --- a/pkg/services/cachelimit/dynamic.go +++ b/pkg/services/dynCache/dynamic.go @@ -12,7 +12,7 @@ // Description: This file is used for dynamic cache limit level setting // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "fmt" @@ -28,7 +28,7 @@ import ( // StartDynamic will continuously run to detect online pod cache miss and // limit offline pod cache usage -func (c *CacheLimit) StartDynamic() { +func (c *DynCache) StartDynamic() { if !c.dynamicExist() { return } @@ -75,7 +75,7 @@ func getPodCacheMiss(pod *typedef.PodInfo, perfDu int) (int, int) { int(100.0 * float64(stat.LLCMiss) / (1.0 + float64(stat.LLCAccess))) } -func (c *CacheLimit) dynamicExist() bool { +func (c *DynCache) dynamicExist() bool { for _, pod := range c.listOfflinePods() { if err := c.syncLevel(pod); err != nil { continue @@ -87,7 +87,7 @@ func (c *CacheLimit) dynamicExist() bool { return false } -func (c *CacheLimit) flush(limitSet *limitSet, step int) error { +func (c *DynCache) flush(limitSet *limitSet, step int) error { var nextPercent = func(value, min, max, step int) int { value += step if value < min { @@ -109,7 +109,7 @@ func (c *CacheLimit) flush(limitSet *limitSet, step int) error { return c.doFlush(limitSet) } -func (c *CacheLimit) doFlush(limitSet *limitSet) error { +func (c *DynCache) doFlush(limitSet *limitSet) error { if err := limitSet.writeResctrlSchemata(c.Attr.NumaNum); err != nil { return fmt.Errorf("adjust dynamic cache limit to l3:%v mb:%v error: %v", limitSet.l3Percent, limitSet.mbPercent, err) @@ -120,7 +120,7 @@ func (c *CacheLimit) doFlush(limitSet *limitSet) error { return nil } -func (c *CacheLimit) listOnlinePods() map[string]*typedef.PodInfo { +func (c *DynCache) listOnlinePods() map[string]*typedef.PodInfo { onlineValue := "false" return c.Viewer.ListPodsWithOptions(func(pi *typedef.PodInfo) bool { return pi.Annotations[constant.PriorityAnnotationKey] == onlineValue diff --git a/pkg/services/cachelimit/dynamic_test.go b/pkg/services/dynCache/dynamic_test.go similarity index 89% rename from pkg/services/cachelimit/dynamic_test.go rename to pkg/services/dynCache/dynamic_test.go index 2f1dde2..5bdca11 100644 --- a/pkg/services/cachelimit/dynamic_test.go +++ b/pkg/services/dynCache/dynamic_test.go @@ -12,7 +12,7 @@ // Description: This file is testcases for dynamic cache limit level setting // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "fmt" @@ -40,8 +40,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { tests := []struct { name string fields fields - preHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) - postHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) + preHook func(t *testing.T, c *DynCache, fakePods []*try.FakePod) + postHook func(t *testing.T, c *DynCache, fakePods []*try.FakePod) }{ { name: "TC-normal dynamic setting", @@ -60,7 +60,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { MinMiss: 10, }, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { resctrlDir := try.GenTestDir().String() setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() @@ -77,7 +77,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { manager := genPodManager(fakePods) c.Viewer = manager }, - postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + postHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { for _, pod := range fakePods { pod.CleanPath() } @@ -102,7 +102,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { MinMiss: 0, }, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { resctrlDir := try.GenTestDir().String() setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() @@ -119,7 +119,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { manager := genPodManager(fakePods) c.Viewer = manager }, - postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + postHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { for _, pod := range fakePods { pod.CleanPath() } @@ -144,7 +144,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { MinMiss: math.MaxInt64, }, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { resctrlDir := try.GenTestDir().String() setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() @@ -161,7 +161,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { manager := genPodManager(fakePods) c.Viewer = manager }, - postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + postHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { for _, pod := range fakePods { pod.CleanPath() } @@ -186,7 +186,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { MinMiss: 0, }, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { resctrlDir := try.GenTestDir().String() setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() @@ -203,7 +203,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { manager := genPodManager(fakePods) c.Viewer = manager }, - postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + postHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { for _, pod := range fakePods { pod.CleanPath() } @@ -228,7 +228,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { MinMiss: 10, }, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { resctrlDir := try.GenTestDir().String() setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() @@ -245,7 +245,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { manager := genPodManager(fakePods) c.Viewer = manager }, - postHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + postHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { for _, pod := range fakePods { pod.CleanPath() } @@ -256,7 +256,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, diff --git a/pkg/services/cachelimit/sync.go b/pkg/services/dynCache/sync.go similarity index 92% rename from pkg/services/cachelimit/sync.go rename to pkg/services/dynCache/sync.go index 18f8679..46262c5 100644 --- a/pkg/services/cachelimit/sync.go +++ b/pkg/services/dynCache/sync.go @@ -12,7 +12,7 @@ // Description: This file is used for cache limit sync setting // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "fmt" @@ -46,7 +46,7 @@ var validLevel = map[string]bool{ } // SyncCacheLimit will continuously set cache limit with corresponding offline pods -func (c *CacheLimit) SyncCacheLimit() { +func (c *DynCache) SyncCacheLimit() { for _, p := range c.listOfflinePods() { if err := c.syncLevel(p); err != nil { log.Errorf("sync cache limit level err: %v", err) @@ -60,7 +60,7 @@ func (c *CacheLimit) SyncCacheLimit() { } // writeTasksToResctrl will write tasks running in containers into resctrl group -func (c *CacheLimit) writeTasksToResctrl(pod *typedef.PodInfo) error { +func (c *DynCache) writeTasksToResctrl(pod *typedef.PodInfo) error { if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", pod.CgroupPath, "")) { // just return since pod maybe deleted return nil @@ -93,7 +93,7 @@ func (c *CacheLimit) writeTasksToResctrl(pod *typedef.PodInfo) error { } // syncLevel sync cache limit level -func (c *CacheLimit) syncLevel(pod *typedef.PodInfo) error { +func (c *DynCache) syncLevel(pod *typedef.PodInfo) error { if pod.Annotations[constant.CacheLimitAnnotationKey] == "" { if c.Config.DefaultLimitMode == modeStatic { pod.Annotations[constant.CacheLimitAnnotationKey] = levelMax @@ -109,7 +109,7 @@ func (c *CacheLimit) syncLevel(pod *typedef.PodInfo) error { return nil } -func (c *CacheLimit) listOfflinePods() map[string]*typedef.PodInfo { +func (c *DynCache) listOfflinePods() map[string]*typedef.PodInfo { offlineValue := "true" return c.Viewer.ListPodsWithOptions(func(pi *typedef.PodInfo) bool { return pi.Annotations[constant.PriorityAnnotationKey] == offlineValue diff --git a/pkg/services/cachelimit/sync_test.go b/pkg/services/dynCache/sync_test.go similarity index 91% rename from pkg/services/cachelimit/sync_test.go rename to pkg/services/dynCache/sync_test.go index 061394c..b6a500a 100644 --- a/pkg/services/cachelimit/sync_test.go +++ b/pkg/services/dynCache/sync_test.go @@ -12,7 +12,7 @@ // Description: This file is testcase for cache limit sync function // Package cachelimit is the service used for cache limit setting -package cachelimit +package dynCache import ( "path/filepath" @@ -71,7 +71,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { tests := []struct { name string fields fields - preHook func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) + preHook func(t *testing.T, c *DynCache, fakePods []*try.FakePod) }{ { name: "TC1-normal case", @@ -84,7 +84,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" @@ -104,7 +104,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") @@ -123,7 +123,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") @@ -143,7 +143,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "invalid" @@ -163,7 +163,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" @@ -183,7 +183,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" @@ -203,7 +203,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" @@ -223,7 +223,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { Config: defaultConfig, Attr: &Attr{}, }, - preHook: func(t *testing.T, c *CacheLimit, fakePods []*try.FakePod) { + preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" @@ -235,7 +235,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &CacheLimit{ + c := &DynCache{ Config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, diff --git a/pkg/services/blkcg/iocost.go b/pkg/services/iocost/iocost.go similarity index 95% rename from pkg/services/blkcg/iocost.go rename to pkg/services/iocost/iocost.go index e45ab7a..951f7a0 100644 --- a/pkg/services/blkcg/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -1,4 +1,4 @@ -package blkcg +package iocost import ( "os" @@ -43,14 +43,13 @@ type IOCostConfig struct { // NodeConfig define the config of node, include iocost type NodeConfig struct { NodeName string `json:"nodeName,omitempty"` - IOCostEnable bool `json:"iocostEnable,omitempty"` IOCostConfig []IOCostConfig `json:"iocostConfig,omitempty"` } // IOCost for iocost class type IOCost struct { - name string - nodeConfig []NodeConfig + name string + nodeConfigs []NodeConfig } var ( @@ -87,7 +86,7 @@ func (b *IOCost) PreStart(viewer api.Viewer) error { func (b *IOCost) loadConfig() error { var nodeConfig *NodeConfig // global will set all node - for _, config := range b.nodeConfig { + for _, config := range b.nodeConfigs { if config.NodeName == nodeName { nodeConfig = &config break @@ -97,20 +96,15 @@ func (b *IOCost) loadConfig() error { } } - // no config, return - if nodeConfig == nil { - log.Warnf("no matching node exist:%v", nodeName) - return nil - } - // ensure that previous configuration is cleared. if err := b.clearIOCost(); err != nil { log.Errorf("clear iocost err:%v", err) return err } - if !nodeConfig.IOCostEnable { - // clear iocost before + // no config, return + if nodeConfig == nil { + log.Warnf("no matching node exist:%v", nodeName) return nil } diff --git a/pkg/services/blkcg/iocost_origin.go b/pkg/services/iocost/iocost_origin.go similarity index 99% rename from pkg/services/blkcg/iocost_origin.go rename to pkg/services/iocost/iocost_origin.go index cd66e11..3597967 100644 --- a/pkg/services/blkcg/iocost_origin.go +++ b/pkg/services/iocost/iocost_origin.go @@ -1,4 +1,4 @@ -package blkcg +package iocost import ( "fmt" diff --git a/pkg/services/blkcg/utils.go b/pkg/services/iocost/utils.go similarity index 98% rename from pkg/services/blkcg/utils.go rename to pkg/services/iocost/utils.go index 9086c19..0b01afc 100644 --- a/pkg/services/blkcg/utils.go +++ b/pkg/services/iocost/utils.go @@ -1,4 +1,4 @@ -package blkcg +package iocost import ( "fmt" diff --git a/pkg/services/blkcg/blkiothrottle.go b/pkg/services/iolimit/iolimit.go similarity index 53% rename from pkg/services/blkcg/blkiothrottle.go rename to pkg/services/iolimit/iolimit.go index 4c23cf2..a2bc8ac 100644 --- a/pkg/services/blkcg/blkiothrottle.go +++ b/pkg/services/iolimit/iolimit.go @@ -13,49 +13,49 @@ type DeviceConfig struct { DeviceValue string `json:"value,omitempty"` } -type BlkioThrottleConfig struct { +type IOLimitAnnoConfig struct { DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` } -type BlkioThrottle struct { +type IOLimit struct { Name string `json:"-"` } func init() { - services.Register("blkio", func() interface{} { + services.Register("ioLimit", func() interface{} { return NewBlkioThrottle() }) } -func NewBlkioThrottle() *BlkioThrottle { - return &BlkioThrottle{Name: "blkiothrottle"} +func NewBlkioThrottle() *IOLimit { + return &IOLimit{Name: "ioLimit"} } -func (b *BlkioThrottle) PreStart(viewer api.Viewer) error { - log.Infof("blkiothrottle prestart") +func (i *IOLimit) PreStart(viewer api.Viewer) error { + log.Infof("ioLimit prestart") return nil } -func (b *BlkioThrottle) Terminate(viewer api.Viewer) error { - log.Infof("blkiothrottle Terminate") +func (i *IOLimit) Terminate(viewer api.Viewer) error { + log.Infof("ioLimit Terminate") return nil } -func (b *BlkioThrottle) ID() string { - return b.Name +func (i *IOLimit) ID() string { + return i.Name } -func (b *BlkioThrottle) AddFunc(podInfo *typedef.PodInfo) error { +func (i *IOLimit) AddFunc(podInfo *typedef.PodInfo) error { return nil } -func (b *BlkioThrottle) UpdateFunc(old, new *typedef.PodInfo) error { +func (i *IOLimit) UpdateFunc(old, new *typedef.PodInfo) error { return nil } -func (b *BlkioThrottle) DeleteFunc(podInfo *typedef.PodInfo) error { +func (i *IOLimit) DeleteFunc(podInfo *typedef.PodInfo) error { return nil } diff --git a/pkg/services/qos/qos.go b/pkg/services/preemption/preemption.go similarity index 67% rename from pkg/services/qos/qos.go rename to pkg/services/preemption/preemption.go index 23b0a3b..8a12dd5 100644 --- a/pkg/services/qos/qos.go +++ b/pkg/services/preemption/preemption.go @@ -12,7 +12,7 @@ // Description: This file implement qos level setting service // Package qos is the service used for qos level setting -package qos +package preemption import ( "fmt" @@ -31,15 +31,15 @@ var supportCgroupTypes = map[string]*cgroup.Key{ "memory": {SubSys: "memory", FileName: constant.MemoryCgroupFileName}, } -// QoS define service which related to qos level setting -type QoS struct { +// Preemption define service which related to qos level setting +type Preemption struct { Name string `json:"-"` Config } // Config contains sub-system that need to set qos level type Config struct { - SubSys []string `json:"subSys"` + Resource []string `json:"resource,omitempty"` } func init() { @@ -49,21 +49,21 @@ func init() { } // NewQoS return qos instance -func NewQoS() *QoS { - return &QoS{ +func NewQoS() *Preemption { + return &Preemption{ Name: "qos", } } // ID return qos service name -func (q *QoS) ID() string { +func (q *Preemption) ID() string { return q.Name } // PreStart is the pre-start action -func (q *QoS) PreStart(viewer api.Viewer) error { +func (q *Preemption) PreStart(viewer api.Viewer) error { for _, pod := range viewer.ListPodsWithOptions() { - if err := q.SetQoS(pod); err != nil { + if err := q.SetQoSLevel(pod); err != nil { log.Errorf("error prestart pod %v: %v", pod.Name, err) } } @@ -71,18 +71,18 @@ func (q *QoS) PreStart(viewer api.Viewer) error { } // AddFunc implement add function when pod is added in k8s -func (q *QoS) AddFunc(pod *typedef.PodInfo) error { - if err := q.SetQoS(pod); err != nil { +func (q *Preemption) AddFunc(pod *typedef.PodInfo) error { + if err := q.SetQoSLevel(pod); err != nil { return err } - if err := q.ValidateQoS(pod); err != nil { + if err := q.ValidateConfig(pod); err != nil { return err } return nil } // UpdateFunc implement update function when pod info is changed -func (q *QoS) UpdateFunc(old, new *typedef.PodInfo) error { +func (q *Preemption) UpdateFunc(old, new *typedef.PodInfo) error { oldQos, newQos := getQoSLevel(old), getQoSLevel(new) switch { case newQos == oldQos: @@ -90,8 +90,8 @@ func (q *QoS) UpdateFunc(old, new *typedef.PodInfo) error { case newQos > oldQos: return fmt.Errorf("not support change qos level from low to high") default: - if err := q.ValidateQoS(new); err != nil { - if err := q.SetQoS(new); err != nil { + if err := q.ValidateConfig(new); err != nil { + if err := q.SetQoSLevel(new); err != nil { return fmt.Errorf("update qos for pod %s(%s) failed: %v", new.Name, new.UID, err) } } @@ -100,20 +100,20 @@ func (q *QoS) UpdateFunc(old, new *typedef.PodInfo) error { } // DeleteFunc implement delete function when pod is deleted by k8s -func (q *QoS) DeleteFunc(pod *typedef.PodInfo) error { +func (q *Preemption) DeleteFunc(pod *typedef.PodInfo) error { return nil } -// ValidateQoS will validate pod's qos level between value from +// ValidateConfig will validate pod's qos level between value from // cgroup file and the one from pod info -func (q *QoS) ValidateQoS(pod *typedef.PodInfo) error { +func (q *Preemption) ValidateConfig(pod *typedef.PodInfo) error { targetLevel := getQoSLevel(pod) - for _, subSys := range q.SubSys { - if err := pod.GetCgroupAttr(supportCgroupTypes[subSys]).Expect(targetLevel); err != nil { + for _, r := range q.Resource { + if err := pod.GetCgroupAttr(supportCgroupTypes[r]).Expect(targetLevel); err != nil { return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) } for _, container := range pod.IDContainersMap { - if err := container.GetCgroupAttr(supportCgroupTypes[subSys]).Expect(targetLevel); err != nil { + if err := container.GetCgroupAttr(supportCgroupTypes[r]).Expect(targetLevel); err != nil { return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) } } @@ -121,8 +121,8 @@ func (q *QoS) ValidateQoS(pod *typedef.PodInfo) error { return nil } -// SetQoS set pod and all containers' qos level within it -func (q *QoS) SetQoS(pod *typedef.PodInfo) error { +// SetQoSLevel set pod and all containers' qos level within it +func (q *Preemption) SetQoSLevel(pod *typedef.PodInfo) error { if pod == nil { return fmt.Errorf("pod info is empty") } @@ -132,12 +132,12 @@ func (q *QoS) SetQoS(pod *typedef.PodInfo) error { return nil } - for _, sys := range q.SubSys { - if err := pod.SetCgroupAttr(supportCgroupTypes[sys], strconv.Itoa(qosLevel)); err != nil { + for _, r := range q.Resource { + if err := pod.SetCgroupAttr(supportCgroupTypes[r], strconv.Itoa(qosLevel)); err != nil { return err } for _, container := range pod.IDContainersMap { - if err := container.SetCgroupAttr(supportCgroupTypes[sys], strconv.Itoa(qosLevel)); err != nil { + if err := container.SetCgroupAttr(supportCgroupTypes[r], strconv.Itoa(qosLevel)); err != nil { return err } } @@ -165,13 +165,13 @@ func getQoSLevel(pod *typedef.PodInfo) int { } // Validate will validate the qos service config -func (q *QoS) Validate() error { - if len(q.SubSys) == 0 { +func (q *Preemption) Validate() error { + if len(q.Resource) == 0 { return fmt.Errorf("empty qos config") } - for _, subSys := range q.SubSys { - if _, ok := supportCgroupTypes[subSys]; !ok { - return fmt.Errorf("not support sub system %s", subSys) + for _, r := range q.Resource { + if _, ok := supportCgroupTypes[r]; !ok { + return fmt.Errorf("not support sub system %s", r) } } return nil diff --git a/pkg/services/qos/qos_test.go b/pkg/services/preemption/preemption_test.go similarity index 93% rename from pkg/services/qos/qos_test.go rename to pkg/services/preemption/preemption_test.go index f4dd395..ff8cec9 100644 --- a/pkg/services/qos/qos_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -12,7 +12,7 @@ // Description: This file test qos level setting service // Package qos is the service used for qos level setting -package qos +package preemption import ( "testing" @@ -46,11 +46,11 @@ type test struct { var getCommonField = func(subSys []string) fields { return fields{ Name: "qos", - Config: Config{SubSys: subSys}, + Config: Config{Resource: subSys}, } } -func TestQoS_AddFunc(t *testing.T) { +func TestPreemptionAddFunc(t *testing.T) { var addFuncTC = []test{ { name: "TC1-set offline pod qos ok", @@ -105,7 +105,7 @@ func TestQoS_AddFunc(t *testing.T) { for _, tt := range addFuncTC { t.Run(tt.name, func(t *testing.T) { - q := &QoS{ + q := &Preemption{ Name: tt.fields.Name, Config: tt.fields.Config, } @@ -123,7 +123,7 @@ func TestQoS_AddFunc(t *testing.T) { } } -func TestQoS_UpdateFunc(t *testing.T) { +func TestPreemptionUpdateFunc(t *testing.T) { var updateFuncTC = []test{ { name: "TC1-online to offline", @@ -163,7 +163,7 @@ func TestQoS_UpdateFunc(t *testing.T) { for _, tt := range updateFuncTC { t.Run(tt.name, func(t *testing.T) { - q := &QoS{ + q := &Preemption{ Name: tt.fields.Name, Config: tt.fields.Config, } @@ -179,20 +179,20 @@ func TestQoS_UpdateFunc(t *testing.T) { } } -func TestQoS_Validate(t *testing.T) { +func TestPreemptionValidate(t *testing.T) { var validateTC = []test{ { name: "TC1-normal config", fields: fields{ Name: "qos", - Config: Config{SubSys: []string{"cpu", "memory"}}, + Config: Config{Resource: []string{"cpu", "memory"}}, }, }, { name: "TC2-abnormal config", fields: fields{ Name: "undefine", - Config: Config{SubSys: []string{"undefine"}}, + Config: Config{Resource: []string{"undefine"}}, }, wantErr: true, }, @@ -204,7 +204,7 @@ func TestQoS_Validate(t *testing.T) { for _, tt := range validateTC { t.Run(tt.name, func(t *testing.T) { - q := &QoS{ + q := &Preemption{ Name: tt.fields.Name, Config: tt.fields.Config, } -- Gitee From b33b5a214fe36b62d7c164bc240136ba15aa9e74 Mon Sep 17 00:00:00 2001 From: hanchao Date: Fri, 10 Mar 2023 19:17:32 +0800 Subject: [PATCH 54/73] refactor: restructure the service module Use dependency registration to implement the factory of related modules. Rubik use the rubik_feature.go to init modules. --- hack/rubik-daemonset.yaml | 12 +- pkg/feature/feature.go | 11 ++ pkg/rubik/rubik.go | 13 +- pkg/rubik/rubik_feature.go | 37 +++++ pkg/{services => rubik}/servicemanager.go | 176 ++++++--------------- pkg/services/component.go | 56 +++++++ pkg/services/dynCache/dynCache.go | 24 +-- pkg/services/dynCache/dynCache_test.go | 6 +- pkg/services/helper/factory.go | 32 ++++ pkg/services/helper/service_base.go | 56 +++++++ pkg/services/iocost/iocost.go | 54 ++++--- pkg/services/iolimit/iolimit.go | 31 ++-- pkg/services/preemption/preemption.go | 45 +++--- pkg/services/preemption/preemption_test.go | 20 +-- pkg/services/quotaburst/quotaburst.go | 30 ++-- pkg/services/quotaburst/quotaburst_test.go | 12 +- pkg/services/quotaturbo/data.go | 2 +- pkg/services/quotaturbo/quotaturbo.go | 22 ++- pkg/services/quotaturbo/quotaturbo_test.go | 4 +- pkg/services/registry.go | 53 ------- pkg/services/service.go | 77 +++++++++ 21 files changed, 486 insertions(+), 287 deletions(-) create mode 100644 pkg/feature/feature.go create mode 100644 pkg/rubik/rubik_feature.go rename pkg/{services => rubik}/servicemanager.go (53%) create mode 100644 pkg/services/component.go create mode 100644 pkg/services/helper/factory.go create mode 100644 pkg/services/helper/service_base.go delete mode 100644 pkg/services/registry.go create mode 100644 pkg/services/service.go diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml index 9ae80d7..b706018 100644 --- a/hack/rubik-daemonset.yaml +++ b/hack/rubik-daemonset.yaml @@ -39,10 +39,16 @@ data: "logDir": "/var/log/rubik", "logSize": 1024, "logLevel": "info", - "cgroupRoot": "/sys/fs/cgroup" + "cgroupRoot": "/sys/fs/cgroup", + "enabledFeatures": [ + "preemption" + ] }, - "qos": { - "subSys": ["cpu", "memory"] + "preemption": { + "resource": [ + "cpu", + "memory" + ] } } --- diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go new file mode 100644 index 0000000..9eb5a52 --- /dev/null +++ b/pkg/feature/feature.go @@ -0,0 +1,11 @@ +package feature + +var ( + FeaturePreemption = "preemption" + FeatureDynCache = "dynCache" + FeatureIOLimit = "ioLimit" + FeatureIOCost = "ioCost" + FeatureDynMemory = "dynMemory" + FeatureQuotaBurst = "quotaBurst" + FeatureQuotaTurbo = "quotaTurbo" +) diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 62fa40f..3c94bac 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -40,14 +40,15 @@ type Agent struct { config *config.Config podManager *podmanager.PodManager informer api.Informer - servicesManager *services.ServiceManager + servicesManager *ServiceManager } // NewAgent returns an agent for given configuration func NewAgent(cfg *config.Config) (*Agent, error) { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) - serviceManager := services.NewServiceManager() - if err := serviceManager.InitServices(cfg.UnwarpServiceConfig(), cfg); err != nil { + serviceManager := NewServiceManager() + if err := serviceManager.InitServices( cfg.Agent.EnabledFeatures, + cfg.UnwarpServiceConfig(), cfg); err != nil { return nil, err } a := &Agent{ @@ -116,6 +117,7 @@ func runAgent(ctx context.Context) error { if err := c.LoadConfig(constant.ConfigFile); err != nil { return fmt.Errorf("error loading config: %v", err) } + // 2. enable log system if err := log.InitConfig(c.Agent.LogDriver, c.Agent.LogDir, c.Agent.LogLevel, c.Agent.LogSize); err != nil { return fmt.Errorf("error initializing log: %v", err) @@ -124,7 +126,10 @@ func runAgent(ctx context.Context) error { // 3. enable cgroup system cgroup.InitMountDir(c.Agent.CgroupRoot) - // 4. Create and run the agent + // 4. init service components + services.InitServiceComponents(defaultRubikFeature) + + // 5. Create and run the agent agent, err := NewAgent(c) if err != nil { return fmt.Errorf("error new agent: %v", err) diff --git a/pkg/rubik/rubik_feature.go b/pkg/rubik/rubik_feature.go new file mode 100644 index 0000000..daa722b --- /dev/null +++ b/pkg/rubik/rubik_feature.go @@ -0,0 +1,37 @@ +package rubik + +import ( + "isula.org/rubik/pkg/feature" + "isula.org/rubik/pkg/services" +) + +var defaultRubikFeature = []services.FeatureSpec{ + { + Name: feature.FeaturePreemption, + Default: true, + }, + { + Name: feature.FeatureDynCache, + Default: true, + }, + { + Name: feature.FeatureIOLimit, + Default: true, + }, + { + Name: feature.FeatureIOCost, + Default: true, + }, + { + Name: feature.FeatureDynMemory, + Default: true, + }, + { + Name: feature.FeatureQuotaBurst, + Default: true, + }, + { + Name: feature.FeatureQuotaTurbo, + Default: true, + }, +} diff --git a/pkg/services/servicemanager.go b/pkg/rubik/servicemanager.go similarity index 53% rename from pkg/services/servicemanager.go rename to pkg/rubik/servicemanager.go index 8130842..a1d46d3 100644 --- a/pkg/services/servicemanager.go +++ b/pkg/rubik/servicemanager.go @@ -12,7 +12,7 @@ // Description: This file defines ServiceManager to manage the lifecycle of services // Package services implements service registration, discovery and management functions -package services +package rubik import ( "context" @@ -27,6 +27,7 @@ import ( "isula.org/rubik/pkg/config" "isula.org/rubik/pkg/core/subscriber" "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/services" ) // serviceManagerName is the unique ID of the service manager @@ -37,40 +38,40 @@ type ServiceManager struct { api.Subscriber api.Viewer sync.RWMutex - RunningServices map[string]api.Service - RunningPersistentServices map[string]api.PersistentService - TerminateFuncs map[string]Terminator + RunningServices map[string]services.Service } // NewServiceManager creates a servicemanager object func NewServiceManager() *ServiceManager { manager := &ServiceManager{ - RunningServices: make(map[string]api.Service), - RunningPersistentServices: make(map[string]api.PersistentService), + RunningServices: make(map[string]services.Service), } manager.Subscriber = subscriber.NewGenericSubscriber(manager, serviceManagerName) return manager } // InitServices parses the to-be-run services config and loads them to the ServiceManager -func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{}, parser config.ConfigParser) error { - for name, config := range serviceConfig { - creator := GetServiceCreator(name) - if creator == nil { - return fmt.Errorf("no corresponding module %v, please check the module name", name) - } - service := creator() - if err := parser.UnmarshalSubConfig(config, service); err != nil { - return fmt.Errorf("error unmarshaling %s config: %v", name, err) +func (manager *ServiceManager) InitServices(features []string, serviceConfig map[string]interface{}, parser config.ConfigParser) error { + for _, feature := range features { + s, err := services.GetServiceComponent(feature) + if err != nil { + return fmt.Errorf("get component failed %s: %v", feature, err) } - // try to verify configuration - if validator, ok := service.(Validator); ok { - if err := validator.Validate(); err != nil { - return fmt.Errorf("error configuring service %s: %v", name, err) + if err := s.SetConfig(func(configName string, v interface{}) error { + config := serviceConfig[configName] + if config == nil { + return fmt.Errorf("this configuration is not available,configName:%v", configName) + } + if err := parser.UnmarshalSubConfig(config, v); err != nil { + return fmt.Errorf("this configuration failed to be serialized,configName:%v,error:%v", configName, err) } + return nil + }); err != nil { + return fmt.Errorf("set configuration failed, err:%v", err) } - if err := manager.AddRunningService(name, service); err != nil { + + if err := manager.AddRunningService(feature, s); err != nil { return err } } @@ -78,21 +79,14 @@ func (manager *ServiceManager) InitServices(serviceConfig map[string]interface{} } // AddRunningService adds a to-be-run service -func (manager *ServiceManager) AddRunningService(name string, service interface{}) error { - manager.RLock() - _, existed1 := manager.RunningServices[name] - _, existed2 := manager.RunningPersistentServices[name] - manager.RUnlock() - if existed1 || existed2 { +func (manager *ServiceManager) AddRunningService(name string, s services.Service) error { + manager.Lock() + defer manager.Unlock() + if _, existed := manager.RunningServices[name]; existed { return fmt.Errorf("service name conflict: %s", name) } - - addService := manager.tryAddService(name, service) - addPersistentService := manager.tryAddPersistentService(name, service) - if addPersistentService || addService { - return nil - } - return fmt.Errorf("invalid service %s (type %T)", name, service) + manager.RunningServices[name] = s + return nil } // HandleEvent is used to handle PodInfo events pushed by the publisher @@ -119,41 +113,19 @@ func (manager *ServiceManager) EventTypes() []typedef.EventType { return []typedef.EventType{typedef.INFOADD, typedef.INFOUPDATE, typedef.INFODELETE} } -// tryAddService determines whether it is a api.Service and adds it to the queue to be run -func (manager *ServiceManager) tryAddService(name string, service interface{}) bool { - s, ok := service.(api.Service) - if ok { - manager.Lock() - manager.RunningServices[name] = s - manager.Unlock() - log.Infof("service %s will run", name) - } - return ok -} - -// tryAddPersistentService determines whether it is a api.PersistentService and adds it to the queue to be run -func (manager *ServiceManager) tryAddPersistentService(name string, service interface{}) bool { - s, ok := service.(api.PersistentService) - if ok { - manager.Lock() - manager.RunningPersistentServices[name] = s - manager.Unlock() - log.Infof("persistent service %s will run", name) - } - return ok -} - // terminatingRunningServices handles services exits during the setup and exit phases -func (manager *ServiceManager) terminatingRunningServices(err error) error { - if manager.TerminateFuncs == nil { - return nil - } - for id, t := range manager.TerminateFuncs { - if termErr := t.Terminate(manager.Viewer); termErr != nil { - log.Errorf("error terminating services %s: %v", id, termErr) +func (manager *ServiceManager) terminatingRunningServices() error { + for _, s := range manager.RunningServices { + if s.IsRunner() { + if err := s.Stop(); err != nil { + log.Errorf("stop service err:%v", err) + } + } + if err := s.Terminate(manager.Viewer); err != nil { + log.Errorf("terminate service err:%v", err) } } - return err + return nil } // Setup pre-starts services, such as preparing the environment, etc. @@ -162,44 +134,10 @@ func (manager *ServiceManager) Setup(v api.Viewer) error { if v == nil { return nil } - preStarted := make(map[string]struct{}, 0) - manager.Viewer = v - manager.TerminateFuncs = make(map[string]Terminator) - setupFunc := func(id string, s interface{}) error { - // 1. record the termination function of the service that has been setup - if t, ok := s.(Terminator); ok { - manager.TerminateFuncs[id] = t - } - // 2. execute the pre-start function of the service - p, ok := s.(PreStarter) - if !ok { - return nil - } - // execute the prestart method only once if the service is both a persistent service and a service - if _, ok := preStarted[id]; ok { - return nil - } - if err := p.PreStart(manager.Viewer); err != nil { - return err - } - preStarted[id] = struct{}{} - return nil - } - // 1. pre-start services for _, s := range manager.RunningServices { - if err := setupFunc(s.ID(), s); err != nil { - /* - handle the error and terminate all services that have been started - when any setup stage failed - */ - return manager.terminatingRunningServices(fmt.Errorf("error running services %s: %v", s.ID(), err)) - } - } - // 2. pre-start persistent services - for _, s := range manager.RunningPersistentServices { - if err := setupFunc(s.ID(), s); err != nil { - return manager.terminatingRunningServices(fmt.Errorf("error running persistent services %s: %v", s.ID(), err)) + if err := s.PreStart(manager.Viewer); err != nil { + log.Errorf("PreStart failed:%v", err) } } return nil @@ -228,14 +166,17 @@ func (manager *ServiceManager) Start(ctx context.Context) { runFunc(ctx) }, restartDuration) } - for id, s := range manager.RunningPersistentServices { - go runner(ctx, id, s.Run) + + for id, s := range manager.RunningServices { + if s.IsRunner() { + go runner(ctx, id, s.Run) + } } } // Stop terminates the running service func (manager *ServiceManager) Stop() error { - return manager.terminatingRunningServices(nil) + return manager.terminatingRunningServices() } // addFunc handles pod addition events @@ -247,10 +188,10 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { } const retryCount = 5 - runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + runOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) for i := 0; i < retryCount; i++ { - if err := s.AddFunc(podInfo); err != nil { + if err := s.AddPod(podInfo); err != nil { log.Errorf("service %s add func failed: %v", s.ID(), err) } else { break @@ -279,10 +220,10 @@ func (manager *ServiceManager) updateFunc(event typedef.Event) { log.Warnf("pod infos contains more than two pods") return } - runOnce := func(s api.Service, old, new *typedef.PodInfo, wg *sync.WaitGroup) { + runOnce := func(s services.Service, old, new *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) log.Debugf("update Func with service: %s", s.ID()) - if err := s.UpdateFunc(old, new); err != nil { + if err := s.UpdatePod(old, new); err != nil { log.Errorf("service %s update func failed: %v", s.ID(), err) } wg.Done() @@ -304,9 +245,9 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { return } - runOnce := func(s api.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + runOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) - if err := s.DeleteFunc(podInfo); err != nil { + if err := s.DeletePod(podInfo); err != nil { log.Errorf("service %s delete func failed: %v", s.ID(), err) } wg.Done() @@ -319,18 +260,3 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { wg.Wait() manager.RUnlock() } - -// Terminator is an interface that calls a collection of methods when the service terminates -type Terminator interface { - Terminate(api.Viewer) error -} - -// PreStarter is an interface for calling a collection of methods when the service is pre-started -type PreStarter interface { - PreStart(api.Viewer) error -} - -// Validator is a function interface to verify whether the configuration is correct or not -type Validator interface { - Validate() error -} diff --git a/pkg/services/component.go b/pkg/services/component.go new file mode 100644 index 0000000..a4bc9dd --- /dev/null +++ b/pkg/services/component.go @@ -0,0 +1,56 @@ +package services + +import ( + "isula.org/rubik/pkg/feature" + "isula.org/rubik/pkg/services/dynCache" + "isula.org/rubik/pkg/services/helper" + "isula.org/rubik/pkg/services/iocost" + "isula.org/rubik/pkg/services/iolimit" + "isula.org/rubik/pkg/services/preemption" + "isula.org/rubik/pkg/services/quotaburst" + "isula.org/rubik/pkg/services/quotaturbo" +) + +type ServiceComponent func(name string) error + +var ( + serviceComponents = map[string]ServiceComponent{ + feature.FeaturePreemption: initPreemptionFactory, + feature.FeatureDynCache: initDynCacheFactory, + feature.FeatureIOLimit: initIOLimitFactory, + feature.FeatureIOCost: initIOCostFactory, + feature.FeatureDynMemory: initDynCacheFactory, + feature.FeatureQuotaBurst: initQuotaBurstFactory, + feature.FeatureQuotaTurbo: initQuotaTurboFactory, + } +) + +func initIOLimitFactory(name string) error { + helper.AddFactory(name, iolimit.IOLimitFactory{ObjName: name}) + return nil +} + +func initIOCostFactory(name string) error { + helper.AddFactory(name, iocost.IOCostFactory{ObjName: name}) + return nil +} + +func initDynCacheFactory(name string) error { + helper.AddFactory(name, dynCache.DynCacheFactory{ObjName: name}) + return nil +} + +func initQuotaTurboFactory(name string) error { + helper.AddFactory(name, quotaturbo.QuotaTurboFactory{ObjName: name}) + return nil +} + +func initQuotaBurstFactory(name string) error { + helper.AddFactory(name, quotaburst.BurstFactory{ObjName: name}) + return nil +} + +func initPreemptionFactory(name string) error { + helper.AddFactory(name, preemption.PreemptionFactory{ObjName: name}) + return nil +} diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index 5dbf63b..4bbd00c 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "isula.org/rubik/pkg/api" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) // default value @@ -37,7 +37,6 @@ const ( defaultHighMB = 50 defaultMaxMiss = 20 defaultMinMiss = 10 - moduleName = "cacheLimit" defaultResctrlDir = "/sys/fs/resctrl" defaultNumaNodeDir = "/sys/devices/system/node" defaultPidNameSpace = "/proc/self/ns/pid" @@ -88,6 +87,7 @@ type Config struct { // DynCache is cache limit service structure type DynCache struct { + helper.ServiceBase *Config Attr *Attr Viewer api.Viewer @@ -112,16 +112,22 @@ type Attr struct { MinMiss int } -func init() { - services.Register(moduleName, func() interface{} { - return NewCacheLimit() - }) +type DynCacheFactory struct { + ObjName string } -// NewCacheLimit return cache limit instance with default settings -func NewCacheLimit() *DynCache { +func (i DynCacheFactory) Name() string { + return "DynCacheFactory" +} + +func (i DynCacheFactory) NewObj() (interface{}, error) { + return NewDynCache(i.ObjName), nil +} + +// NewDynCache return cache limit instance with default settings +func NewDynCache(name string) *DynCache { return &DynCache{ - Name: moduleName, + Name: name, Attr: &Attr{ NumaNodeDir: defaultNumaNodeDir, MaxMiss: defaultMaxMiss, diff --git a/pkg/services/dynCache/dynCache_test.go b/pkg/services/dynCache/dynCache_test.go index 3a43bd9..3c57ec4 100644 --- a/pkg/services/dynCache/dynCache_test.go +++ b/pkg/services/dynCache/dynCache_test.go @@ -23,6 +23,10 @@ import ( "isula.org/rubik/pkg/api" ) +const ( + moduleName = "dynCache" +) + func TestCacheLimit_Validate(t *testing.T) { type fields struct { Config *Config @@ -225,7 +229,7 @@ func TestNewCacheLimit(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NewCacheLimit(); !reflect.DeepEqual(got, tt.want) { + if got := NewDynCache(moduleName); !reflect.DeepEqual(got, tt.want) { t.Errorf("NewCacheLimit() = %v, want %v", got, tt.want) } }) diff --git a/pkg/services/helper/factory.go b/pkg/services/helper/factory.go new file mode 100644 index 0000000..5d50e24 --- /dev/null +++ b/pkg/services/helper/factory.go @@ -0,0 +1,32 @@ +package helper + +import ( + "errors" + "sync" +) + +type ServiceFactory interface { + Name() string + NewObj() (interface{}, error) +} + +var ( + rwlock sync.RWMutex + serviceFactories = map[string]ServiceFactory{} +) + +func AddFactory(name string, factory ServiceFactory) { + rwlock.Lock() + defer rwlock.Unlock() + serviceFactories[name] = factory +} + +func GetComponent(name string) (interface{}, error) { + rwlock.RLock() + defer rwlock.RUnlock() + if f, found := serviceFactories[name]; found { + return f.NewObj() + } else { + return nil, errors.New("factory is not found") + } +} diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go new file mode 100644 index 0000000..35aa984 --- /dev/null +++ b/pkg/services/helper/service_base.go @@ -0,0 +1,56 @@ +package helper + +import ( + "context" + "fmt" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/core/typedef" +) + +type ServiceBase struct{} +type HandlerConfig func(configName string, d interface{}) error + +// ID is the name of plugin, must be unique. +func (s *ServiceBase) ID() string { + panic("this interface must be implemented.") +} + +// PreStarter is an interface for calling a collection of methods when the service is pre-started +func (s *ServiceBase) PreStart(api.Viewer) error { + return nil +} + +// Terminator is an interface that calls a collection of methods when the service terminates +func (s *ServiceBase) Terminate(api.Viewer) error { + return nil +} + +// Confirm whether it is +func (s *ServiceBase) IsRunner() bool { + return false +} + +// Start runner +func (s *ServiceBase) Run(ctx context.Context) {} + +// Stop runner +func (s *ServiceBase) Stop() error { + return fmt.Errorf("i am not runner") +} + +func (s *ServiceBase) AddPod(podInfo *typedef.PodInfo) error { + return nil +} + +func (S *ServiceBase) UpdatePod(old, new *typedef.PodInfo) error { + return nil +} + +func (s *ServiceBase) DeletePod(podInfo *typedef.PodInfo) error { + return nil +} + +func (s *ServiceBase) SetConfig(h HandlerConfig) error { + return nil +} diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 951f7a0..ac9116c 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -1,6 +1,7 @@ package iocost import ( + "fmt" "os" "strings" "unicode" @@ -11,7 +12,7 @@ import ( "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) const ( @@ -48,19 +49,28 @@ type NodeConfig struct { // IOCost for iocost class type IOCost struct { + helper.ServiceBase name string - nodeConfigs []NodeConfig } var ( nodeName string ) -// init iocost: register service and ensure this platform support iocost. -func init() { - services.Register("iocost", func() interface{} { - return &IOCost{name: "iocost"} - }) +type IOCostFactory struct { + ObjName string +} + +func (i IOCostFactory) Name() string { + return "IOCostFactory" +} + +func (i IOCostFactory) NewObj() (interface{}, error) { + if IOCostSupport() { + nodeName = os.Getenv(constant.NodeNameEnvKey) + return &IOCost{name: i.ObjName}, nil + } + return nil, fmt.Errorf("this machine not support iocost") } // IOCostSupport tell if the os support iocost. @@ -71,22 +81,18 @@ func IOCostSupport() bool { } // ID for get the name of iocost -func (b *IOCost) ID() string { - return b.name +func (io *IOCost) ID() string { + return io.name } -func (b *IOCost) PreStart(viewer api.Viewer) error { - nodeName = os.Getenv(constant.NodeNameEnvKey) - if err := b.loadConfig(); err != nil { +func (io *IOCost) SetConfig(f helper.HandlerConfig) error { + var nodeConfigs []NodeConfig + var nodeConfig *NodeConfig + if err := f(io.name, nodeConfigs); err != nil { return err } - return b.dealExistedPods(viewer) -} -func (b *IOCost) loadConfig() error { - var nodeConfig *NodeConfig - // global will set all node - for _, config := range b.nodeConfigs { + for _, config := range nodeConfigs { if config.NodeName == nodeName { nodeConfig = &config break @@ -95,9 +101,12 @@ func (b *IOCost) loadConfig() error { nodeConfig = &config } } + return io.loadConfig(nodeConfig) +} +func (io *IOCost) loadConfig(nodeConfig *NodeConfig) error { // ensure that previous configuration is cleared. - if err := b.clearIOCost(); err != nil { + if err := io.clearIOCost(); err != nil { log.Errorf("clear iocost err:%v", err) return err } @@ -108,8 +117,13 @@ func (b *IOCost) loadConfig() error { return nil } - b.configIOCost(nodeConfig.IOCostConfig) + io.configIOCost(nodeConfig.IOCostConfig) return nil + +} + +func (io *IOCost) PreStart(viewer api.Viewer) error { + return io.dealExistedPods(viewer) } func (b *IOCost) Terminate(viewer api.Viewer) error { diff --git a/pkg/services/iolimit/iolimit.go b/pkg/services/iolimit/iolimit.go index a2bc8ac..9deb755 100644 --- a/pkg/services/iolimit/iolimit.go +++ b/pkg/services/iolimit/iolimit.go @@ -1,10 +1,10 @@ -package blkcg +package iolimit import ( "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) // DeviceConfig defines blkio device configurations @@ -21,17 +21,26 @@ type IOLimitAnnoConfig struct { } type IOLimit struct { - Name string `json:"-"` + helper.ServiceBase + name string } -func init() { - services.Register("ioLimit", func() interface{} { - return NewBlkioThrottle() - }) +type IOLimitFactory struct { + ObjName string } -func NewBlkioThrottle() *IOLimit { - return &IOLimit{Name: "ioLimit"} +func (i IOLimitFactory) Name() string { + return "IOLimitFactory" +} + +func (i IOLimitFactory) NewObj() (interface{}, error) { + return &IOLimit{name: i.ObjName}, nil +} + +// ========================= + +func (i *IOLimit) ID() string { + return i.name } func (i *IOLimit) PreStart(viewer api.Viewer) error { @@ -44,10 +53,6 @@ func (i *IOLimit) Terminate(viewer api.Viewer) error { return nil } -func (i *IOLimit) ID() string { - return i.Name -} - func (i *IOLimit) AddFunc(podInfo *typedef.PodInfo) error { return nil } diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index 8a12dd5..874e921 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -23,7 +23,7 @@ import ( "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) var supportCgroupTypes = map[string]*cgroup.Key{ @@ -33,31 +33,40 @@ var supportCgroupTypes = map[string]*cgroup.Key{ // Preemption define service which related to qos level setting type Preemption struct { - Name string `json:"-"` - Config + helper.ServiceBase + name string + config PreemptionConfig } // Config contains sub-system that need to set qos level -type Config struct { +type PreemptionConfig struct { Resource []string `json:"resource,omitempty"` } -func init() { - services.Register("qos", func() interface{} { - return NewQoS() - }) +type PreemptionFactory struct { + ObjName string } -// NewQoS return qos instance -func NewQoS() *Preemption { - return &Preemption{ - Name: "qos", - } +func (i PreemptionFactory) Name() string { + return "PreemptionFactory" +} + +func (i PreemptionFactory) NewObj() (interface{}, error) { + return &Preemption{name: i.ObjName}, nil } // ID return qos service name func (q *Preemption) ID() string { - return q.Name + return q.name +} + +func (q *Preemption) SetConfig(f helper.HandlerConfig) error { + var c PreemptionConfig + if err := f(q.name, c); err != nil { + return err + } + q.config = c + return nil } // PreStart is the pre-start action @@ -108,7 +117,7 @@ func (q *Preemption) DeleteFunc(pod *typedef.PodInfo) error { // cgroup file and the one from pod info func (q *Preemption) ValidateConfig(pod *typedef.PodInfo) error { targetLevel := getQoSLevel(pod) - for _, r := range q.Resource { + for _, r := range q.config.Resource { if err := pod.GetCgroupAttr(supportCgroupTypes[r]).Expect(targetLevel); err != nil { return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) } @@ -132,7 +141,7 @@ func (q *Preemption) SetQoSLevel(pod *typedef.PodInfo) error { return nil } - for _, r := range q.Resource { + for _, r := range q.config.Resource { if err := pod.SetCgroupAttr(supportCgroupTypes[r], strconv.Itoa(qosLevel)); err != nil { return err } @@ -166,10 +175,10 @@ func getQoSLevel(pod *typedef.PodInfo) int { // Validate will validate the qos service config func (q *Preemption) Validate() error { - if len(q.Resource) == 0 { + if len(q.config.Resource) == 0 { return fmt.Errorf("empty qos config") } - for _, r := range q.Resource { + for _, r := range q.config.Resource { if _, ok := supportCgroupTypes[r]; !ok { return fmt.Errorf("not support sub system %s", r) } diff --git a/pkg/services/preemption/preemption_test.go b/pkg/services/preemption/preemption_test.go index ff8cec9..746704c 100644 --- a/pkg/services/preemption/preemption_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -28,7 +28,7 @@ func init() { type fields struct { Name string - Config Config + Config PreemptionConfig } type args struct { old *try.FakePod @@ -46,7 +46,7 @@ type test struct { var getCommonField = func(subSys []string) fields { return fields{ Name: "qos", - Config: Config{Resource: subSys}, + Config: PreemptionConfig{Resource: subSys}, } } @@ -106,8 +106,8 @@ func TestPreemptionAddFunc(t *testing.T) { for _, tt := range addFuncTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - Name: tt.fields.Name, - Config: tt.fields.Config, + name: tt.fields.Name, + config: tt.fields.Config, } if tt.preHook != nil { tt.preHook(tt.args.new) @@ -164,8 +164,8 @@ func TestPreemptionUpdateFunc(t *testing.T) { for _, tt := range updateFuncTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - Name: tt.fields.Name, - Config: tt.fields.Config, + name: tt.fields.Name, + config: tt.fields.Config, } if tt.preHook != nil { tt.args.new = tt.preHook(tt.args.old) @@ -185,14 +185,14 @@ func TestPreemptionValidate(t *testing.T) { name: "TC1-normal config", fields: fields{ Name: "qos", - Config: Config{Resource: []string{"cpu", "memory"}}, + Config: PreemptionConfig{Resource: []string{"cpu", "memory"}}, }, }, { name: "TC2-abnormal config", fields: fields{ Name: "undefine", - Config: Config{Resource: []string{"undefine"}}, + Config: PreemptionConfig{Resource: []string{"undefine"}}, }, wantErr: true, }, @@ -205,8 +205,8 @@ func TestPreemptionValidate(t *testing.T) { for _, tt := range validateTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - Name: tt.fields.Name, - Config: tt.fields.Config, + name: tt.fields.Name, + config: tt.fields.Config, } if err := q.Validate(); (err != nil) != tt.wantErr { t.Errorf("QoS.Validate() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index db6bbb7..b5c9946 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -28,34 +28,30 @@ import ( "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) -const ( - moduleName = "quotaburst" -) +// Burst is used to control cpu burst +type Burst struct { + helper.ServiceBase + name string +} -func init() { - services.Register(moduleName, func() interface{} { - return NewBurst() - }) +type BurstFactory struct { + ObjName string } -// Burst is used to control cpu burst -type Burst struct { - Name string `json:"-"` +func (i BurstFactory) Name() string { + return "BurstFactory" } -// NewBurst return an new Burst pointer -func NewBurst() *Burst { - return &Burst{ - Name: moduleName, - } +func (i BurstFactory) NewObj() (interface{}, error) { + return &Burst{name: i.ObjName}, nil } // ID returns the module name func (b *Burst) ID() string { - return moduleName + return b.name } // AddFunc implement add function when pod is added in k8s diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go index 6a2442e..5525edb 100644 --- a/pkg/services/quotaburst/quotaburst_test.go +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -27,6 +27,10 @@ import ( "isula.org/rubik/test/try" ) +const ( + moduleName = "quotaburst" +) + var ( cfsBurstUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_burst_us"} cfsQuotaUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} @@ -151,7 +155,7 @@ func TestBurst_AddFunc(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := NewBurst() + conf := Burst{name: moduleName} if tt.args.burst != "" { tt.args.pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.args.burst } @@ -168,7 +172,7 @@ func TestBurst_AddFunc(t *testing.T) { func TestOther(t *testing.T) { const tcName = "TC1-test Other" t.Run(tcName, func(t *testing.T) { - got := NewBurst() + got := Burst{name: moduleName} assert.NoError(t, got.DeleteFunc(&typedef.PodInfo{})) assert.Equal(t, moduleName, got.ID()) }) @@ -216,7 +220,7 @@ func TestBurst_UpdateFunc(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := NewBurst() + conf := Burst{name: moduleName} if err := conf.UpdateFunc(tt.args.oldPod, tt.args.newPod); (err != nil) != tt.wantErr { t.Errorf("Burst.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) } @@ -250,7 +254,7 @@ func TestBurst_PreStart(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := NewBurst() + conf := Burst{name: moduleName} if err := conf.PreStart(tt.args.viewer); (err != nil) != tt.wantErr { t.Errorf("Burst.PreStart() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go index c777490..38bf4b7 100644 --- a/pkg/services/quotaturbo/data.go +++ b/pkg/services/quotaturbo/data.go @@ -143,7 +143,7 @@ func (d *NodeData) updateCPUUtils() error { return nil } -// UpdateClusterContainers synchronizes data from given containers +// UpdateClusterContainers synchronizes data from given containers func (d *NodeData) UpdateClusterContainers(conts map[string]*typedef.ContainerInfo) error { var toBeDeletedList []string for _, cont := range conts { diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index f217b76..c5a1639 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -26,20 +26,25 @@ import ( "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/services" + "isula.org/rubik/pkg/services/helper" ) -const moduleName = "quotaturbo" +type QuotaTurboFactory struct { + ObjName string +} +func (i QuotaTurboFactory) Name() string { + return "BlkioThrottleFactory" +} -func init() { - services.Register(moduleName, func() interface{} { - return NewQuotaTurbo() - }) +func (i QuotaTurboFactory) NewObj() (interface{}, error) { + return NewQuotaTurbo(i.ObjName), nil } // QuotaTurbo manages all container CPU quota data on the current node. type QuotaTurbo struct { + helper.ServiceBase + name string // NodeData including the container data, CPU usage, and QuotaTurbo configuration of the local node *NodeData // interfaces with different policies @@ -49,8 +54,9 @@ type QuotaTurbo struct { } // NewQuotaTurbo generate quota turbo objects -func NewQuotaTurbo() *QuotaTurbo { +func NewQuotaTurbo(n string) *QuotaTurbo { return &QuotaTurbo{ + name: n, NodeData: NewNodeData(), Driver: &EventDriver{}, } @@ -58,7 +64,7 @@ func NewQuotaTurbo() *QuotaTurbo { // ID returns the module name func (qt *QuotaTurbo) ID() string { - return moduleName + return qt.name } // AdjustQuota adjusts the quota of a container at a time diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index aa9d3c9..18b6269 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -28,6 +28,8 @@ import ( "isula.org/rubik/pkg/podmanager" ) +const moduleName = "quotaturbo" + // TestQuotaTurbo_Validatec test Validate function func TestQuotaTurbo_Validate(t *testing.T) { tests := []struct { @@ -337,7 +339,7 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { func TestNewQuotaTurbo(t *testing.T) { testName := "TC1-test otherv functions" t.Run(testName, func(t *testing.T) { - got := NewQuotaTurbo() + got := NewQuotaTurbo(moduleName) assert.Equal(t, moduleName, got.ID()) got.Viewer = &podmanager.PodManager{ Pods: &podmanager.PodCache{ diff --git a/pkg/services/registry.go b/pkg/services/registry.go deleted file mode 100644 index 28f2d99..0000000 --- a/pkg/services/registry.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Create: 2023-01-28 -// Description: This file defines registry for service registration - -// Package services implements service registration, discovery and management functions -package services - -import ( - "sync" -) - -type ( - // Creator creates Service objects - Creator func() interface{} - // registry is used for service registration - registry struct { - sync.RWMutex - // services is a collection of all registered service - services map[string]Creator - } -) - -// servicesRegistry is the globally unique registry -var servicesRegistry = ®istry{ - services: make(map[string]Creator, 0), -} - -// Register is used to register the service creators -func Register(name string, creator Creator) { - servicesRegistry.Lock() - servicesRegistry.services[name] = creator - servicesRegistry.Unlock() -} - -// GetServiceCreator returns the service creator based on the incoming service name -func GetServiceCreator(name string) Creator { - servicesRegistry.RLock() - creator, ok := servicesRegistry.services[name] - servicesRegistry.RUnlock() - if !ok { - return nil - } - return creator -} diff --git a/pkg/services/service.go b/pkg/services/service.go new file mode 100644 index 0000000..d0da7ce --- /dev/null +++ b/pkg/services/service.go @@ -0,0 +1,77 @@ +package services + +import ( + "context" + "fmt" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/services/helper" +) + +// PodEvent for listening to pod changes. +type PodEvent interface { + // Deal processing adding a pod. + AddPod(podInfo *typedef.PodInfo) error + // Deal processing update a pod config. + UpdatePod(old, new *typedef.PodInfo) error + // Deal processing delete a pod. + DeletePod(podInfo *typedef.PodInfo) error +} + +// Runner for background service process. +type Runner interface { + // Confirm whether it is + IsRunner() bool + // Start runner + Run(ctx context.Context) + // Stop runner + Stop() error +} + +type HandlerConfig helper.HandlerConfig + +// Service interface contains methods which must be implemented by all services. +type Service interface { + Runner + PodEvent + // ID is the name of plugin, must be unique. + ID() string + // SetConfig is an interface that invoke the HandlerConfig to obtain the corresponding configuration. + SetConfig(h HandlerConfig) error + // PreStarter is an interface for calling a collection of methods when the service is pre-started + PreStart(api.Viewer) error + // Terminator is an interface that calls a collection of methods when the service terminates + Terminate(api.Viewer) error +} + +type FeatureSpec struct { + // feature name + Name string + // Default is the default enablement state for the feature + Default bool +} + +func InitServiceComponents(specs []FeatureSpec) { + for _, spec := range specs { + if spec.Default { + if initFunc, found := serviceComponents[spec.Name]; found { + initFunc(spec.Name) + } else { + log.Errorf("init service failed, name:%v", spec.Name) + } + } else { + log.Errorf("disable feature:%v", spec.Name) + } + } +} + +func GetServiceComponent(name string) (Service, error) { + if s, err := helper.GetComponent(name); err == nil { + if si, ok := s.(Service); ok { + return si, nil + } + } + return nil, fmt.Errorf("get service failed, name:%v", name) +} -- Gitee From 7227c09602f6121b3fdab1be291562b82859f9f3 Mon Sep 17 00:00:00 2001 From: hanchao Date: Sat, 11 Mar 2023 16:21:12 +0800 Subject: [PATCH 55/73] refactor: clean code and add copyright --- pkg/services/component.go | 35 +++++++---- pkg/services/dynCache/dynCache.go | 11 +++- pkg/services/dynCache/dynCache_init.go | 4 +- pkg/services/dynCache/dynCache_init_test.go | 4 +- pkg/services/dynCache/dynCache_test.go | 4 +- pkg/services/dynCache/dynamic.go | 4 +- pkg/services/dynCache/dynamic_test.go | 4 +- pkg/services/dynCache/sync.go | 7 ++- pkg/services/dynCache/sync_test.go | 4 +- pkg/services/helper/factory.go | 24 +++++++- pkg/services/helper/service_base.go | 44 +++++++++----- pkg/services/iocost/iocost.go | 27 ++++++++- pkg/services/iocost/iocost_origin.go | 14 +++++ pkg/services/iolimit/iolimit.go | 35 +++-------- pkg/services/preemption/preemption.go | 8 ++- pkg/services/quotaburst/quotaburst.go | 3 + pkg/services/service.go | 66 ++++++++++++++------- 17 files changed, 200 insertions(+), 98 deletions(-) diff --git a/pkg/services/component.go b/pkg/services/component.go index a4bc9dd..9254cc8 100644 --- a/pkg/services/component.go +++ b/pkg/services/component.go @@ -1,8 +1,22 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to initilize all components + +// Package services package services import ( "isula.org/rubik/pkg/feature" - "isula.org/rubik/pkg/services/dynCache" + dyncache "isula.org/rubik/pkg/services/dynCache" "isula.org/rubik/pkg/services/helper" "isula.org/rubik/pkg/services/iocost" "isula.org/rubik/pkg/services/iolimit" @@ -11,6 +25,7 @@ import ( "isula.org/rubik/pkg/services/quotaturbo" ) +// ServiceComponent is the handler function of initialization. type ServiceComponent func(name string) error var ( @@ -26,31 +41,25 @@ var ( ) func initIOLimitFactory(name string) error { - helper.AddFactory(name, iolimit.IOLimitFactory{ObjName: name}) - return nil + return helper.AddFactory(name, iolimit.IOLimitFactory{ObjName: name}) } func initIOCostFactory(name string) error { - helper.AddFactory(name, iocost.IOCostFactory{ObjName: name}) - return nil + return helper.AddFactory(name, iocost.IOCostFactory{ObjName: name}) } func initDynCacheFactory(name string) error { - helper.AddFactory(name, dynCache.DynCacheFactory{ObjName: name}) - return nil + return helper.AddFactory(name, dyncache.DynCacheFactory{ObjName: name}) } func initQuotaTurboFactory(name string) error { - helper.AddFactory(name, quotaturbo.QuotaTurboFactory{ObjName: name}) - return nil + return helper.AddFactory(name, quotaturbo.QuotaTurboFactory{ObjName: name}) } func initQuotaBurstFactory(name string) error { - helper.AddFactory(name, quotaburst.BurstFactory{ObjName: name}) - return nil + return helper.AddFactory(name, quotaburst.BurstFactory{ObjName: name}) } func initPreemptionFactory(name string) error { - helper.AddFactory(name, preemption.PreemptionFactory{ObjName: name}) - return nil + return helper.AddFactory(name, preemption.PreemptionFactory{ObjName: name}) } diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index 4bbd00c..5a4fda7 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is cache limit service -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "context" @@ -112,14 +112,17 @@ type Attr struct { MinMiss int } +// DynCacheFactory is the factory os dyncache. type DynCacheFactory struct { ObjName string } +// Name to get the dyncache factory name. func (i DynCacheFactory) Name() string { return "DynCacheFactory" } +// NewObj to create object of dyncache. func (i DynCacheFactory) NewObj() (interface{}, error) { return NewDynCache(i.ObjName), nil } @@ -188,7 +191,9 @@ func (c *DynCache) Validate() error { if c.Config.PerfDuration < minPerfDur || c.Config.PerfDuration > maxPerfDur { return fmt.Errorf("perf duration %d out of range [%d,%d]", c.Config.PerfDuration, minPerfDur, maxPerfDur) } - for _, per := range []int{c.Config.L3Percent.Low, c.Config.L3Percent.Mid, c.Config.L3Percent.High, c.Config.MemBandPercent.Low, + for _, per := range []int{ + c.Config.L3Percent.Low, c.Config.L3Percent.Mid, + c.Config.L3Percent.High, c.Config.MemBandPercent.Low, c.Config.MemBandPercent.Mid, c.Config.MemBandPercent.High} { if per < minPercent || per > maxPercent { return fmt.Errorf("cache limit percentage %d out of range [%d,%d]", per, minPercent, maxPercent) diff --git a/pkg/services/dynCache/dynCache_init.go b/pkg/services/dynCache/dynCache_init.go index 198be96..0f4d36b 100644 --- a/pkg/services/dynCache/dynCache_init.go +++ b/pkg/services/dynCache/dynCache_init.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file will init cache limit directories before services running -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "fmt" diff --git a/pkg/services/dynCache/dynCache_init_test.go b/pkg/services/dynCache/dynCache_init_test.go index fe07a01..779de6f 100644 --- a/pkg/services/dynCache/dynCache_init_test.go +++ b/pkg/services/dynCache/dynCache_init_test.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file will init cache limit directories before services running -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "fmt" diff --git a/pkg/services/dynCache/dynCache_test.go b/pkg/services/dynCache/dynCache_test.go index 3c57ec4..35874b6 100644 --- a/pkg/services/dynCache/dynCache_test.go +++ b/pkg/services/dynCache/dynCache_test.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is testcase for cache limit service -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "reflect" diff --git a/pkg/services/dynCache/dynamic.go b/pkg/services/dynCache/dynamic.go index 147efde..e73601a 100644 --- a/pkg/services/dynCache/dynamic.go +++ b/pkg/services/dynCache/dynamic.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is used for dynamic cache limit level setting -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "fmt" diff --git a/pkg/services/dynCache/dynamic_test.go b/pkg/services/dynCache/dynamic_test.go index 5bdca11..3093ae3 100644 --- a/pkg/services/dynCache/dynamic_test.go +++ b/pkg/services/dynCache/dynamic_test.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is testcases for dynamic cache limit level setting -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "fmt" diff --git a/pkg/services/dynCache/sync.go b/pkg/services/dynCache/sync.go index 46262c5..4855438 100644 --- a/pkg/services/dynCache/sync.go +++ b/pkg/services/dynCache/sync.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is used for cache limit sync setting -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "fmt" @@ -78,7 +78,8 @@ func (c *DynCache) writeTasksToResctrl(pod *typedef.PodInfo) error { return nil } - resctrlTaskFile := filepath.Join(c.Config.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks") + resctrlTaskFile := filepath.Join(c.Config.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks") for _, task := range taskList { if err := util.WriteFile(resctrlTaskFile, task); err != nil { if strings.Contains(err.Error(), "no such process") { diff --git a/pkg/services/dynCache/sync_test.go b/pkg/services/dynCache/sync_test.go index b6a500a..0f43a89 100644 --- a/pkg/services/dynCache/sync_test.go +++ b/pkg/services/dynCache/sync_test.go @@ -11,8 +11,8 @@ // Create: 2023-02-21 // Description: This file is testcase for cache limit sync function -// Package cachelimit is the service used for cache limit setting -package dynCache +// Package dyncache is the service used for cache limit setting +package dyncache import ( "path/filepath" diff --git a/pkg/services/helper/factory.go b/pkg/services/helper/factory.go index 5d50e24..4134bff 100644 --- a/pkg/services/helper/factory.go +++ b/pkg/services/helper/factory.go @@ -1,10 +1,26 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to implement the factory + +// Package helper package helper import ( "errors" + "fmt" "sync" ) +// ServiceFactory is to define Service Factory type ServiceFactory interface { Name() string NewObj() (interface{}, error) @@ -15,12 +31,18 @@ var ( serviceFactories = map[string]ServiceFactory{} ) -func AddFactory(name string, factory ServiceFactory) { +// AddFactory is to add a service factory +func AddFactory(name string, factory ServiceFactory) error { rwlock.Lock() defer rwlock.Unlock() + if _, found := serviceFactories[name]; found { + return fmt.Errorf("factory is already exists") + } serviceFactories[name] = factory + return nil } +// GetComponent is to get the interface of object. func GetComponent(name string) (interface{}, error) { rwlock.RLock() defer rwlock.RUnlock() diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go index 35aa984..3d07890 100644 --- a/pkg/services/helper/service_base.go +++ b/pkg/services/helper/service_base.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is the base of service. + +// Package helper package helper import ( @@ -8,12 +22,15 @@ import ( "isula.org/rubik/pkg/core/typedef" ) +// ServiceBase is the basic class of a service. type ServiceBase struct{} -type HandlerConfig func(configName string, d interface{}) error -// ID is the name of plugin, must be unique. -func (s *ServiceBase) ID() string { - panic("this interface must be implemented.") +// ConfigHandler is that obtains the configured callback function. +type ConfigHandler func(configName string, d interface{}) error + +// SetConfig is an interface that invoke the ConfigHandler to obtain the corresponding configuration. +func (s *ServiceBase) SetConfig(ConfigHandler) error { + return nil } // PreStarter is an interface for calling a collection of methods when the service is pre-started @@ -26,31 +43,30 @@ func (s *ServiceBase) Terminate(api.Viewer) error { return nil } -// Confirm whether it is +// IsRunner to Confirm whether it is a runner func (s *ServiceBase) IsRunner() bool { return false } -// Start runner -func (s *ServiceBase) Run(ctx context.Context) {} +// Run to start runner +func (s *ServiceBase) Run(context.Context) {} -// Stop runner +// Stop to stop runner func (s *ServiceBase) Stop() error { return fmt.Errorf("i am not runner") } -func (s *ServiceBase) AddPod(podInfo *typedef.PodInfo) error { +// AddPod to deal the event of adding a pod. +func (s *ServiceBase) AddPod(*typedef.PodInfo) error { return nil } +// UpdatePod to deal the pod update event. func (S *ServiceBase) UpdatePod(old, new *typedef.PodInfo) error { return nil } -func (s *ServiceBase) DeletePod(podInfo *typedef.PodInfo) error { - return nil -} - -func (s *ServiceBase) SetConfig(h HandlerConfig) error { +// DeletePod to deal the pod deletion event. +func (s *ServiceBase) DeletePod(*typedef.PodInfo) error { return nil } diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index ac9116c..959e3fe 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to implement iocost + +// Package iocost package iocost import ( @@ -75,6 +89,17 @@ func (i IOCostFactory) NewObj() (interface{}, error) { // IOCostSupport tell if the os support iocost. func IOCostSupport() bool { + cmdLine, err := os.ReadFile("/proc/cmdline") + if err != nil { + log.Warnf("get /pro/cmdline error") + return false + } + + if !strings.Contains(string(cmdLine), "cgroup1_writeback") { + log.Warnf("this machine not support writeback, please add 'cgroup1_writeback' to cmdline") + return false + } + qosFile := cgroup.AbsoluteCgroupPath(blkcgRootDir, iocostQosFile) modelFile := cgroup.AbsoluteCgroupPath(blkcgRootDir, iocostModelFile) return util.PathExist(qosFile) && util.PathExist(modelFile) @@ -85,7 +110,7 @@ func (io *IOCost) ID() string { return io.name } -func (io *IOCost) SetConfig(f helper.HandlerConfig) error { +func (io *IOCost) SetConfig(f helper.ConfigHandler) error { var nodeConfigs []NodeConfig var nodeConfig *NodeConfig if err := f(io.name, nodeConfigs); err != nil { diff --git a/pkg/services/iocost/iocost_origin.go b/pkg/services/iocost/iocost_origin.go index 3597967..e5c56dd 100644 --- a/pkg/services/iocost/iocost_origin.go +++ b/pkg/services/iocost/iocost_origin.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to implement system iocost interface + +// Package iocost package iocost import ( diff --git a/pkg/services/iolimit/iolimit.go b/pkg/services/iolimit/iolimit.go index 9deb755..7dd7f13 100644 --- a/pkg/services/iolimit/iolimit.go +++ b/pkg/services/iolimit/iolimit.go @@ -1,18 +1,16 @@ package iolimit import ( - "isula.org/rubik/pkg/api" - "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/services/helper" ) -// DeviceConfig defines blkio device configurations +// DeviceConfig defines blkio device configurations. type DeviceConfig struct { DeviceName string `json:"device,omitempty"` DeviceValue string `json:"value,omitempty"` } +// IOLimitAnnoConfig defines the annotation config of iolimit. type IOLimitAnnoConfig struct { DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` @@ -20,47 +18,28 @@ type IOLimitAnnoConfig struct { DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` } +// IOLimit is the class of IOLimit. type IOLimit struct { helper.ServiceBase name string } +// IOLimitFactory is the factory of IOLimit. type IOLimitFactory struct { ObjName string } +// Name to get the IOLimit factory name. func (i IOLimitFactory) Name() string { return "IOLimitFactory" } +// NewObj to create object of IOLimit. func (i IOLimitFactory) NewObj() (interface{}, error) { return &IOLimit{name: i.ObjName}, nil } -// ========================= - +// ID to get the name of IOLimit. func (i *IOLimit) ID() string { return i.name } - -func (i *IOLimit) PreStart(viewer api.Viewer) error { - log.Infof("ioLimit prestart") - return nil -} - -func (i *IOLimit) Terminate(viewer api.Viewer) error { - log.Infof("ioLimit Terminate") - return nil -} - -func (i *IOLimit) AddFunc(podInfo *typedef.PodInfo) error { - return nil -} - -func (i *IOLimit) UpdateFunc(old, new *typedef.PodInfo) error { - return nil -} - -func (i *IOLimit) DeleteFunc(podInfo *typedef.PodInfo) error { - return nil -} diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index 874e921..81de0ba 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -38,19 +38,22 @@ type Preemption struct { config PreemptionConfig } -// Config contains sub-system that need to set qos level +// PreemptionConfig define which resources need to use the preemption type PreemptionConfig struct { Resource []string `json:"resource,omitempty"` } +// PreemptionFactory is the factory os Preemption. type PreemptionFactory struct { ObjName string } +// Name to get the Preemption factory name. func (i PreemptionFactory) Name() string { return "PreemptionFactory" } +// NewObj to create object of Preemption. func (i PreemptionFactory) NewObj() (interface{}, error) { return &Preemption{name: i.ObjName}, nil } @@ -60,7 +63,8 @@ func (q *Preemption) ID() string { return q.name } -func (q *Preemption) SetConfig(f helper.HandlerConfig) error { +// SetConfig to config Preemption configure +func (q *Preemption) SetConfig(f helper.ConfigHandler) error { var c PreemptionConfig if err := f(q.name, c); err != nil { return err diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index b5c9946..7d2d45e 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -37,14 +37,17 @@ type Burst struct { name string } +// BurstFactory is the factory os Burst. type BurstFactory struct { ObjName string } +// Name to get the Burst factory name. func (i BurstFactory) Name() string { return "BurstFactory" } +// NewObj to create object of Burst. func (i BurstFactory) NewObj() (interface{}, error) { return &Burst{name: i.ObjName}, nil } diff --git a/pkg/services/service.go b/pkg/services/service.go index d0da7ce..b18f607 100644 --- a/pkg/services/service.go +++ b/pkg/services/service.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is the Interface set of services + +// Package services package services import ( @@ -13,39 +27,39 @@ import ( // PodEvent for listening to pod changes. type PodEvent interface { // Deal processing adding a pod. - AddPod(podInfo *typedef.PodInfo) error + AddPod(*typedef.PodInfo) error // Deal processing update a pod config. UpdatePod(old, new *typedef.PodInfo) error // Deal processing delete a pod. - DeletePod(podInfo *typedef.PodInfo) error + DeletePod(*typedef.PodInfo) error } // Runner for background service process. type Runner interface { - // Confirm whether it is + // IsRunner for Confirm whether it is IsRunner() bool // Start runner - Run(ctx context.Context) + Run(context.Context) // Stop runner Stop() error } -type HandlerConfig helper.HandlerConfig - // Service interface contains methods which must be implemented by all services. type Service interface { Runner PodEvent // ID is the name of plugin, must be unique. ID() string - // SetConfig is an interface that invoke the HandlerConfig to obtain the corresponding configuration. - SetConfig(h HandlerConfig) error + // SetConfig is an interface that invoke the ConfigHandler to obtain the corresponding configuration. + SetConfig(helper.ConfigHandler) error // PreStarter is an interface for calling a collection of methods when the service is pre-started PreStart(api.Viewer) error // Terminator is an interface that calls a collection of methods when the service terminates + // it will stop runner and clear configuration Terminate(api.Viewer) error } +// FeatureSpec to defines the feature name and whether the feature is enabled. type FeatureSpec struct { // feature name Name string @@ -53,25 +67,35 @@ type FeatureSpec struct { Default bool } +// InitServiceComponents for initilize serverice components func InitServiceComponents(specs []FeatureSpec) { for _, spec := range specs { - if spec.Default { - if initFunc, found := serviceComponents[spec.Name]; found { - initFunc(spec.Name) - } else { - log.Errorf("init service failed, name:%v", spec.Name) - } - } else { - log.Errorf("disable feature:%v", spec.Name) + if !spec.Default { + log.Warnf("feature is disabled by default:%v", spec.Name) + continue + } + + initFunc, found := serviceComponents[spec.Name] + if !found { + log.Errorf("init service failed, name:%v", spec.Name) + continue + } + + if err := initFunc(spec.Name); err != nil { + log.Warnf("init component failed, name:%v,error:%v", spec.Name, err) } } } +// GetServiceComponent to get the component service interface. func GetServiceComponent(name string) (Service, error) { - if s, err := helper.GetComponent(name); err == nil { - if si, ok := s.(Service); ok { - return si, nil - } + si, err := helper.GetComponent(name) + if err != nil { + return nil, fmt.Errorf("get service failed, name:%v,err:%v", name, err) + } + srv, ok := si.(Service) + if !ok || srv == nil { + return nil, fmt.Errorf("failed to convert the type,name:%v", name) } - return nil, fmt.Errorf("get service failed, name:%v", name) + return srv, nil } -- Gitee From 966fa6eaaa53db76e73bfdb23eeb3d1a9fa364da Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 11 Mar 2023 10:57:09 +0800 Subject: [PATCH 56/73] optimize: adapt quotaturbo service Use the quotaTurbo underlying library to realize the quota adjustment function, and the service pays attention to the quota verification (reliability guarantee) when starting and stopping, and the logic of adding and deleting containers Signed-off-by: vegbir --- pkg/core/typedef/cgroup/common.go | 5 + pkg/lib/cpu/quotaturbo/statusstore.go | 9 + pkg/rubik/servicemanager.go | 15 +- pkg/services/quotaturbo/cpu.go | 111 ---- pkg/services/quotaturbo/cpu_test.go | 57 -- pkg/services/quotaturbo/cpuquota.go | 207 ------- pkg/services/quotaturbo/cpuquota_test.go | 236 -------- pkg/services/quotaturbo/data.go | 225 -------- pkg/services/quotaturbo/data_test.go | 232 -------- pkg/services/quotaturbo/driverevent.go | 197 ------- pkg/services/quotaturbo/driverevent_test.go | 593 -------------------- pkg/services/quotaturbo/quotaturbo.go | 135 ++++- pkg/services/quotaturbo/quotaturbo_test.go | 385 ------------- 13 files changed, 133 insertions(+), 2274 deletions(-) delete mode 100644 pkg/services/quotaturbo/cpu.go delete mode 100644 pkg/services/quotaturbo/cpu_test.go delete mode 100644 pkg/services/quotaturbo/cpuquota.go delete mode 100644 pkg/services/quotaturbo/cpuquota_test.go delete mode 100644 pkg/services/quotaturbo/data.go delete mode 100644 pkg/services/quotaturbo/data_test.go delete mode 100644 pkg/services/quotaturbo/driverevent.go delete mode 100644 pkg/services/quotaturbo/driverevent_test.go delete mode 100644 pkg/services/quotaturbo/quotaturbo_test.go diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index b750615..396a67a 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -60,6 +60,11 @@ func InitMountDir(arg string) { rootDir = arg } +// GetMountDir returns the mount point path of the cgroup +func GetMountDir() string { + return rootDir +} + type ( // Key uniquely determines the cgroup value of the container or pod Key struct { diff --git a/pkg/lib/cpu/quotaturbo/statusstore.go b/pkg/lib/cpu/quotaturbo/statusstore.go index 0b7e067..82422a6 100644 --- a/pkg/lib/cpu/quotaturbo/statusstore.go +++ b/pkg/lib/cpu/quotaturbo/statusstore.go @@ -106,6 +106,15 @@ func (store *StatusStore) RemoveCgroup(cgroupPath string) error { return safeDel(cgroupPath) } +// GetAllCgroup returns all cgroup paths that are adjusting quota +func (store *StatusStore) GetAllCgroup() []string { + var res = make([]string, 0) + for _, cq := range store.cpuQuotas { + res = append(res, cq.Path) + } + return res +} + // getLastCPUUtil obtain the latest cpu utilization func (store *StatusStore) getLastCPUUtil() float64 { if len(store.cpuUtils) == 0 { diff --git a/pkg/rubik/servicemanager.go b/pkg/rubik/servicemanager.go index a1d46d3..35aeff3 100644 --- a/pkg/rubik/servicemanager.go +++ b/pkg/rubik/servicemanager.go @@ -132,14 +132,23 @@ func (manager *ServiceManager) terminatingRunningServices() error { func (manager *ServiceManager) Setup(v api.Viewer) error { // only when viewer is prepared if v == nil { - return nil + return fmt.Errorf("viewer should not be empty") } - - for _, s := range manager.RunningServices { + manager.Viewer = v + manager.Lock() + var unavailable []string + for name, s := range manager.RunningServices { if err := s.PreStart(manager.Viewer); err != nil { log.Errorf("PreStart failed:%v", err) + unavailable = append(unavailable, name) } } + + for _, name := range unavailable { + delete(manager.RunningServices, name) + log.Infof("service %v is unavailable", name) + } + manager.Unlock() return nil } diff --git a/pkg/services/quotaturbo/cpu.go b/pkg/services/quotaturbo/cpu.go deleted file mode 100644 index 8b2d3fc..0000000 --- a/pkg/services/quotaturbo/cpu.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-08 -// Description: This file is used for computing cpu utilization - -// Package quotaturbo is for Quota Turbo -package quotaturbo - -import ( - "fmt" - "io/ioutil" - "math" - "strings" - - "isula.org/rubik/pkg/common/util" -) - -const ( - maximumUtilization float64 = 100 - minimumUtilization float64 = 0 -) - -// ProcStat store /proc/stat data -type ProcStat struct { - name string - user float64 - nice float64 - system float64 - idle float64 - iowait float64 - irq float64 - softirq float64 - steal float64 - guest float64 - guestNice float64 - total float64 - busy float64 -} - -// getProcStat create a proc stat object -func getProcStat() (ProcStat, error) { - const ( - procStatFilePath = "/proc/stat" - nameLineNum = 0 - userIndex = 0 - niceIndex = 1 - systemIndex = 2 - idleIndex = 3 - iowaitIndex = 4 - irqIndex = 5 - softirqIndex = 6 - stealIndex = 7 - guestIndex = 8 - guestNiceIndex = 9 - statsFieldsCount = 10 - supportFieldNumber = 11 - ) - data, err := ioutil.ReadFile(procStatFilePath) - if err != nil { - return ProcStat{}, err - } - // format of the first line of the file /proc/stat : - // name user nice system idle iowait irq softirq steal guest guest_nice - line := strings.Fields(strings.Split(string(data), "\n")[0]) - if len(line) < supportFieldNumber { - return ProcStat{}, fmt.Errorf("too few fields and check the kernel version") - } - var fields [statsFieldsCount]float64 - for i := 0; i < statsFieldsCount; i++ { - fields[i], err = util.ParseFloat64(line[i+1]) - if err != nil { - return ProcStat{}, err - } - } - ps := ProcStat{ - name: line[nameLineNum], - user: fields[userIndex], - nice: fields[niceIndex], - system: fields[systemIndex], - idle: fields[idleIndex], - iowait: fields[iowaitIndex], - irq: fields[irqIndex], - softirq: fields[softirqIndex], - steal: fields[stealIndex], - guest: fields[guestIndex], - guestNice: fields[guestNiceIndex], - } - ps.busy = ps.user + ps.system + ps.nice + ps.iowait + ps.irq + ps.softirq + ps.steal - ps.total = ps.busy + ps.idle - return ps, nil -} - -// calculateUtils calculate the CPU utilization rate based on the two interval /proc/stat -func calculateUtils(t1, t2 ProcStat) float64 { - if t2.busy <= t1.busy { - return minimumUtilization - } - if t2.total <= t1.total { - return maximumUtilization - } - return math.Min(maximumUtilization, - math.Max(minimumUtilization, util.Div(t2.busy-t1.busy, t2.total-t1.total)*maximumUtilization)) -} diff --git a/pkg/services/quotaturbo/cpu_test.go b/pkg/services/quotaturbo/cpu_test.go deleted file mode 100644 index ead2d05..0000000 --- a/pkg/services/quotaturbo/cpu_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-20 -// Description: This file is used for testing cpu.go - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// TestCalculateUtils tests calculateUtils -func TestCalculateUtils(t *testing.T) { - var ( - n1 float64 = 1 - n2 float64 = 2 - n3 float64 = 3 - n4 float64 = 4 - ) - - var ( - t1 = ProcStat{ - total: n2, - busy: n1, - } - t2 = ProcStat{ - total: n4, - busy: n2, - } - t3 = ProcStat{ - total: n3, - busy: n3, - } - ) - // normal return result - const ( - util float64 = 50 - minimumUtilization float64 = 0 - maximumUtilization float64 = 100 - ) - assert.Equal(t, util, calculateUtils(t1, t2)) - // busy errors - assert.Equal(t, minimumUtilization, calculateUtils(t2, t1)) - // total errors - assert.Equal(t, maximumUtilization, calculateUtils(t2, t3)) -} diff --git a/pkg/services/quotaturbo/cpuquota.go b/pkg/services/quotaturbo/cpuquota.go deleted file mode 100644 index 86c837a..0000000 --- a/pkg/services/quotaturbo/cpuquota.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-20 -// Description: cpu container cpu quota data and methods - -// Package quotaturbo is for Quota Turbo -package quotaturbo - -import ( - "fmt" - "path" - "time" - - "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/core/typedef/cgroup" -) - -const ( - // numberOfRestrictedCycles is the number of periods in which the quota limits the CPU usage. - numberOfRestrictedCycles = 60 - // The default value of the cfs_period_us file is 100ms - defaultCFSPeriodUs int64 = 100000 -) - -var ( - cpuPeriodKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} - cpuQuotaKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} - cpuAcctUsageKey = &cgroup.Key{SubSys: "cpuacct", FileName: "cpuacct.usage"} - cpuStatKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.stat"} -) - -// cpuUsage cpu time used by the container at timestamp -type cpuUsage struct { - timestamp int64 - usage int64 -} - -// CPUQuota stores the CPU quota information of a single container. -type CPUQuota struct { - // basic container information - *typedef.ContainerInfo - // current throttling data for the container - curThrottle *cgroup.CPUStat - // previous throttling data for container - preThrottle *cgroup.CPUStat - // container cfs_period_us - period int64 - // current cpu quota of the container - curQuota int64 - // cpu quota of the container in the next period - nextQuota int64 - // the delta of the cpu quota to be adjusted based on the decision. - quotaDelta float64 - // the upper limit of the container cpu quota - heightLimit float64 - // maximum quota that can be used by a container in the next period, - // calculated based on the total usage in the past N-1 cycles - maxQuotaNextPeriod float64 - // container cpu usage sequence - cpuUsages []cpuUsage -} - -// NewCPUQuota create a cpu quota object -func NewCPUQuota(ci *typedef.ContainerInfo) (*CPUQuota, error) { - defaultQuota := ci.LimitResources[typedef.ResourceCPU] * float64(defaultCFSPeriodUs) - cq := &CPUQuota{ - ContainerInfo: ci, - cpuUsages: make([]cpuUsage, 0), - quotaDelta: 0, - curThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, - preThrottle: &cgroup.CPUStat{NrThrottled: 0, ThrottledTime: 0}, - period: defaultCFSPeriodUs, - curQuota: int64(defaultQuota), - nextQuota: int64(defaultQuota), - heightLimit: defaultQuota, - maxQuotaNextPeriod: defaultQuota, - } - - if err := cq.updatePeriod(); err != nil { - return cq, err - } - - if err := cq.updateThrottle(); err != nil { - return cq, err - } - // The throttle data before and after the initialization is the same. - cq.preThrottle = cq.curThrottle - - if err := cq.updateQuota(); err != nil { - return cq, err - } - - if err := cq.updateUsage(); err != nil { - return cq, err - } - return cq, nil -} - -func (c *CPUQuota) updatePeriod() error { - us, err := c.ContainerInfo.GetCgroupAttr(cpuPeriodKey).Int64() - // If an error occurs, the period remains unchanged or the default value is used. - if err != nil { - return err - } - c.period = us - return nil -} - -func (c *CPUQuota) updateThrottle() error { - // update suppression times and duration - // if data cannot be obtained from cpu.stat, the value remains unchanged. - c.preThrottle = c.curThrottle - cs, err := c.ContainerInfo.GetCgroupAttr(cpuStatKey).CPUStat() - if err != nil { - return err - } - c.curThrottle = cs - return nil -} - -func (c *CPUQuota) updateQuota() error { - c.quotaDelta = 0 - curQuota, err := c.ContainerInfo.GetCgroupAttr(cpuQuotaKey).Int64() - if err != nil { - return err - } - c.curQuota = curQuota - return nil -} - -func (c *CPUQuota) updateUsage() error { - latest, err := c.ContainerInfo.GetCgroupAttr(cpuAcctUsageKey).Int64() - if err != nil { - return err - } - c.cpuUsages = append(c.cpuUsages, cpuUsage{timestamp: time.Now().UnixNano(), usage: latest}) - // ensure that the CPU usage of the container does not exceed the upper limit. - if len(c.cpuUsages) >= numberOfRestrictedCycles { - c.cpuUsages = c.cpuUsages[1:] - } - return nil -} - -func (c *CPUQuota) writePodQuota(delta int64) error { - pod := &typedef.PodInfo{ - CgroupPath: path.Dir(c.CgroupPath), - } - podQuota, err := pod.GetCgroupAttr(cpuQuotaKey).Int64() - if err == nil && podQuota == -1 { - return nil - } - podQuota += delta - return pod.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(podQuota)) -} - -func (c *CPUQuota) writeContainerQuota() error { - return c.ContainerInfo.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(c.curQuota)) -} - -// WriteQuota use to modify quota for container -func (c *CPUQuota) WriteQuota() error { - delta := c.nextQuota - c.curQuota - tmp := c.curQuota - c.curQuota = c.nextQuota - if delta < 0 { - // update container data first - if err := c.writeContainerQuota(); err != nil { - c.curQuota = tmp - return fmt.Errorf("fail to write container's quota for container %v: %v", c.ID, err) - } - // then update the pod data - if err := c.writePodQuota(delta); err != nil { - // recover - c.curQuota = tmp - if recoverErr := c.writeContainerQuota(); recoverErr != nil { - log.Errorf("fail to recover contaienr's quota for container %v: %v", c.ID, recoverErr) - } - return fmt.Errorf("fail to write pod's quota for container %v: %v", c.ID, err) - } - } else if delta > 0 { - // update pod data first - if err := c.writePodQuota(delta); err != nil { - c.curQuota = tmp - return fmt.Errorf("fail to write pod's quota for container %v: %v", c.ID, err) - } - // then update the container data - if err := c.writeContainerQuota(); err != nil { - // recover - c.curQuota = tmp - if recoverErr := c.writePodQuota(-delta); recoverErr != nil { - log.Errorf("fail to recover pod's quota for container %v: %v", c.ID, recoverErr) - } - return fmt.Errorf("fail to write container's quota for container %v: %v", c.ID, err) - } - } - return nil -} diff --git a/pkg/services/quotaturbo/cpuquota_test.go b/pkg/services/quotaturbo/cpuquota_test.go deleted file mode 100644 index 847b1d6..0000000 --- a/pkg/services/quotaturbo/cpuquota_test.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-20 -// Description: This file is used for test cpu_quota - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "path" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/core/typedef/cgroup" - "isula.org/rubik/test/try" -) - -// TestSaveQuota tests SaveQuota of CPUQuota -func TestSaveQuota(t *testing.T) { - const ( - largerQuota = "200000" - largerQuotaVal int64 = 200000 - smallerQuota = "100000" - smallerQuotaVal int64 = 100000 - unlimitedQuota = "-1" - unlimitedQuotaVal int64 = -1 - periodUs = "100000" - cpuPeriodFile = "cpu.cfs_period_us" - cpuQuotaFile = "cpu.cfs_quota_us" - ) - cgroup.InitMountDir(constant.TmpTestDir) - var ( - cq = &CPUQuota{ - ContainerInfo: &typedef.ContainerInfo{ - Name: "Foo", - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", - }, - } - contPath = cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath, "") - podPeriodPath = filepath.Join(filepath.Dir(contPath), cpuPeriodFile) - podQuotaPath = filepath.Join(filepath.Dir(contPath), cpuQuotaFile) - contPeriodPath = filepath.Join(contPath, cpuPeriodFile) - contQuotaPath = filepath.Join(contPath, cpuQuotaFile) - ) - - try.RemoveAll(constant.TmpTestDir) - defer try.RemoveAll(constant.TmpTestDir) - - assertValue := func(t *testing.T, paths []string, value string) { - for _, p := range paths { - data, err := util.ReadFile(p) - assert.NoError(t, err) - assert.Equal(t, value, strings.TrimSpace(string(data))) - } - } - - // case1: Only one pod or container exists at a time - func() { - try.MkdirAll(contPath, constant.DefaultDirMode) - defer try.RemoveAll(path.Dir(contPath)) - // None of the paths exist - cq.nextQuota = largerQuotaVal - cq.curQuota = smallerQuotaVal - assert.Error(t, cq.WriteQuota()) - cq.nextQuota = smallerQuotaVal - cq.curQuota = largerQuotaVal - assert.Error(t, cq.WriteQuota()) - - // only Pod path existed - cq.nextQuota = largerQuotaVal - cq.curQuota = smallerQuotaVal - try.WriteFile(podQuotaPath, unlimitedQuota) - try.WriteFile(podPeriodPath, periodUs) - try.RemoveAll(contQuotaPath) - try.RemoveAll(contPeriodPath) - assert.Error(t, cq.WriteQuota()) - - // only container path existed - cq.nextQuota = smallerQuotaVal - cq.curQuota = largerQuotaVal - try.RemoveAll(podQuotaPath) - try.RemoveAll(podPeriodPath) - try.WriteFile(contQuotaPath, largerQuota) - try.WriteFile(contPeriodPath, periodUs) - assert.Error(t, cq.WriteQuota()) - - }() - - // case2: success - func() { - try.MkdirAll(contPath, constant.DefaultDirMode) - defer try.RemoveAll(path.Dir(contPath)) - - try.WriteFile(podQuotaPath, smallerQuota) - try.WriteFile(podPeriodPath, periodUs) - try.WriteFile(contQuotaPath, smallerQuota) - try.WriteFile(contPeriodPath, periodUs) - - // delta > 0 - cq.nextQuota = largerQuotaVal - cq.curQuota = smallerQuotaVal - assert.NoError(t, cq.WriteQuota()) - assertValue(t, []string{podQuotaPath, contQuotaPath}, largerQuota) - // delta < 0 - cq.nextQuota = smallerQuotaVal - cq.curQuota = largerQuotaVal - assert.NoError(t, cq.WriteQuota()) - assertValue(t, []string{podQuotaPath, contQuotaPath}, smallerQuota) - }() - - cgroup.InitMountDir(constant.DefaultCgroupRoot) -} - -// TestNewCPUQuota tests NewCPUQuota -func TestNewCPUQuota(t *testing.T) { - const ( - cpuPeriodFile = "cpu.cfs_period_us" - cpuQuotaFile = "cpu.cfs_quota_us" - cpuUsageFile = "cpuacct.usage" - cpuStatFile = "cpu.stat" - validStat = `nr_periods 1 - nr_throttled 1 - throttled_time 1 - ` - throttleTime int64 = 1 - quota = "200000" - quotaValue int64 = 200000 - period = "100000" - periodValue int64 = 100000 - usage = "1234567" - usageValue int64 = 1234567 - ) - cgroup.InitMountDir(constant.TmpTestDir) - - var ( - ci = &typedef.ContainerInfo{ - Name: "Foo", - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", - } - contPath = cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") - contPeriodPath = filepath.Join(contPath, cpuPeriodFile) - contQuotaPath = filepath.Join(contPath, cpuQuotaFile) - contUsagePath = cgroup.AbsoluteCgroupPath("cpuacct", ci.CgroupPath, cpuUsageFile) - contStatPath = filepath.Join(contPath, cpuStatFile) - ) - - try.RemoveAll(constant.TmpTestDir) - try.MkdirAll(contPath, constant.DefaultDirMode) - try.MkdirAll(path.Dir(contUsagePath), constant.DefaultDirMode) - defer try.RemoveAll(constant.TmpTestDir) - - // absent of period file - try.RemoveAll(contPeriodPath) - _, err := NewCPUQuota(ci) - assert.Error(t, err, "should lacking of period file") - try.WriteFile(contPeriodPath, period) - - // absent of throttle file - try.RemoveAll(contStatPath) - _, err = NewCPUQuota(ci) - assert.Error(t, err, "should lacking of throttle file") - try.WriteFile(contStatPath, validStat) - - // absent of quota file - try.RemoveAll(contQuotaPath) - _, err = NewCPUQuota(ci) - assert.Error(t, err, "should lacking of quota file") - try.WriteFile(contQuotaPath, quota) - - // absent of usage file - try.RemoveAll(contUsagePath) - _, err = NewCPUQuota(ci) - assert.Error(t, err, "should lacking of usage file") - try.WriteFile(contUsagePath, usage) - - cq, err := NewCPUQuota(ci) - assert.NoError(t, err) - assert.Equal(t, usageValue, cq.cpuUsages[0].usage) - assert.Equal(t, quotaValue, cq.curQuota) - assert.Equal(t, periodValue, cq.period) - - cu := make([]cpuUsage, numberOfRestrictedCycles) - for i := 0; i < numberOfRestrictedCycles; i++ { - cu[i] = cpuUsage{} - } - cq.cpuUsages = cu - assert.NoError(t, cq.updateUsage()) - cgroup.InitMountDir(constant.DefaultCgroupRoot) -} - -var containerInfos = []*typedef.ContainerInfo{ - { - Name: "Foo", - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", - LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, - RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, - }, - { - Name: "Bar", - ID: "testCon2", - CgroupPath: "kubepods/testPod2/testCon2", - LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 3}, - RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 3}, - }, - { - Name: "Biu", - ID: "testCon3", - CgroupPath: "kubepods/testPod3/testCon3", - LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, - RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 2}, - }, - { - Name: "Pah", - ID: "testCon4", - CgroupPath: "kubepods/testPod4/testCon4", - LimitResources: typedef.ResourceMap{typedef.ResourceCPU: 0}, - RequestResources: typedef.ResourceMap{typedef.ResourceCPU: 0}, - }, -} diff --git a/pkg/services/quotaturbo/data.go b/pkg/services/quotaturbo/data.go deleted file mode 100644 index 38bf4b7..0000000 --- a/pkg/services/quotaturbo/data.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-20 -// Description: QuotaTurbo driver interface - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "fmt" - "runtime" - "sync" - "time" - - "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/core/typedef/cgroup" -) - -const ( - defaultHightWaterMark = 60 - defaultAlarmWaterMark = 80 - defaultQuotaTurboSyncInterval = 100 -) - -// cpuUtil is used to store the cpu usage at a specific time -type cpuUtil struct { - timestamp int64 - util float64 -} - -// Config defines configuration of QuotaTurbo -type Config struct { - HighWaterMark int `json:"highWaterMark,omitempty"` - AlarmWaterMark int `json:"alarmWaterMark,omitempty"` - SyncInterval int `json:"syncInterval,omitempty"` -} - -// NodeData is the information of node/containers obtained for quotaTurbo -type NodeData struct { - // configuration of the QuotaTurbo - *Config - // ensuring Concurrent Sequential Consistency - sync.RWMutex - // map between container IDs and container CPU quota - containers map[string]*CPUQuota - // cpu utilization sequence for N consecutive cycles - cpuUtils []cpuUtil - // /proc/stat of the previous period - lastProcStat ProcStat -} - -// NewNodeData returns a pointer to NodeData -func NewNodeData() *NodeData { - return &NodeData{ - Config: &Config{ - HighWaterMark: defaultHightWaterMark, - AlarmWaterMark: defaultAlarmWaterMark, - SyncInterval: defaultQuotaTurboSyncInterval, - }, - lastProcStat: ProcStat{ - total: -1, - busy: -1, - }, - containers: make(map[string]*CPUQuota, 0), - cpuUtils: make([]cpuUtil, 0), - } -} - -// getLastCPUUtil obtain the latest cpu utilization -func (d *NodeData) getLastCPUUtil() float64 { - if len(d.cpuUtils) == 0 { - return 0 - } - return d.cpuUtils[len(d.cpuUtils)-1].util -} - -// removeContainer deletes the list of pods that do not need to be adjusted. -func (d *NodeData) removeContainer(id string) error { - d.RLock() - cq, ok := d.containers[id] - d.RUnlock() - if !ok { - return nil - } - safeDel := func(id string) error { - d.Lock() - delete(d.containers, id) - d.Unlock() - return nil - } - - if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", cq.CgroupPath)) { - return safeDel(id) - } - // cq.Period ranges from 1000(us) to 1000000(us) and does not overflow. - origin := int64(cq.LimitResources[typedef.ResourceCPU] * float64(cq.period)) - if err := cq.SetCgroupAttr(cpuQuotaKey, util.FormatInt64(origin)); err != nil { - return fmt.Errorf("fail to recover cpu.cfs_quota_us for container %s : %v", cq.Name, err) - } - return safeDel(id) -} - -// updateCPUUtils updates the cpu usage of a node -func (d *NodeData) updateCPUUtils() error { - var ( - curUtil float64 = 0 - index = 0 - t cpuUtil - ) - ps, err := getProcStat() - if err != nil { - return err - } - if d.lastProcStat.total >= 0 { - curUtil = calculateUtils(d.lastProcStat, ps) - } - d.lastProcStat = ps - cur := time.Now().UnixNano() - d.cpuUtils = append(d.cpuUtils, cpuUtil{ - timestamp: cur, - util: curUtil, - }) - // retain utilization data for only one minute - const minuteTimeDelta = int64(time.Minute) - for index, t = range d.cpuUtils { - if cur-t.timestamp <= minuteTimeDelta { - break - } - } - if index > 0 { - d.cpuUtils = d.cpuUtils[index:] - } - return nil -} - -// UpdateClusterContainers synchronizes data from given containers -func (d *NodeData) UpdateClusterContainers(conts map[string]*typedef.ContainerInfo) error { - var toBeDeletedList []string - for _, cont := range conts { - old, ok := d.containers[cont.ID] - // delete or skip containers that do not meet the conditions. - if !isAdjustmentAllowed(cont) { - if ok { - toBeDeletedList = append(toBeDeletedList, cont.ID) - } - continue - } - // add container - if !ok { - log.Debugf("add container %v (name : %v)", cont.ID, cont.Name) - if newQuota, err := NewCPUQuota(cont); err != nil { - log.Errorf("failed to create cpu quota object %v, error: %v", cont.Name, err) - } else { - d.containers[cont.ID] = newQuota - } - continue - } - // update data container in the quotaTurboList - old.ContainerInfo = cont - if err := old.updatePeriod(); err != nil { - log.Errorf("fail to update period : %v", err) - } - if err := old.updateThrottle(); err != nil { - log.Errorf("fail to update throttle time : %v", err) - } - if err := old.updateQuota(); err != nil { - log.Errorf("fail to update quota : %v", err) - } - if err := old.updateUsage(); err != nil { - log.Errorf("fail to update cpu usage : %v", err) - } - } - // non trust list container - for id := range d.containers { - // the container is removed from the trust list - if _, ok := conts[id]; !ok { - toBeDeletedList = append(toBeDeletedList, id) - } - } - - for _, id := range toBeDeletedList { - if err := d.removeContainer(id); err != nil { - log.Errorf(err.Error()) - } - } - return nil -} - -// WriteQuota saves the quota value of the container -func (d *NodeData) WriteQuota() { - for _, c := range d.containers { - if err := c.WriteQuota(); err != nil { - log.Errorf(err.Error()) - } - } -} - -// isAdjustmentAllowed judges whether quota adjustment is allowed -func isAdjustmentAllowed(ci *typedef.ContainerInfo) bool { - // 1. containers whose cgroup path does not exist are not considered. - if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "")) { - return false - } - - // 2. abnormal CPULimit - // a). containers that do not limit the quota - // b). CPULimit = 0 : k8s allows the CPULimit to be 0, but the quota is not limited. - if ci.LimitResources[typedef.ResourceCPU] <= 0 || - ci.RequestResources[typedef.ResourceCPU] <= 0 || - ci.LimitResources[typedef.ResourceCPU] == float64(runtime.NumCPU()) || - ci.RequestResources[typedef.ResourceCPU] == float64(runtime.NumCPU()) { - return false - } - return true -} diff --git a/pkg/services/quotaturbo/data_test.go b/pkg/services/quotaturbo/data_test.go deleted file mode 100644 index 5f516c3..0000000 --- a/pkg/services/quotaturbo/data_test.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-16 -// Description: This file is used for testing data.go - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "fmt" - "path" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/core/typedef/cgroup" - "isula.org/rubik/test/try" -) - -// TestNodeDataGetLastCPUUtil tests getLastCPUUtil of NodeData -func TestNodeDataGetLastCPUUtil(t *testing.T) { - // 1. empty CPU Utils - d := &NodeData{} - t.Run("TC1-empty CPU Util", func(t *testing.T) { - util := float64(0.0) - assert.Equal(t, util, d.getLastCPUUtil()) - }) - // 2. CPU Utils - cpuUtil20 := 20 - d = &NodeData{cpuUtils: []cpuUtil{{ - util: float64(cpuUtil20), - }}} - t.Run("TC2-CPU Util is 20", func(t *testing.T) { - util := float64(20.0) - assert.Equal(t, util, d.getLastCPUUtil()) - }) -} - -// TestNodeDataRemoveContainer tests removeContainer of NodeData -func TestNodeDataRemoveContainer(t *testing.T) { - var nodeDataRemoveContainerTests = []struct { - data *NodeData - name string - id string - num int - }{ - { - name: "TC1-no container exists", - id: "", - num: 1, - data: &NodeData{ - containers: map[string]*CPUQuota{ - "testCon1": {}, - }, - }, - }, - { - name: "TC2-the container path does not exist", - id: "testCon3", - num: 1, - data: &NodeData{ - containers: map[string]*CPUQuota{ - "testCon1": {}, - "testCon3": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon3", - CgroupPath: "kubepods/testPod3/testCon3", - }, - }, - }, - }, - }, - { - name: "TC3-delete a container normally", - id: containerInfos[0].ID, - num: 0, - data: &NodeData{ - containers: map[string]*CPUQuota{ - containerInfos[0].ID: { - ContainerInfo: containerInfos[0].DeepCopy(), - period: 100000, - }, - }, - }, - }, - { - name: "TC4-write an invalid value", - id: containerInfos[0].ID, - num: 1, - data: &NodeData{ - containers: map[string]*CPUQuota{ - containerInfos[0].ID: { - ContainerInfo: containerInfos[0].DeepCopy(), - period: 1000000000000000000, - }, - }, - }, - }, - } - cis := []*typedef.ContainerInfo{containerInfos[0].DeepCopy()} - mkCgDirs(cis) - defer rmCgDirs(cis) - - for _, tt := range nodeDataRemoveContainerTests { - t.Run(tt.name, func(t *testing.T) { - tt.data.removeContainer(tt.id) - assert.Equal(t, tt.num, len(tt.data.containers)) - }) - } - us, err := cis[0].GetCgroupAttr(cpuQuotaKey).Int64() - if err != nil { - assert.NoError(t, err) - } - assert.Equal(t, int64(cis[0].LimitResources[typedef.ResourceCPU]*100000), us) -} - -// TestQuotaTurboUpdateCPUUtils tests updateCPUUtils of QuotaTurbo and NewProcStat -func TestQuotaTurboUpdateCPUUtils(t *testing.T) { - data := NewNodeData() - // 1. obtain the cpu usage for the first time - if err := data.updateCPUUtils(); err != nil { - assert.NoError(t, err) - } - num1 := 1 - assert.Equal(t, num1, len(data.cpuUtils)) - // 2. obtain the cpu usage for the second time - if err := data.updateCPUUtils(); err != nil { - assert.NoError(t, err) - } - num2 := 2 - assert.Equal(t, num2, len(data.cpuUtils)) - // 3. obtain the cpu usage after 1 minute - var minuteTimeDelta int64 = 60000000001 - data.cpuUtils[0].timestamp -= minuteTimeDelta - if err := data.updateCPUUtils(); err != nil { - assert.NoError(t, err) - } - assert.Equal(t, num2, len(data.cpuUtils)) -} - -// TestIsAdjustmentAllowed tests isAdjustmentAllowed -func TestIsAdjustmentAllowed(t *testing.T) { - cis := []*typedef.ContainerInfo{containerInfos[0].DeepCopy(), containerInfos[3].DeepCopy()} - mkCgDirs(cis) - defer rmCgDirs(cis) - - tests := []struct { - ci *typedef.ContainerInfo - name string - want bool - }{ - { - name: "TC1-allow adjustment", - ci: cis[0], - want: true, - }, - { - name: "TC2-cgroup path is not existed", - ci: containerInfos[1], - want: false, - }, - { - name: "TC3-cpulimit = 0", - ci: cis[1], - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, isAdjustmentAllowed(tt.ci), tt.want) - }) - } -} - -// TestUpdateClusterContainers tests UpdateClusterContainers -func TestUpdateClusterContainers(t *testing.T) { - cis := []*typedef.ContainerInfo{containerInfos[0]} - mkCgDirs(cis) - defer rmCgDirs(cis) - - qt := &QuotaTurbo{ - NodeData: &NodeData{ - containers: make(map[string]*CPUQuota, 0), - }, - } - - // 1. add container - qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{cis[0].ID: cis[0]}) - conNum1 := 1 - assert.Equal(t, conNum1, len(qt.containers)) - - // 2. updated container - qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{cis[0].ID: cis[0]}) - assert.Equal(t, conNum1, len(qt.containers)) - - // 3. deleting a container that does not meet the conditions (cgroup path is not existed) - // 4. delete a container whose checkpoint does not exist. - qt.containers[containerInfos[2].ID] = &CPUQuota{ContainerInfo: containerInfos[2].DeepCopy()} - conNum2 := 2 - assert.Equal(t, conNum2, len(qt.containers)) - qt.UpdateClusterContainers(map[string]*typedef.ContainerInfo{containerInfos[2].ID: containerInfos[2].DeepCopy()}) - conNum0 := 0 - assert.Equal(t, conNum0, len(qt.containers)) -} - -// mkCgDirs creates the cgroup folder for the container -func mkCgDirs(cc []*typedef.ContainerInfo) { - for _, ci := range cc { - dirPath := cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") - fmt.Println("path : " + dirPath) - try.MkdirAll(dirPath, constant.DefaultDirMode) - } -} - -// rmCgDirs deletes the cgroup folder of the container -func rmCgDirs(cc []*typedef.ContainerInfo) { - for _, ci := range cc { - dirPath := cgroup.AbsoluteCgroupPath("cpu", ci.CgroupPath, "") - try.RemoveAll(dirPath) - try.RemoveAll(path.Dir(dirPath)) - } -} diff --git a/pkg/services/quotaturbo/driverevent.go b/pkg/services/quotaturbo/driverevent.go deleted file mode 100644 index 75025ca..0000000 --- a/pkg/services/quotaturbo/driverevent.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-16 -// Description: event driver method for quota turbo - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "math" - "runtime" - - "isula.org/rubik/pkg/common/log" - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" -) - -// Driver uses different methods based on different policies. -type Driver interface { - // adjustQuota calculate the quota in the next period based on the customized policy, upper limit, and quota. - adjustQuota(data *NodeData) -} - -const ( - // upperLimitOfIncrease is the maximum percentage of. - // the total amount of a single promotion to the total amount of nodes. - upperLimitOfIncrease = 1.0 - // slowFallbackRatio is the rate of slow fallback. - slowFallbackRatio = 0.1 - // cpuUtilMaxChange is the upper limit of the node change rate in one minute. - cpuUtilMaxChange float64 = 10 -) - -// EventDriver event based quota adjustment driver. -type EventDriver struct{} - -// adjustQuota calculates quota delta based on events -func (e *EventDriver) adjustQuota(data *NodeData) { - e.slowFallback(data) - e.fastFallback(data) - // Ensure that the CPU usage does not change by more than 10% within one minute. - // Otherwise, the available quota rollback continues but does not increase. - if !sharpFluctuates(data) { - e.elevate(data) - } else { - log.Infof("the CPU usage changes by more than %.2f, "+ - "the quota will not adjusted in this round", cpuUtilMaxChange) - } - for _, c := range data.containers { - // get height limit - const easingMultiple = 2.0 - // c.Period - c.heightLimit = easingMultiple * c.LimitResources[typedef.ResourceCPU] * float64(c.period) - // get the maximum available ensuring that the overall utilization does not exceed the limit. - c.maxQuotaNextPeriod = getMaxQuota(c) - // c.Period ranges from 1000(us) to 1000000(us) and does not overflow. - c.nextQuota = int64(math.Max(math.Min(float64(c.curQuota)+c.quotaDelta, c.maxQuotaNextPeriod), - c.LimitResources[typedef.ResourceCPU]*float64(c.period))) - } -} - -// elevate boosts when cpu is suppressed -func (e *EventDriver) elevate(data *NodeData) { - // the CPU usage of the current node is lower than the warning watermark. - // U + R <= a & a > U ======> a - U >= R && a - U > 0 =====> a - U >= R - if float64(data.AlarmWaterMark)-data.getLastCPUUtil() < upperLimitOfIncrease { - return - } - // sumDelta : total number of cores to be adjusted - var sumDelta float64 = 0 - delta := make(map[string]float64, 0) - for _, c := range data.containers { - if c.curThrottle.NrThrottled > c.preThrottle.NrThrottled { - delta[c.ID] = NsToUs(c.curThrottle.ThrottledTime-c.preThrottle.ThrottledTime) / - float64(c.curThrottle.NrThrottled-c.preThrottle.NrThrottled) / float64(c.period) - sumDelta += delta[c.ID] - } - } - // the container quota does not need to be increased in this round. - if sumDelta == 0 { - return - } - // the total increase cannot exceed (upperLimitOfIncrease%) of the total available CPUs of the node. - A := math.Min(sumDelta, util.PercentageToDecimal(upperLimitOfIncrease)*float64(runtime.NumCPU())) - coefficient := A / sumDelta - for id, quotaDelta := range delta { - data.containers[id].quotaDelta += coefficient * quotaDelta * float64(data.containers[id].period) - } -} - -// fastFallback decreases the quota to ensure that the CPU utilization of the node is below the warning water level -// when the water level exceeds the warning water level. -func (e *EventDriver) fastFallback(data *NodeData) { - // The CPU usage of the current node is greater than the warning watermark, triggering a fast rollback. - if float64(data.AlarmWaterMark) > data.getLastCPUUtil() { - return - } - // sub : the total number of CPU quotas to be reduced on a node. - sub := util.PercentageToDecimal(float64(data.AlarmWaterMark)-data.getLastCPUUtil()) * float64(runtime.NumCPU()) - // sumDelta ：total number of cpu cores that can be decreased. - var sumDelta float64 = 0 - delta := make(map[string]float64, 0) - for _, c := range data.containers { - delta[c.ID] = float64(c.curQuota)/float64(c.period) - c.LimitResources[typedef.ResourceCPU] - sumDelta += delta[c.ID] - } - if sumDelta <= 0 { - return - } - // proportional adjustment of each business quota. - for id, quotaDelta := range delta { - data.containers[id].quotaDelta += (quotaDelta / sumDelta) * sub * float64(data.containers[id].period) - } -} - -// slowFallback triggers quota callback of unpressed containers when the CPU utilization exceeds the control watermark. -func (e *EventDriver) slowFallback(data *NodeData) { - // The CPU usage of the current node is greater than the high watermark, triggering a slow rollback. - if float64(data.HighWaterMark) > data.getLastCPUUtil() { - return - } - coefficient := (data.getLastCPUUtil() - float64(data.HighWaterMark)) / - float64(data.AlarmWaterMark-data.HighWaterMark) * slowFallbackRatio - for id, c := range data.containers { - originQuota := int64(c.LimitResources[typedef.ResourceCPU] * float64(c.period)) - if c.curQuota > originQuota && c.curThrottle.NrThrottled == c.preThrottle.NrThrottled { - data.containers[id].quotaDelta += coefficient * float64(originQuota-c.curQuota) - } - } -} - -// sharpFluctuates checks whether the node CPU utilization exceeds the specified value within one minute. -func sharpFluctuates(data *NodeData) bool { - var ( - min float64 = maximumUtilization - max float64 = minimumUtilization - ) - for _, u := range data.cpuUtils { - min = math.Min(min, u.util) - max = math.Max(max, u.util) - } - if max-min > cpuUtilMaxChange { - log.Debugf("The resources changed drastically, and no adjustment will be made this time") - return true - } - return false -} - -// getMaxQuota calculate the maximum available quota in the next period based on the container CPU usage in N-1 periods. -func getMaxQuota(c *CPUQuota) float64 { - if len(c.cpuUsages) <= 1 { - return c.heightLimit - } - // the time unit is nanosecond - first := c.cpuUsages[0] - last := c.cpuUsages[len(c.cpuUsages)-1] - timeDelta := NsToUs(last.timestamp - first.timestamp) - coefficient := float64(len(c.cpuUsages)) / float64(len(c.cpuUsages)-1) - maxAvailable := c.LimitResources[typedef.ResourceCPU] * timeDelta * coefficient - used := NsToUs(last.usage - first.usage) - remainingUsage := maxAvailable - used - origin := c.LimitResources[typedef.ResourceCPU] * float64(c.period) - const ( - // To prevent sharp service jitters, the Rubik proactively decreases the traffic in advance - // when the available balance reaches a certain threshold. - // The limitMultiplier is used to control the relationship between the upper limit and the threshold. - // Experiments show that the value 3 is efficient and secure. - limitMultiplier = 3 - precision = 1e-10 - ) - var threshold = limitMultiplier * c.heightLimit - remainingQuota := util.Div(remainingUsage, timeDelta, math.MaxFloat64, precision) * - float64(len(c.cpuUsages)-1) * float64(c.period) - - // gradually decrease beyond the threshold to prevent sudden dips. - res := remainingQuota - if remainingQuota <= threshold { - res = origin + util.Div((c.heightLimit-origin)*remainingQuota, threshold, threshold, precision) - } - // The utilization must not exceed the height limit and must not be less than the cpuLimit. - return math.Max(math.Min(res, c.heightLimit), origin) -} - -// NsToUs converts nanoseconds into microseconds -func NsToUs(ns int64) float64 { - // number of nanoseconds contained in 1 microsecond - const nanoSecPerMicroSec float64 = 1000 - return util.Div(float64(ns), nanoSecPerMicroSec) -} diff --git a/pkg/services/quotaturbo/driverevent_test.go b/pkg/services/quotaturbo/driverevent_test.go deleted file mode 100644 index a951e0d..0000000 --- a/pkg/services/quotaturbo/driverevent_test.go +++ /dev/null @@ -1,593 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-20 -// Description: This file is used for testing driverevent.go - -// Package quotaturbo is for Quota Turbo feature -package quotaturbo - -import ( - "math" - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/core/typedef/cgroup" -) - -// TestEventDriverElevate tests elevate of EventDriver -func TestEventDriverElevate(t *testing.T) { - var elevateTests = []struct { - data *NodeData - judgements func(t *testing.T, data *NodeData) - name string - }{ - { - name: "TC1 - CPU usage >= the alarmWaterMark.", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 60, - }, - containers: map[string]*CPUQuota{ - "testCon1": {}, - }, - cpuUtils: []cpuUtil{ - { - util: 90, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - var delta float64 = 0 - conID := "testCon1" - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC2 - the container is not suppressed.", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 70, - }, - containers: map[string]*CPUQuota{ - "testCon2": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon2", - }, - // currently not suppressed - curThrottle: &cgroup.CPUStat{ - NrThrottled: 1, - ThrottledTime: 10, - }, - preThrottle: &cgroup.CPUStat{ - NrThrottled: 1, - ThrottledTime: 10, - }, - period: 100000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 60, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - var delta float64 = 0 - conID := "testCon2" - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC3 - increase the quota of the suppressed container", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 60, - }, - containers: map[string]*CPUQuota{ - "testCon3": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon3", - }, - curThrottle: &cgroup.CPUStat{ - NrThrottled: 50, - ThrottledTime: 200000, - }, - preThrottle: &cgroup.CPUStat{ - NrThrottled: 40, - ThrottledTime: 100000, - }, - period: 100000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 40, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon3" - c := data.containers[conID] - coefficient := math.Min(float64(0.0001), util.PercentageToDecimal(upperLimitOfIncrease)* - float64(runtime.NumCPU())) / float64(0.0001) - delta := coefficient * float64(0.0001) * float64(c.period) - assert.True(t, data.containers[conID].quotaDelta == delta) - }, - }, - } - - e := &EventDriver{} - for _, tt := range elevateTests { - t.Run(tt.name, func(t *testing.T) { - e.elevate(tt.data) - tt.judgements(t, tt.data) - }) - } -} - -// TestSlowFallback tests slowFallback of EventDriver -func TestSlowFallback(t *testing.T) { - var slowFallBackTests = []struct { - data *NodeData - judgements func(t *testing.T, data *NodeData) - name string - }{ - { - name: "TC1-CPU usage <= the highWaterMark.", - data: &NodeData{ - Config: &Config{ - HighWaterMark: 60, - }, - containers: map[string]*CPUQuota{ - "testCon4": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon4", - }, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 40, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon4" - var delta float64 = 0 - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC2-the container is suppressed.", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 80, - HighWaterMark: 50, - }, - containers: map[string]*CPUQuota{ - "testCon5": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon5", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 1, - }, - }, - curThrottle: &cgroup.CPUStat{ - NrThrottled: 10, - }, - preThrottle: &cgroup.CPUStat{ - NrThrottled: 0, - }, - period: 100000, - curQuota: 200000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 70, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - var delta float64 = 0 - conID := "testCon5" - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC3-decrease the quota of the uncompressed containers", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 90, - HighWaterMark: 40, - }, - containers: map[string]*CPUQuota{ - "testCon6": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon6", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 2, - }, - }, - // currently not suppressed - curThrottle: &cgroup.CPUStat{ - NrThrottled: 10, - ThrottledTime: 100000, - }, - preThrottle: &cgroup.CPUStat{ - NrThrottled: 10, - ThrottledTime: 100000, - }, - period: 100000, - curQuota: 400000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 60.0, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon6" - c := data.containers[conID] - coefficient := (data.getLastCPUUtil() - float64(data.HighWaterMark)) / - float64(data.AlarmWaterMark-data.HighWaterMark) * slowFallbackRatio - delta := coefficient * - ((float64(c.LimitResources[typedef.ResourceCPU]) * float64(c.period)) - float64(c.curQuota)) - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - } - e := &EventDriver{} - for _, tt := range slowFallBackTests { - t.Run(tt.name, func(t *testing.T) { - e.slowFallback(tt.data) - tt.judgements(t, tt.data) - }) - } -} - -// TestFastFallback tests fastFallback of EventDriver -func TestFastFallback(t *testing.T) { - var fastFallBackTests = []struct { - data *NodeData - judgements func(t *testing.T, data *NodeData) - name string - }{ - { - name: "TC1-CPU usage <= the AlarmWaterMark.", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 30, - }, - containers: map[string]*CPUQuota{ - "testCon7": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon7", - }, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 10, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon7" - var delta float64 = 0 - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC2-the quota of container is not increased.", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 30, - }, - containers: map[string]*CPUQuota{ - "testCon8": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon8", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 1, - }, - }, - period: 100, - curQuota: 100, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 48, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - var delta float64 = 0 - conID := "testCon8" - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC3-decrease the quota of the containers", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 65, - }, - containers: map[string]*CPUQuota{ - "testCon9": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon9", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 3, - }, - }, - period: 10000, - curQuota: 40000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 90, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon9" - c := data.containers[conID] - delta := util.PercentageToDecimal(float64(data.AlarmWaterMark)-data.getLastCPUUtil()) * - float64(runtime.NumCPU()) * float64(c.period) - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - } - e := &EventDriver{} - for _, tt := range fastFallBackTests { - t.Run(tt.name, func(t *testing.T) { - e.fastFallback(tt.data) - tt.judgements(t, tt.data) - }) - } -} - -// TestSharpFluctuates tests sharpFluctuates -func TestSharpFluctuates(t *testing.T) { - var sharpFluctuatesTests = []struct { - data *NodeData - want bool - name string - }{ - { - name: "TC1-the cpu changes rapidly", - data: &NodeData{ - cpuUtils: []cpuUtil{ - { - util: 90, - }, - { - util: 90 - cpuUtilMaxChange - 1, - }, - }, - }, - want: true, - }, - { - name: "TC2-the cpu changes steadily", - data: &NodeData{ - cpuUtils: []cpuUtil{ - { - util: 90, - }, - { - util: 90 - cpuUtilMaxChange + 1, - }, - }, - }, - want: false, - }, - } - for _, tt := range sharpFluctuatesTests { - t.Run(tt.name, func(t *testing.T) { - assert.True(t, sharpFluctuates(tt.data) == tt.want) - }) - } -} - -// TestEventDriverAdjustQuota tests adjustQuota of EventDriver -func TestEventDriverAdjustQuota(t *testing.T) { - var eDriverAdjustQuotaTests = []struct { - data *NodeData - judgements func(t *testing.T, data *NodeData) - name string - }{ - { - name: "TC1-no promotion", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 80, - HighWaterMark: 73, - }, - containers: map[string]*CPUQuota{ - "testCon10": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon10", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 1, - }, - }, - period: 80, - curQuota: 100, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 1, - }, - { - util: -cpuUtilMaxChange, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - var delta float64 = 0 - conID := "testCon10" - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - { - name: "TC2-make a promotion", - data: &NodeData{ - Config: &Config{ - AlarmWaterMark: 97, - HighWaterMark: 73, - }, - containers: map[string]*CPUQuota{ - "testCon11": { - ContainerInfo: &typedef.ContainerInfo{ - ID: "testCon11", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 2, - }, - }, - curThrottle: &cgroup.CPUStat{ - NrThrottled: 1, - ThrottledTime: 200, - }, - preThrottle: &cgroup.CPUStat{ - NrThrottled: 0, - ThrottledTime: 100, - }, - period: 2000, - curQuota: 5000, - }, - }, - cpuUtils: []cpuUtil{ - { - util: 10, - }, - }, - }, - judgements: func(t *testing.T, data *NodeData) { - conID := "testCon11" - c := data.containers[conID] - coefficient := math.Min(float64(0.00005), util.PercentageToDecimal(upperLimitOfIncrease)* - float64(runtime.NumCPU())) / float64(0.00005) - delta := coefficient * float64(0.00005) * float64(c.period) - assert.Equal(t, delta, data.containers[conID].quotaDelta) - }, - }, - } - e := &EventDriver{} - for _, tt := range eDriverAdjustQuotaTests { - t.Run(tt.name, func(t *testing.T) { - e.adjustQuota(tt.data) - tt.judgements(t, tt.data) - }) - } -} - -// TestGetMaxQuota tests getMaxQuota -func TestGetMaxQuota(t *testing.T) { - var getMaxQuotaTests = []struct { - cq *CPUQuota - judgements func(t *testing.T, cq *CPUQuota) - name string - }{ - { - name: "TC1-empty cpu usage", - cq: &CPUQuota{ - heightLimit: 100, - cpuUsages: []cpuUsage{}, - }, - judgements: func(t *testing.T, cq *CPUQuota) { - var res float64 = 100 - assert.Equal(t, res, getMaxQuota(cq)) - }, - }, - { - name: "TC2-The remaining value is less than 3 times the upper limit.", - cq: &CPUQuota{ - cpuUsages: []cpuUsage{ - {100000, 100000}, - {200000, 200000}, - }, - ContainerInfo: &typedef.ContainerInfo{ - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 4, - }, - }, - period: 100, - heightLimit: 800, - }, - judgements: func(t *testing.T, cq *CPUQuota) { - const res = 400 + float64(400*700)/float64(3*800) - assert.Equal(t, res, getMaxQuota(cq)) - }, - }, - { - name: "TC3-The remaining value is greater than 3 times the limit height.", - cq: &CPUQuota{ - cpuUsages: []cpuUsage{ - {10000, 0}, - {20000, 0}, - {30000, 0}, - {40000, 0}, - {50000, 0}, - {60000, 0}, - {70000, 0}, - {80000, 100}, - }, - ContainerInfo: &typedef.ContainerInfo{ - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 1, - }, - }, - period: 100, - heightLimit: 200, - }, - judgements: func(t *testing.T, cq *CPUQuota) { - var res float64 = 200 - assert.Equal(t, res, getMaxQuota(cq)) - }, - }, - { - name: "TC4-The remaining value is less than the initial value.", - cq: &CPUQuota{ - cpuUsages: []cpuUsage{ - {100, 0}, - {200, 1000000}, - }, - ContainerInfo: &typedef.ContainerInfo{ - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 10, - }, - }, - period: 10, - heightLimit: 150, - }, - judgements: func(t *testing.T, cq *CPUQuota) { - var res float64 = 100 - assert.Equal(t, res, getMaxQuota(cq)) - }, - }, - } - for _, tt := range getMaxQuotaTests { - t.Run(tt.name, func(t *testing.T) { - tt.judgements(t, tt.cq) - }) - } -} diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index c5a1639..b6c203d 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -17,6 +17,7 @@ package quotaturbo import ( "context" "fmt" + "path/filepath" "time" "k8s.io/apimachinery/pkg/util/wait" @@ -26,39 +27,68 @@ import ( "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/lib/cpu/quotaturbo" "isula.org/rubik/pkg/services/helper" ) +const ( + defaultHightWaterMark = 60 + defaultAlarmWaterMark = 80 + defaultQuotaTurboSyncInterval = 100 +) + +var ( + cpuPeriodKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} + cpuQuotaKey = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_quota_us"} +) + +// QuotaTurboFactory is the QuotaTurbo factory class type QuotaTurboFactory struct { ObjName string } +// Name returns the factory class name func (i QuotaTurboFactory) Name() string { - return "BlkioThrottleFactory" + return "QuotaTurboFactory" } +// NewObj returns a QuotaTurbo object func (i QuotaTurboFactory) NewObj() (interface{}, error) { return NewQuotaTurbo(i.ObjName), nil } +// Config is the config of QuotaTurbo +type Config struct { + HighWaterMark int `json:"highWaterMark,omitempty"` + AlarmWaterMark int `json:"alarmWaterMark,omitempty"` + SyncInterval int `json:"syncInterval,omitempty"` +} + +// NewConfig returns quotaTurbo config instance +func NewConfig() *Config { + return &Config{ + HighWaterMark: defaultHightWaterMark, + AlarmWaterMark: defaultAlarmWaterMark, + SyncInterval: defaultQuotaTurboSyncInterval, + } +} + // QuotaTurbo manages all container CPU quota data on the current node. type QuotaTurbo struct { - helper.ServiceBase - name string - // NodeData including the container data, CPU usage, and QuotaTurbo configuration of the local node - *NodeData - // interfaces with different policies - Driver - // referenced object to list pods + name string + conf *Config + client *quotaturbo.Client Viewer api.Viewer + helper.ServiceBase } // NewQuotaTurbo generate quota turbo objects func NewQuotaTurbo(n string) *QuotaTurbo { return &QuotaTurbo{ - name: n, - NodeData: NewNodeData(), - Driver: &EventDriver{}, + name: n, + conf: NewConfig(), + client: quotaturbo.NewClient(), } } @@ -67,18 +97,43 @@ func (qt *QuotaTurbo) ID() string { return qt.name } -// AdjustQuota adjusts the quota of a container at a time -func (qt *QuotaTurbo) AdjustQuota(cc map[string]*typedef.ContainerInfo) { - qt.UpdateClusterContainers(cc) - if err := qt.updateCPUUtils(); err != nil { - log.Errorf("fail to get current cpu utilization : %v", err) - return +// syncCgroups updates the cgroup in cilent according to the current whitelist pod list +func (qt *QuotaTurbo) syncCgroups(conts map[string]*typedef.ContainerInfo) { + var ( + existedCgroupPaths = qt.client.GetAllCgroup() + existedCgroupPathMap = make(map[string]struct{}, len(existedCgroupPaths)) + ) + // delete containers marked as no need to adjust quota + for _, path := range existedCgroupPaths { + id := filepath.Base(path) + existedCgroupPathMap[id] = struct{}{} + if _, found := conts[id]; !found { + if err := qt.client.RemoveCgroup(path); err != nil { + log.Errorf("error removing container %v: %v", id, err) + } + } } - if len(qt.containers) == 0 { - return + for id, cont := range conts { + /* + Currently, modifying the cpu limit and container id of the container will cause the container to restart, + so it is considered that the cgroup path and cpulimit will not change during the life cycle of the container + */ + if _, ok := existedCgroupPathMap[id]; ok { + continue + } + // add container to quotaturbo + if err := qt.client.AddCgroup(cont.CgroupPath, cont.LimitResources[typedef.ResourceCPU]); err != nil { + log.Errorf("error adding container %v: %v", cont.Name, err) + } + } +} + +// AdjustQuota adjusts the quota of a container at a time +func (qt *QuotaTurbo) AdjustQuota(conts map[string]*typedef.ContainerInfo) { + qt.syncCgroups(conts) + if err := qt.client.AdjustQuota(); err != nil { + log.Errorf("error occur when adjust quota: %v", err) } - qt.adjustQuota(qt.NodeData) - qt.WriteQuota() } // Run adjusts the quota of the trust list container cyclically. @@ -90,12 +145,12 @@ func (qt *QuotaTurbo) Run(ctx context.Context) { return pod.Annotations[constant.QuotaAnnotationKey] == "true" })) }, - time.Millisecond*time.Duration(qt.SyncInterval), + time.Millisecond*time.Duration(qt.conf.SyncInterval), ctx.Done()) } -// Validate Validate verifies that the quotaTurbo parameter is set correctly -func (qt *QuotaTurbo) Validate() error { +// Validate verifies that the quotaTurbo parameter is set correctly +func (conf *Config) Validate() error { const ( minQuotaTurboWaterMark, maxQuotaTurboWaterMark = 0, 100 minQuotaTurboSyncInterval, maxQuotaTurboSyncInterval = 100, 10000 @@ -106,20 +161,44 @@ func (qt *QuotaTurbo) Validate() error { } return false } - if qt.AlarmWaterMark <= qt.HighWaterMark || - outOfRange(qt.HighWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) || - outOfRange(qt.AlarmWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) { + if conf.AlarmWaterMark <= conf.HighWaterMark || + outOfRange(conf.HighWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) || + outOfRange(conf.AlarmWaterMark, minQuotaTurboWaterMark, maxQuotaTurboWaterMark) { return fmt.Errorf("alarmWaterMark >= highWaterMark, both of which ranges from 0 to 100") } - if outOfRange(qt.SyncInterval, minQuotaTurboSyncInterval, maxQuotaTurboSyncInterval) { + if outOfRange(conf.SyncInterval, minQuotaTurboSyncInterval, maxQuotaTurboSyncInterval) { return fmt.Errorf("synchronization time ranges from 100 (0.1s) to 10000 (10s)") } return nil } +// SetConfig sets and checks Config +func (qt *QuotaTurbo) SetConfig(f helper.ConfigHandler) error { + var conf = NewConfig() + if err := f(qt.name, conf); err != nil { + return err + } + if err := conf.Validate(); err != nil { + return err + } + qt.conf = conf + return nil +} + +// IsRunner returns true that tells other quotaTurbo is a persistent service +func (qt *QuotaTurbo) IsRunner() bool { + return true +} + // PreStart is the pre-start action func (qt *QuotaTurbo) PreStart(viewer api.Viewer) error { + // 1. set the parameters of the quotaturbo client + qt.client.CgroupRoot = cgroup.GetMountDir() + qt.client.HighWaterMark = qt.conf.HighWaterMark + qt.client.AlarmWaterMark = qt.conf.AlarmWaterMark qt.Viewer = viewer + + // 2. attempts to fix all currently running pods and containers pods := viewer.ListPodsWithOptions() for _, pod := range pods { recoverOnePodQuota(pod) diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go deleted file mode 100644 index 18b6269..0000000 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. -// rubik licensed under the Mulan PSL v2. -// You can use this software according to the terms and conditions of the Mulan PSL v2. -// You may obtain a copy of Mulan PSL v2 at: -// http://license.coscl.org.cn/MulanPSL2 -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -// PURPOSE. -// See the Mulan PSL v2 for more details. -// Author: Jiaqi Yang -// Date: 2023-02-16 -// Description: This file is used for test quota turbo - -// Package quotaturbo is for Quota Turbo -package quotaturbo - -import ( - "context" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/common/constant" - "isula.org/rubik/pkg/common/util" - "isula.org/rubik/pkg/core/typedef" - "isula.org/rubik/pkg/podmanager" -) - -const moduleName = "quotaturbo" - -// TestQuotaTurbo_Validatec test Validate function -func TestQuotaTurbo_Validate(t *testing.T) { - tests := []struct { - name string - NodeData *NodeData - wantErr bool - }{ - { - name: "TC1-alarmWaterMark is less or equal to highWaterMark", - NodeData: &NodeData{Config: &Config{ - HighWaterMark: 100, - AlarmWaterMark: 60, - SyncInterval: 1000, - }}, - wantErr: true, - }, - { - name: "TC2-highWater mark exceed the max quota turbo water mark(100)", - NodeData: &NodeData{Config: &Config{ - HighWaterMark: 1000, - AlarmWaterMark: 100000, - SyncInterval: 1000, - }}, - wantErr: true, - }, - { - name: "TC3-sync interval out of range(100-10000)", - NodeData: &NodeData{Config: &Config{ - HighWaterMark: 60, - AlarmWaterMark: 80, - SyncInterval: 1, - }}, - wantErr: true, - }, - { - name: "TC4-normal case", - NodeData: &NodeData{Config: &Config{ - HighWaterMark: 60, - AlarmWaterMark: 100, - SyncInterval: 1000, - }}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - qt := &QuotaTurbo{ - NodeData: tt.NodeData, - } - if err := qt.Validate(); (err != nil) != tt.wantErr { - t.Errorf("QuotaTurbo.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func sameQuota(t *testing.T, path string, want int64) bool { - const cfsQuotaUsFileName = "cpu.cfs_quota_us" - data, err := util.ReadSmallFile(filepath.Join(path, cfsQuotaUsFileName)) - if err != nil { - assert.NoError(t, err) - return false - } - quota, err := util.ParseInt64(strings.ReplaceAll(string(data), "\n", "")) - if err != nil { - assert.NoError(t, err) - return false - } - if quota != want { - return false - } - return true -} - -// TestQuotaTurbo_Terminate tests Terminate function -func TestQuotaTurbo_Terminate(t *testing.T) { - const ( - fooContName = "Foo" - barContName = "Bar" - podUID = "testPod1" - wrongPodQuota = "600000" - wrongFooQuota = "300000" - wrongBarQuota = "200000" - podPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/" - fooPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon1" - barPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon2" - ) - - var ( - tests = []struct { - postfunc func(t *testing.T) - fooCPULimit float64 - barCPULimit float64 - name string - }{ - { - name: "TC1-one unlimited container is existed", - fooCPULimit: 2, - barCPULimit: 0, - postfunc: func(t *testing.T) { - var ( - unlimited int64 = -1 - correctFooQuota int64 = 200000 - ) - assert.True(t, sameQuota(t, podPath, unlimited)) - assert.True(t, sameQuota(t, fooPath, correctFooQuota)) - assert.True(t, sameQuota(t, barPath, unlimited)) - }, - }, - { - name: "TC2-all containers are unlimited", - fooCPULimit: 2, - barCPULimit: 1, - postfunc: func(t *testing.T) { - var ( - correctPodQuota int64 = 300000 - correctFooQuota int64 = 200000 - correctBarQuota int64 = 100000 - ) - assert.True(t, sameQuota(t, podPath, correctPodQuota)) - assert.True(t, sameQuota(t, fooPath, correctFooQuota)) - assert.True(t, sameQuota(t, barPath, correctBarQuota)) - }, - }, - { - name: "TC3-all containers are limited", - fooCPULimit: 0, - barCPULimit: 0, - postfunc: func(t *testing.T) { - var unLimited int64 = -1 - assert.True(t, sameQuota(t, podPath, unLimited)) - assert.True(t, sameQuota(t, fooPath, unLimited)) - assert.True(t, sameQuota(t, barPath, unLimited)) - }, - }, - } - ) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var ( - fooCont = &typedef.ContainerInfo{ - Name: fooContName, - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", - LimitResources: make(typedef.ResourceMap), - } - barCont = &typedef.ContainerInfo{ - Name: barContName, - ID: "testCon2", - CgroupPath: "kubepods/testPod1/testCon2", - LimitResources: make(typedef.ResourceMap), - } - contList = []*typedef.ContainerInfo{ - fooCont, - barCont, - } - pod = &typedef.PodInfo{ - UID: "testPod1", - CgroupPath: "kubepods/testPod1", - IDContainersMap: map[string]*typedef.ContainerInfo{ - fooCont.ID: fooCont, - barCont.ID: barCont, - }, - } - pm = &podmanager.PodManager{ - Pods: &podmanager.PodCache{ - Pods: map[string]*typedef.PodInfo{ - podUID: pod, - }, - }, - } - qt = &QuotaTurbo{ - Viewer: pm, - } - ) - - mkCgDirs(contList) - defer rmCgDirs(contList) - - fooCont.LimitResources[typedef.ResourceCPU] = tt.fooCPULimit - barCont.LimitResources[typedef.ResourceCPU] = tt.barCPULimit - - assert.NoError(t, pod.SetCgroupAttr(cpuQuotaKey, wrongPodQuota)) - assert.NoError(t, fooCont.SetCgroupAttr(cpuQuotaKey, wrongFooQuota)) - assert.NoError(t, barCont.SetCgroupAttr(cpuQuotaKey, wrongBarQuota)) - qt.Terminate(pm) - tt.postfunc(t) - }) - } -} - -// TestQuotaTurbo_AdjustQuota tests AdjustQuota function -func TestQuotaTurbo_AdjustQuota(t *testing.T) { - type fields struct { - NodeData *NodeData - Driver Driver - pm *podmanager.PodManager - } - type args struct { - cc map[string]*typedef.ContainerInfo - } - var ( - fooCont = &typedef.ContainerInfo{ - Name: "Foo", - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", - LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: 2, - }, - RequestResources: typedef.ResourceMap{ - typedef.ResourceCPU: 2, - }, - } - pod1 = &typedef.PodInfo{ - UID: "testPod1", - CgroupPath: "kubepods/testPod1", - IDContainersMap: map[string]*typedef.ContainerInfo{ - fooCont.ID: fooCont, - }, - } - pod2 = &typedef.PodInfo{ - UID: "testPod2", - CgroupPath: "kubepods/testPod2", - IDContainersMap: map[string]*typedef.ContainerInfo{}, - } - ) - - tests := []struct { - name string - fields fields - args args - }{ - { - name: "TC1-empty data", - args: args{ - cc: map[string]*typedef.ContainerInfo{}, - }, - fields: fields{ - NodeData: &NodeData{ - Config: &Config{ - AlarmWaterMark: 80, - HighWaterMark: 60, - }, - containers: make(map[string]*CPUQuota), - }, - Driver: &EventDriver{}, - pm: &podmanager.PodManager{ - Pods: &podmanager.PodCache{ - Pods: map[string]*typedef.PodInfo{ - pod1.UID: pod1, - pod2.UID: pod2, - }, - }, - }, - }, - }, - { - name: "TC2-existed data", - args: args{ - cc: map[string]*typedef.ContainerInfo{ - "testCon1": fooCont, - }, - }, - fields: fields{ - NodeData: &NodeData{ - Config: &Config{ - AlarmWaterMark: 80, - HighWaterMark: 60, - }, - containers: make(map[string]*CPUQuota), - }, - Driver: &EventDriver{}, - pm: &podmanager.PodManager{ - Pods: &podmanager.PodCache{ - Pods: map[string]*typedef.PodInfo{ - pod1.UID: pod1, - pod2.UID: pod2, - }, - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - qt := &QuotaTurbo{ - NodeData: tt.fields.NodeData, - Driver: tt.fields.Driver, - Viewer: tt.fields.pm, - } - conts := []*typedef.ContainerInfo{} - for _, p := range tt.fields.pm.Pods.Pods { - for _, c := range p.IDContainersMap { - conts = append(conts, c) - } - } - mkCgDirs(conts) - defer rmCgDirs(conts) - - qt.AdjustQuota(tt.args.cc) - }) - } -} - -// TestNewQuotaTurbo tests NewQuotaTurbo -func TestNewQuotaTurbo(t *testing.T) { - testName := "TC1-test otherv functions" - t.Run(testName, func(t *testing.T) { - got := NewQuotaTurbo(moduleName) - assert.Equal(t, moduleName, got.ID()) - got.Viewer = &podmanager.PodManager{ - Pods: &podmanager.PodCache{ - Pods: map[string]*typedef.PodInfo{ - "testPod1": { - UID: "testPod1", - CgroupPath: "kubepods/testPod1", - Annotations: map[string]string{ - constant.QuotaAnnotationKey: "true", - }, - }, - }, - }, - } - - ctx, cancle := context.WithCancel(context.Background()) - go got.Run(ctx) - time.Sleep(time.Second) - cancle() - }) -} - -// TestQuotaTurbo_PreStart tests PreStart -func TestQuotaTurbo_PreStart(t *testing.T) { - var ( - pm = &podmanager.PodManager{ - Pods: &podmanager.PodCache{ - Pods: map[string]*typedef.PodInfo{ - "testPod1": { - UID: "testPod1", - CgroupPath: "kubepods/testPod1", - IDContainersMap: make(map[string]*typedef.ContainerInfo), - }, - }, - }, - } - qt = &QuotaTurbo{} - ) - testName := "TC1- test Prestart" - t.Run(testName, func(t *testing.T) { - qt.PreStart(pm) - }) -} -- Gitee From fa373079a59d21bfd9045106d208f6d980b5870a Mon Sep 17 00:00:00 2001 From: hanchao Date: Sat, 11 Mar 2023 20:51:09 +0800 Subject: [PATCH 57/73] refactor: clean code --- pkg/feature/feature.go | 30 ++++++++++++++++------ pkg/rubik/rubik_feature.go | 28 +++++++++++++++----- pkg/services/component.go | 14 +++++----- pkg/services/dynCache/dynCache.go | 3 ++- pkg/services/helper/service_base.go | 8 +++++- pkg/services/iocost/iocost.go | 25 ++++++++++++------ pkg/services/iocost/iocost_origin.go | 8 +++--- pkg/services/iocost/utils.go | 14 ++++++++++ pkg/services/iolimit/iolimit.go | 20 +++++++++++++-- pkg/services/preemption/preemption.go | 12 ++++----- pkg/services/preemption/preemption_test.go | 10 ++++---- pkg/services/quotaburst/quotaburst.go | 12 ++++----- pkg/services/quotaburst/quotaburst_test.go | 18 ++++++------- 13 files changed, 138 insertions(+), 64 deletions(-) diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 9eb5a52..59c41bf 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -1,11 +1,25 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file for defining Features + +// Package feature package feature -var ( - FeaturePreemption = "preemption" - FeatureDynCache = "dynCache" - FeatureIOLimit = "ioLimit" - FeatureIOCost = "ioCost" - FeatureDynMemory = "dynMemory" - FeatureQuotaBurst = "quotaBurst" - FeatureQuotaTurbo = "quotaTurbo" +const ( + PreemptionFeature = "preemption" + DynCacheFeature = "dynCache" + IOLimitFeature = "ioLimit" + IOCostFeature = "ioCost" + DynMemoryFeature = "dynMemory" + QuotaBurstFeature = "quotaBurst" + QuotaTurboFeature = "quotaTurbo" ) diff --git a/pkg/rubik/rubik_feature.go b/pkg/rubik/rubik_feature.go index daa722b..9cd41bc 100644 --- a/pkg/rubik/rubik_feature.go +++ b/pkg/rubik/rubik_feature.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file for defining rubik support features + +// Package rubik package rubik import ( @@ -7,31 +21,31 @@ import ( var defaultRubikFeature = []services.FeatureSpec{ { - Name: feature.FeaturePreemption, + Name: feature.PreemptionFeature, Default: true, }, { - Name: feature.FeatureDynCache, + Name: feature.DynCacheFeature, Default: true, }, { - Name: feature.FeatureIOLimit, + Name: feature.IOLimitFeature, Default: true, }, { - Name: feature.FeatureIOCost, + Name: feature.IOCostFeature, Default: true, }, { - Name: feature.FeatureDynMemory, + Name: feature.DynMemoryFeature, Default: true, }, { - Name: feature.FeatureQuotaBurst, + Name: feature.QuotaBurstFeature, Default: true, }, { - Name: feature.FeatureQuotaTurbo, + Name: feature.QuotaTurboFeature, Default: true, }, } diff --git a/pkg/services/component.go b/pkg/services/component.go index 9254cc8..08f28b4 100644 --- a/pkg/services/component.go +++ b/pkg/services/component.go @@ -30,13 +30,13 @@ type ServiceComponent func(name string) error var ( serviceComponents = map[string]ServiceComponent{ - feature.FeaturePreemption: initPreemptionFactory, - feature.FeatureDynCache: initDynCacheFactory, - feature.FeatureIOLimit: initIOLimitFactory, - feature.FeatureIOCost: initIOCostFactory, - feature.FeatureDynMemory: initDynCacheFactory, - feature.FeatureQuotaBurst: initQuotaBurstFactory, - feature.FeatureQuotaTurbo: initQuotaTurboFactory, + feature.PreemptionFeature: initPreemptionFactory, + feature.DynCacheFeature: initDynCacheFactory, + feature.IOLimitFeature: initIOLimitFactory, + feature.IOCostFeature: initIOCostFactory, + feature.DynMemoryFeature: initDynCacheFactory, + feature.QuotaBurstFeature: initQuotaBurstFactory, + feature.QuotaTurboFeature: initQuotaTurboFactory, } ) diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index 5a4fda7..a0460a3 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -202,7 +202,8 @@ func (c *DynCache) Validate() error { if c.Config.L3Percent.Low > c.Config.L3Percent.Mid || c.Config.L3Percent.Mid > c.Config.L3Percent.High { return fmt.Errorf("cache limit config L3Percent does not satisfy constraint low<=mid<=high") } - if c.Config.MemBandPercent.Low > c.Config.MemBandPercent.Mid || c.Config.MemBandPercent.Mid > c.Config.MemBandPercent.High { + if c.Config.MemBandPercent.Low > c.Config.MemBandPercent.Mid || + c.Config.MemBandPercent.Mid > c.Config.MemBandPercent.High { return fmt.Errorf("cache limit config MemBandPercent does not satisfy constraint low<=mid<=high") } return nil diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go index 3d07890..6b71000 100644 --- a/pkg/services/helper/service_base.go +++ b/pkg/services/helper/service_base.go @@ -19,6 +19,7 @@ import ( "fmt" "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/core/typedef" ) @@ -35,11 +36,13 @@ func (s *ServiceBase) SetConfig(ConfigHandler) error { // PreStarter is an interface for calling a collection of methods when the service is pre-started func (s *ServiceBase) PreStart(api.Viewer) error { + log.Warnf("this interface is not implemented.") return nil } // Terminator is an interface that calls a collection of methods when the service terminates func (s *ServiceBase) Terminate(api.Viewer) error { + log.Warnf("this interface is not implemented.") return nil } @@ -53,20 +56,23 @@ func (s *ServiceBase) Run(context.Context) {} // Stop to stop runner func (s *ServiceBase) Stop() error { - return fmt.Errorf("i am not runner") + return fmt.Errorf("this interface is not implemented") } // AddPod to deal the event of adding a pod. func (s *ServiceBase) AddPod(*typedef.PodInfo) error { + log.Warnf("this interface is not implemented.") return nil } // UpdatePod to deal the pod update event. func (S *ServiceBase) UpdatePod(old, new *typedef.PodInfo) error { + log.Warnf("this interface is not implemented.") return nil } // DeletePod to deal the pod deletion event. func (s *ServiceBase) DeletePod(*typedef.PodInfo) error { + log.Warnf("this interface is not implemented.") return nil } diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 959e3fe..8336e70 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -71,24 +71,27 @@ var ( nodeName string ) +// IOCostFactory is the factory of IOCost. type IOCostFactory struct { ObjName string } +// Name to get the IOCost factory name. func (i IOCostFactory) Name() string { return "IOCostFactory" } +// NewObj to create object of IOCost. func (i IOCostFactory) NewObj() (interface{}, error) { - if IOCostSupport() { + if ioCostSupport() { nodeName = os.Getenv(constant.NodeNameEnvKey) return &IOCost{name: i.ObjName}, nil } return nil, fmt.Errorf("this machine not support iocost") } -// IOCostSupport tell if the os support iocost. -func IOCostSupport() bool { +// ioCostSupport tell if the os support iocost. +func ioCostSupport() bool { cmdLine, err := os.ReadFile("/proc/cmdline") if err != nil { log.Warnf("get /pro/cmdline error") @@ -110,6 +113,7 @@ func (io *IOCost) ID() string { return io.name } +// SetConfig to config nodeConfig configure func (io *IOCost) SetConfig(f helper.ConfigHandler) error { var nodeConfigs []NodeConfig var nodeConfig *NodeConfig @@ -147,6 +151,7 @@ func (io *IOCost) loadConfig(nodeConfig *NodeConfig) error { } +// PreStart is the pre-start action func (io *IOCost) PreStart(viewer api.Viewer) error { return io.dealExistedPods(viewer) } @@ -161,21 +166,25 @@ func (b *IOCost) Terminate(viewer api.Viewer) error { func (b *IOCost) dealExistedPods(viewer api.Viewer) error { pods := viewer.ListPodsWithOptions() for _, pod := range pods { - b.configPodIOCostWeight(pod) + if err := b.configPodIOCostWeight(pod); err != nil { + log.Errorf("config pod iocost failed, err:%v", err) + } } return nil } -func (b *IOCost) AddFunc(podInfo *typedef.PodInfo) error { +// AddPod to deal the event of adding a pod. +func (b *IOCost) AddPod(podInfo *typedef.PodInfo) error { return b.configPodIOCostWeight(podInfo) } -func (b *IOCost) UpdateFunc(old, new *typedef.PodInfo) error { +// UpdatePod to deal the pod update event. +func (b *IOCost) UpdatePod(old, new *typedef.PodInfo) error { return b.configPodIOCostWeight(new) } -// deal with deleted pod. -func (b *IOCost) DeleteFunc(podInfo *typedef.PodInfo) error { +// DeletePod to deal the pod deletion event. +func (b *IOCost) DeletePod(podInfo *typedef.PodInfo) error { return nil } diff --git a/pkg/services/iocost/iocost_origin.go b/pkg/services/iocost/iocost_origin.go index e5c56dd..d37109f 100644 --- a/pkg/services/iocost/iocost_origin.go +++ b/pkg/services/iocost/iocost_origin.go @@ -23,13 +23,13 @@ import ( const ( // iocost model file - iocostModelFile = "blkio.cost.model" + iocostModelFile = "blkio.cost.model" // iocost weight file iocostWeightFile = "blkio.cost.weight" // iocost weight qos file - iocostQosFile = "blkio.cost.qos" - // cgroup writeback file - wbBlkioinoFile = "memory.wb_blkio_ino" + iocostQosFile = "blkio.cost.qos" + // cgroup writeback file + wbBlkioinoFile = "memory.wb_blkio_ino" ) // ConfigIOCostQoS for config iocost qos. diff --git a/pkg/services/iocost/utils.go b/pkg/services/iocost/utils.go index 0b01afc..7b6fd9f 100644 --- a/pkg/services/iocost/utils.go +++ b/pkg/services/iocost/utils.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to implement blkio system call + +// Package iocost package iocost import ( diff --git a/pkg/services/iolimit/iolimit.go b/pkg/services/iolimit/iolimit.go index 7dd7f13..376e026 100644 --- a/pkg/services/iolimit/iolimit.go +++ b/pkg/services/iolimit/iolimit.go @@ -1,3 +1,17 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: hanchao +// Create: 2023-03-11 +// Description: This file is used to implement iolimit + +// Package iolimit package iolimit import ( @@ -10,13 +24,15 @@ type DeviceConfig struct { DeviceValue string `json:"value,omitempty"` } -// IOLimitAnnoConfig defines the annotation config of iolimit. -type IOLimitAnnoConfig struct { +// annoConfig defines the annotation config of iolimit. +/* +type annoConfig struct { DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` } +*/ // IOLimit is the class of IOLimit. type IOLimit struct { diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index 81de0ba..ceb3916 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -83,8 +83,8 @@ func (q *Preemption) PreStart(viewer api.Viewer) error { return nil } -// AddFunc implement add function when pod is added in k8s -func (q *Preemption) AddFunc(pod *typedef.PodInfo) error { +// AddPod implement add function when pod is added in k8s +func (q *Preemption) AddPod(pod *typedef.PodInfo) error { if err := q.SetQoSLevel(pod); err != nil { return err } @@ -94,8 +94,8 @@ func (q *Preemption) AddFunc(pod *typedef.PodInfo) error { return nil } -// UpdateFunc implement update function when pod info is changed -func (q *Preemption) UpdateFunc(old, new *typedef.PodInfo) error { +// UpdatePod implement update function when pod info is changed +func (q *Preemption) UpdatePod(old, new *typedef.PodInfo) error { oldQos, newQos := getQoSLevel(old), getQoSLevel(new) switch { case newQos == oldQos: @@ -112,8 +112,8 @@ func (q *Preemption) UpdateFunc(old, new *typedef.PodInfo) error { return nil } -// DeleteFunc implement delete function when pod is deleted by k8s -func (q *Preemption) DeleteFunc(pod *typedef.PodInfo) error { +// DeletePod implement delete function when pod is deleted by k8s +func (q *Preemption) DeletePod(pod *typedef.PodInfo) error { return nil } diff --git a/pkg/services/preemption/preemption_test.go b/pkg/services/preemption/preemption_test.go index 746704c..68cf741 100644 --- a/pkg/services/preemption/preemption_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -113,8 +113,8 @@ func TestPreemptionAddFunc(t *testing.T) { tt.preHook(tt.args.new) } if tt.args.new != nil { - if err := q.AddFunc(tt.args.new.PodInfo); (err != nil) != tt.wantErr { - t.Errorf("QoS.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + if err := q.AddPod(tt.args.new.PodInfo); (err != nil) != tt.wantErr { + t.Errorf("QoS.AddPod() error = %v, wantErr %v", err, tt.wantErr) } } @@ -123,7 +123,7 @@ func TestPreemptionAddFunc(t *testing.T) { } } -func TestPreemptionUpdateFunc(t *testing.T) { +func TestPreemptionUpdatePod(t *testing.T) { var updateFuncTC = []test{ { name: "TC1-online to offline", @@ -170,8 +170,8 @@ func TestPreemptionUpdateFunc(t *testing.T) { if tt.preHook != nil { tt.args.new = tt.preHook(tt.args.old) } - if err := q.UpdateFunc(tt.args.old.PodInfo, tt.args.new.PodInfo); (err != nil) != tt.wantErr { - t.Errorf("QoS.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) + if err := q.UpdatePod(tt.args.old.PodInfo, tt.args.new.PodInfo); (err != nil) != tt.wantErr { + t.Errorf("QoS.UpdatePod() error = %v, wantErr %v", err, tt.wantErr) } tt.args.new.CleanPath().OrDie() tt.args.old.CleanPath().OrDie() diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index 7d2d45e..75f9bbf 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -57,21 +57,21 @@ func (b *Burst) ID() string { return b.name } -// AddFunc implement add function when pod is added in k8s -func (conf *Burst) AddFunc(podInfo *typedef.PodInfo) error { +// AddPod implement add function when pod is added in k8s +func (conf *Burst) AddPod(podInfo *typedef.PodInfo) error { return setPodQuotaBurst(podInfo) } -// UpdateFunc implement update function when pod info is changed -func (conf *Burst) UpdateFunc(oldPod, newPod *typedef.PodInfo) error { +// UpdatePod implement update function when pod info is changed +func (conf *Burst) UpdatePod(oldPod, newPod *typedef.PodInfo) error { if oldPod.Annotations[constant.QuotaBurstAnnotationKey] == newPod.Annotations[constant.QuotaBurstAnnotationKey] { return nil } return setPodQuotaBurst(newPod) } -// DeleteFunc implement delete function when pod is deleted by k8s -func (conf *Burst) DeleteFunc(podInfo *typedef.PodInfo) error { +// DeletePod implement delete function when pod is deleted by k8s +func (conf *Burst) DeletePod(podInfo *typedef.PodInfo) error { return nil } diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go index 5525edb..0e2ec08 100644 --- a/pkg/services/quotaburst/quotaburst_test.go +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -37,8 +37,8 @@ var ( cfsPeriodUs = &cgroup.Key{SubSys: "cpu", FileName: "cpu.cfs_period_us"} ) -// TestBurst_AddFunc tests AddFunc -func TestBurst_AddFunc(t *testing.T) { +// TestBurst_AddPod tests AddPod +func TestBurst_AddPod(t *testing.T) { type args struct { pod *try.FakePod burst string @@ -159,8 +159,8 @@ func TestBurst_AddFunc(t *testing.T) { if tt.args.burst != "" { tt.args.pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.args.burst } - if err := conf.AddFunc(tt.args.pod.PodInfo); (err != nil) != tt.wantErr { - t.Errorf("Burst.AddFunc() error = %v, wantErr %v", err, tt.wantErr) + if err := conf.AddPod(tt.args.pod.PodInfo); (err != nil) != tt.wantErr { + t.Errorf("Burst.AddPod() error = %v, wantErr %v", err, tt.wantErr) } tt.args.pod.CleanPath().OrDie() }) @@ -173,13 +173,13 @@ func TestOther(t *testing.T) { const tcName = "TC1-test Other" t.Run(tcName, func(t *testing.T) { got := Burst{name: moduleName} - assert.NoError(t, got.DeleteFunc(&typedef.PodInfo{})) + assert.NoError(t, got.DeletePod(&typedef.PodInfo{})) assert.Equal(t, moduleName, got.ID()) }) } -// TestBurst_UpdateFunc tests UpdateFunc -func TestBurst_UpdateFunc(t *testing.T) { +// TestBurst_UpdatePod tests UpdatePod +func TestBurst_UpdatePod(t *testing.T) { type args struct { oldPod *typedef.PodInfo newPod *typedef.PodInfo @@ -221,8 +221,8 @@ func TestBurst_UpdateFunc(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { conf := Burst{name: moduleName} - if err := conf.UpdateFunc(tt.args.oldPod, tt.args.newPod); (err != nil) != tt.wantErr { - t.Errorf("Burst.UpdateFunc() error = %v, wantErr %v", err, tt.wantErr) + if err := conf.UpdatePod(tt.args.oldPod, tt.args.newPod); (err != nil) != tt.wantErr { + t.Errorf("Burst.UpdatePod() error = %v, wantErr %v", err, tt.wantErr) } }) } -- Gitee From 1681ae986f283c9e1fd0f989efcc3a11be5e2c00 Mon Sep 17 00:00:00 2001 From: hanchao Date: Sat, 11 Mar 2023 22:01:42 +0800 Subject: [PATCH 58/73] bugfix: fix that invoke SetConfig.ConfigHandler failed --- pkg/services/dynCache/dynCache.go | 44 +++++++++++++-------- pkg/services/dynCache/dynCache_init.go | 16 ++++---- pkg/services/dynCache/dynCache_init_test.go | 42 ++++++++++---------- pkg/services/dynCache/dynCache_test.go | 10 ++--- pkg/services/dynCache/dynamic.go | 6 +-- pkg/services/dynCache/dynamic_test.go | 42 ++++++++++---------- pkg/services/dynCache/sync.go | 4 +- pkg/services/dynCache/sync_test.go | 6 +-- pkg/services/iocost/iocost.go | 2 +- pkg/services/preemption/preemption.go | 11 ++++-- pkg/services/preemption/preemption_test.go | 2 +- 11 files changed, 100 insertions(+), 85 deletions(-) diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index a0460a3..042ab96 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -88,7 +88,7 @@ type Config struct { // DynCache is cache limit service structure type DynCache struct { helper.ServiceBase - *Config + config *Config Attr *Attr Viewer api.Viewer Name string `json:"-"` @@ -136,7 +136,7 @@ func NewDynCache(name string) *DynCache { MaxMiss: defaultMaxMiss, MinMiss: defaultMinMiss, }, - Config: &Config{ + config: &Config{ DefaultLimitMode: modeStatic, DefaultResctrlDir: defaultResctrlDir, DefaultPidNameSpace: defaultPidNameSpace, @@ -174,36 +174,48 @@ func (c *DynCache) ID() string { // Run implement service run function func (c *DynCache) Run(ctx context.Context) { go wait.Until(c.SyncCacheLimit, time.Second, ctx.Done()) - wait.Until(c.StartDynamic, time.Millisecond*time.Duration(c.Config.AdjustInterval), ctx.Done()) + wait.Until(c.StartDynamic, time.Millisecond*time.Duration(c.config.AdjustInterval), ctx.Done()) +} + +func (c *DynCache) SetConfig(f helper.ConfigHandler) error { + var config Config + if err := f(c.Name, &config); err != nil { + return err + } + if err := config.Validate(); err != nil { + return err + } + c.config = &config + return nil } // Validate validate service's config -func (c *DynCache) Validate() error { - defaultLimitMode := c.Config.DefaultLimitMode +func (conf *Config) Validate() error { + defaultLimitMode := conf.DefaultLimitMode if defaultLimitMode != modeStatic && defaultLimitMode != modeDynamic { return fmt.Errorf("invalid cache limit mode: %s, should be %s or %s", - c.Config.DefaultLimitMode, modeStatic, modeDynamic) + conf.DefaultLimitMode, modeStatic, modeDynamic) } - if c.Config.AdjustInterval < minAdjustInterval || c.Config.AdjustInterval > maxAdjustInterval { + if conf.AdjustInterval < minAdjustInterval || conf.AdjustInterval > maxAdjustInterval { return fmt.Errorf("adjust interval %d out of range [%d,%d]", - c.Config.AdjustInterval, minAdjustInterval, maxAdjustInterval) + conf.AdjustInterval, minAdjustInterval, maxAdjustInterval) } - if c.Config.PerfDuration < minPerfDur || c.Config.PerfDuration > maxPerfDur { - return fmt.Errorf("perf duration %d out of range [%d,%d]", c.Config.PerfDuration, minPerfDur, maxPerfDur) + if conf.PerfDuration < minPerfDur || conf.PerfDuration > maxPerfDur { + return fmt.Errorf("perf duration %d out of range [%d,%d]", conf.PerfDuration, minPerfDur, maxPerfDur) } for _, per := range []int{ - c.Config.L3Percent.Low, c.Config.L3Percent.Mid, - c.Config.L3Percent.High, c.Config.MemBandPercent.Low, - c.Config.MemBandPercent.Mid, c.Config.MemBandPercent.High} { + conf.L3Percent.Low, conf.L3Percent.Mid, + conf.L3Percent.High, conf.MemBandPercent.Low, + conf.MemBandPercent.Mid, conf.MemBandPercent.High} { if per < minPercent || per > maxPercent { return fmt.Errorf("cache limit percentage %d out of range [%d,%d]", per, minPercent, maxPercent) } } - if c.Config.L3Percent.Low > c.Config.L3Percent.Mid || c.Config.L3Percent.Mid > c.Config.L3Percent.High { + if conf.L3Percent.Low > conf.L3Percent.Mid || conf.L3Percent.Mid > conf.L3Percent.High { return fmt.Errorf("cache limit config L3Percent does not satisfy constraint low<=mid<=high") } - if c.Config.MemBandPercent.Low > c.Config.MemBandPercent.Mid || - c.Config.MemBandPercent.Mid > c.Config.MemBandPercent.High { + if conf.MemBandPercent.Low > conf.MemBandPercent.Mid || + conf.MemBandPercent.Mid > conf.MemBandPercent.High { return fmt.Errorf("cache limit config MemBandPercent does not satisfy constraint low<=mid<=high") } return nil diff --git a/pkg/services/dynCache/dynCache_init.go b/pkg/services/dynCache/dynCache_init.go index 0f4d36b..55ef642 100644 --- a/pkg/services/dynCache/dynCache_init.go +++ b/pkg/services/dynCache/dynCache_init.go @@ -48,10 +48,10 @@ func (c *DynCache) InitCacheLimitDir() error { if !perf.Support() { return fmt.Errorf("perf event need by service %s not supported", c.ID()) } - if err := checkHostPidns(c.Config.DefaultPidNameSpace); err != nil { + if err := checkHostPidns(c.config.DefaultPidNameSpace); err != nil { return err } - if err := checkResctrlPath(c.Config.DefaultResctrlDir); err != nil { + if err := checkResctrlPath(c.config.DefaultResctrlDir); err != nil { return err } numaNum, err := getNUMANum(c.Attr.NumaNodeDir) @@ -59,14 +59,14 @@ func (c *DynCache) InitCacheLimitDir() error { return fmt.Errorf("get NUMA nodes number error: %v", err) } c.Attr.NumaNum = numaNum - c.Attr.L3PercentDynamic = c.Config.L3Percent.Low - c.Attr.MemBandPercentDynamic = c.Config.MemBandPercent.Low + c.Attr.L3PercentDynamic = c.config.L3Percent.Low + c.Attr.MemBandPercentDynamic = c.config.MemBandPercent.Low cacheLimitList := []*limitSet{ c.newCacheLimitSet(levelDynamic, c.Attr.L3PercentDynamic, c.Attr.MemBandPercentDynamic), - c.newCacheLimitSet(levelLow, c.Config.L3Percent.Low, c.Config.MemBandPercent.Low), - c.newCacheLimitSet(levelMiddle, c.Config.L3Percent.Mid, c.Config.MemBandPercent.Mid), - c.newCacheLimitSet(levelHigh, c.Config.L3Percent.High, c.Config.MemBandPercent.High), + c.newCacheLimitSet(levelLow, c.config.L3Percent.Low, c.config.MemBandPercent.Low), + c.newCacheLimitSet(levelMiddle, c.config.L3Percent.Mid, c.config.MemBandPercent.Mid), + c.newCacheLimitSet(levelHigh, c.config.L3Percent.High, c.config.MemBandPercent.High), c.newCacheLimitSet(levelMax, defaultL3PercentMax, defaultMbPercentMax), } @@ -85,7 +85,7 @@ func (c *DynCache) newCacheLimitSet(level string, l3Per, mbPer int) *limitSet { level: level, l3Percent: l3Per, mbPercent: mbPer, - dir: filepath.Join(filepath.Clean(c.Config.DefaultResctrlDir), resctrlDirPrefix+level), + dir: filepath.Join(filepath.Clean(c.config.DefaultResctrlDir), resctrlDirPrefix+level), } } diff --git a/pkg/services/dynCache/dynCache_init_test.go b/pkg/services/dynCache/dynCache_init_test.go index 779de6f..28f5c84 100644 --- a/pkg/services/dynCache/dynCache_init_test.go +++ b/pkg/services/dynCache/dynCache_init_test.go @@ -64,9 +64,9 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr: &Attr{}, }, preHook: func(t *testing.T, c *DynCache) { - c.Config.DefaultResctrlDir = try.GenTestDir().String() - c.Config.DefaultLimitMode = modeStatic - setMaskFile(t, c.Config.DefaultResctrlDir, "7fff") + c.config.DefaultResctrlDir = try.GenTestDir().String() + c.config.DefaultLimitMode = modeStatic + setMaskFile(t, c.config.DefaultResctrlDir, "7fff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) @@ -80,11 +80,11 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { "rubik_dynamic": "L3:0=7;1=7;2=7;3=7\nMB:0=10;1=10;2=10;3=10\n", } for level, expect := range resctrlLevelMap { - schemataFile := filepath.Join(c.DefaultResctrlDir, level, schemataFileName) + schemataFile := filepath.Join(c.config.DefaultResctrlDir, level, schemataFileName) content := try.ReadFile(schemataFile).String() assert.Equal(t, expect, content) } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -101,10 +101,10 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") os.Symlink(pidNameSpaceFileOri.String(), pidNameSpace) - c.Config.DefaultPidNameSpace = pidNameSpace + c.config.DefaultPidNameSpace = pidNameSpace }, postHook: func(t *testing.T, c *DynCache) { - try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) + try.RemoveAll(filepath.Dir(c.config.DefaultPidNameSpace)) }, }, { @@ -120,10 +120,10 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") os.Link(pidNameSpaceFileOri.String(), pidNameSpace) - c.Config.DefaultPidNameSpace = pidNameSpace + c.config.DefaultPidNameSpace = pidNameSpace }, postHook: func(t *testing.T, c *DynCache) { - try.RemoveAll(filepath.Dir(c.DefaultPidNameSpace)) + try.RemoveAll(filepath.Dir(c.config.DefaultPidNameSpace)) }, }, { @@ -134,7 +134,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr: &Attr{}, }, preHook: func(t *testing.T, c *DynCache) { - c.Config.DefaultResctrlDir = "/resctrl/path/is/not/exist" + c.config.DefaultResctrlDir = "/resctrl/path/is/not/exist" }, }, { @@ -145,10 +145,10 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr: &Attr{}, }, preHook: func(t *testing.T, c *DynCache) { - c.Config.DefaultResctrlDir = try.GenTestDir().String() + c.config.DefaultResctrlDir = try.GenTestDir().String() }, postHook: func(t *testing.T, c *DynCache) { - try.RemoveAll(c.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) }, }, { @@ -170,15 +170,15 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr: &Attr{}, }, preHook: func(t *testing.T, c *DynCache) { - c.Config.DefaultResctrlDir = try.GenTestDir().String() - c.Config.DefaultLimitMode = modeStatic - setMaskFile(t, c.Config.DefaultResctrlDir, "") + c.config.DefaultResctrlDir = try.GenTestDir().String() + c.config.DefaultLimitMode = modeStatic + setMaskFile(t, c.config.DefaultResctrlDir, "") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 0) }, postHook: func(t *testing.T, c *DynCache) { - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -189,15 +189,15 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr: &Attr{}, }, preHook: func(t *testing.T, c *DynCache) { - c.Config.DefaultResctrlDir = try.GenTestDir().String() - c.Config.DefaultLimitMode = modeStatic - setMaskFile(t, c.Config.DefaultResctrlDir, "1") + c.config.DefaultResctrlDir = try.GenTestDir().String() + c.config.DefaultLimitMode = modeStatic + setMaskFile(t, c.config.DefaultResctrlDir, "1") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 0) }, postHook: func(t *testing.T, c *DynCache) { - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -205,7 +205,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, } diff --git a/pkg/services/dynCache/dynCache_test.go b/pkg/services/dynCache/dynCache_test.go index 35874b6..e1d0459 100644 --- a/pkg/services/dynCache/dynCache_test.go +++ b/pkg/services/dynCache/dynCache_test.go @@ -178,11 +178,11 @@ func TestCacheLimit_Validate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, } - err := c.Validate() + err := c.config.Validate() if (err != nil) != tt.wantErr { t.Errorf("CacheLimit.Validate() error = %v, wantErr %v", err, tt.wantErr) } @@ -207,7 +207,7 @@ func TestNewCacheLimit(t *testing.T) { MaxMiss: defaultMaxMiss, MinMiss: defaultMinMiss, }, - Config: &Config{ + config: &Config{ DefaultLimitMode: modeStatic, DefaultResctrlDir: defaultResctrlDir, DefaultPidNameSpace: defaultPidNameSpace, @@ -266,7 +266,7 @@ func TestCacheLimit_PreStart(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, Name: tt.fields.Name, @@ -301,7 +301,7 @@ func TestCacheLimit_ID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, Name: tt.fields.Name, diff --git a/pkg/services/dynCache/dynamic.go b/pkg/services/dynCache/dynamic.go index e73601a..c199a38 100644 --- a/pkg/services/dynCache/dynamic.go +++ b/pkg/services/dynCache/dynamic.go @@ -38,7 +38,7 @@ func (c *DynCache) StartDynamic() { limiter := c.newCacheLimitSet(levelDynamic, c.Attr.L3PercentDynamic, c.Attr.MemBandPercentDynamic) for _, p := range c.listOnlinePods() { - cacheMiss, llcMiss := getPodCacheMiss(p, c.Config.PerfDuration) + cacheMiss, llcMiss := getPodCacheMiss(p, c.config.PerfDuration) if cacheMiss >= c.Attr.MaxMiss || llcMiss >= c.Attr.MaxMiss { log.Infof("online pod %v cache miss: %v LLC miss: %v exceeds maxmiss, lower offline cache limit", p.UID, cacheMiss, llcMiss) @@ -99,8 +99,8 @@ func (c *DynCache) flush(limitSet *limitSet, step int) error { return value } - l3 := nextPercent(c.Attr.L3PercentDynamic, c.Config.L3Percent.Low, c.Config.L3Percent.High, step) - mb := nextPercent(c.Attr.MemBandPercentDynamic, c.Config.MemBandPercent.Low, c.Config.MemBandPercent.High, step) + l3 := nextPercent(c.Attr.L3PercentDynamic, c.config.L3Percent.Low, c.config.L3Percent.High, step) + mb := nextPercent(c.Attr.MemBandPercentDynamic, c.config.MemBandPercent.Low, c.config.MemBandPercent.High, step) if c.Attr.L3PercentDynamic == l3 && c.Attr.MemBandPercentDynamic == mb { return nil } diff --git a/pkg/services/dynCache/dynamic_test.go b/pkg/services/dynCache/dynamic_test.go index 3093ae3..4a5c12a 100644 --- a/pkg/services/dynCache/dynamic_test.go +++ b/pkg/services/dynCache/dynamic_test.go @@ -66,9 +66,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) - c.Config.DefaultResctrlDir = resctrlDir - c.Config.DefaultLimitMode = modeDynamic - c.Config.PerfDuration = 10 + c.config.DefaultResctrlDir = resctrlDir + c.config.DefaultLimitMode = modeDynamic + c.config.PerfDuration = 10 for _, pod := range fakePods { if pod.Annotations[constant.PriorityAnnotationKey] == "true" { pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" @@ -81,7 +81,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, pod := range fakePods { pod.CleanPath() } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -108,9 +108,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) - c.Config.DefaultResctrlDir = resctrlDir - c.Config.DefaultLimitMode = modeDynamic - c.Config.PerfDuration = 10 + c.config.DefaultResctrlDir = resctrlDir + c.config.DefaultLimitMode = modeDynamic + c.config.PerfDuration = 10 for _, pod := range fakePods { if pod.Annotations[constant.PriorityAnnotationKey] == "true" { pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" @@ -123,7 +123,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, pod := range fakePods { pod.CleanPath() } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -150,9 +150,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) - c.Config.DefaultResctrlDir = resctrlDir - c.Config.DefaultLimitMode = modeDynamic - c.Config.PerfDuration = 10 + c.config.DefaultResctrlDir = resctrlDir + c.config.DefaultLimitMode = modeDynamic + c.config.PerfDuration = 10 for _, pod := range fakePods { if pod.Annotations[constant.PriorityAnnotationKey] == "true" { pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" @@ -165,7 +165,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, pod := range fakePods { pod.CleanPath() } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -192,9 +192,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) - c.Config.DefaultResctrlDir = resctrlDir - c.Config.DefaultLimitMode = modeDynamic - c.Config.PerfDuration = 10 + c.config.DefaultResctrlDir = resctrlDir + c.config.DefaultLimitMode = modeDynamic + c.config.PerfDuration = 10 for _, pod := range fakePods { if pod.Annotations[constant.PriorityAnnotationKey] == "true" { pod.Annotations[constant.CacheLimitAnnotationKey] = "dynamic" @@ -207,7 +207,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, pod := range fakePods { pod.CleanPath() } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -234,9 +234,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir genNumaNodes(c.Attr.NumaNodeDir, 4) - c.Config.DefaultResctrlDir = resctrlDir - c.Config.DefaultLimitMode = modeDynamic - c.Config.PerfDuration = 10 + c.config.DefaultResctrlDir = resctrlDir + c.config.DefaultLimitMode = modeDynamic + c.config.PerfDuration = 10 for _, pod := range fakePods { if pod.Annotations[constant.PriorityAnnotationKey] == "true" { pod.Annotations[constant.CacheLimitAnnotationKey] = "static" @@ -249,7 +249,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, pod := range fakePods { pod.CleanPath() } - try.RemoveAll(c.Config.DefaultResctrlDir) + try.RemoveAll(c.config.DefaultResctrlDir) try.RemoveAll(c.Attr.NumaNodeDir) }, }, @@ -257,7 +257,7 @@ func TestCacheLimit_StartDynamic(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, } diff --git a/pkg/services/dynCache/sync.go b/pkg/services/dynCache/sync.go index 4855438..a9b2fb3 100644 --- a/pkg/services/dynCache/sync.go +++ b/pkg/services/dynCache/sync.go @@ -78,7 +78,7 @@ func (c *DynCache) writeTasksToResctrl(pod *typedef.PodInfo) error { return nil } - resctrlTaskFile := filepath.Join(c.Config.DefaultResctrlDir, + resctrlTaskFile := filepath.Join(c.config.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks") for _, task := range taskList { if err := util.WriteFile(resctrlTaskFile, task); err != nil { @@ -96,7 +96,7 @@ func (c *DynCache) writeTasksToResctrl(pod *typedef.PodInfo) error { // syncLevel sync cache limit level func (c *DynCache) syncLevel(pod *typedef.PodInfo) error { if pod.Annotations[constant.CacheLimitAnnotationKey] == "" { - if c.Config.DefaultLimitMode == modeStatic { + if c.config.DefaultLimitMode == modeStatic { pod.Annotations[constant.CacheLimitAnnotationKey] = levelMax } else { pod.Annotations[constant.CacheLimitAnnotationKey] = levelDynamic diff --git a/pkg/services/dynCache/sync_test.go b/pkg/services/dynCache/sync_test.go index 0f43a89..0a12579 100644 --- a/pkg/services/dynCache/sync_test.go +++ b/pkg/services/dynCache/sync_test.go @@ -129,7 +129,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager - c.Config.DefaultLimitMode = levelDynamic + c.config.DefaultLimitMode = levelDynamic }, }, { @@ -229,14 +229,14 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" } c.Viewer = manager - c.Config.DefaultResctrlDir = "/dev/null" + c.config.DefaultResctrlDir = "/dev/null" }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &DynCache{ - Config: tt.fields.Config, + config: tt.fields.Config, Attr: tt.fields.Attr, Name: tt.fields.Name, } diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 8336e70..11fe078 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -117,7 +117,7 @@ func (io *IOCost) ID() string { func (io *IOCost) SetConfig(f helper.ConfigHandler) error { var nodeConfigs []NodeConfig var nodeConfig *NodeConfig - if err := f(io.name, nodeConfigs); err != nil { + if err := f(io.name, &nodeConfigs); err != nil { return err } diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index ceb3916..4c91537 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -66,7 +66,10 @@ func (q *Preemption) ID() string { // SetConfig to config Preemption configure func (q *Preemption) SetConfig(f helper.ConfigHandler) error { var c PreemptionConfig - if err := f(q.name, c); err != nil { + if err := f(q.name, &c); err != nil { + return err + } + if err := c.Validate(); err != nil { return err } q.config = c @@ -178,11 +181,11 @@ func getQoSLevel(pod *typedef.PodInfo) int { } // Validate will validate the qos service config -func (q *Preemption) Validate() error { - if len(q.config.Resource) == 0 { +func (conf *PreemptionConfig) Validate() error { + if len(conf.Resource) == 0 { return fmt.Errorf("empty qos config") } - for _, r := range q.config.Resource { + for _, r := range conf.Resource { if _, ok := supportCgroupTypes[r]; !ok { return fmt.Errorf("not support sub system %s", r) } diff --git a/pkg/services/preemption/preemption_test.go b/pkg/services/preemption/preemption_test.go index 68cf741..d494ff0 100644 --- a/pkg/services/preemption/preemption_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -208,7 +208,7 @@ func TestPreemptionValidate(t *testing.T) { name: tt.fields.Name, config: tt.fields.Config, } - if err := q.Validate(); (err != nil) != tt.wantErr { + if err := q.config.Validate(); (err != nil) != tt.wantErr { t.Errorf("QoS.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) -- Gitee From 7c604b684de01bfd42da9939eedae2971fa25966 Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 11 Mar 2023 21:47:16 +0800 Subject: [PATCH 59/73] test: add quotaturbo DT test coverage: 95.1% of statements Signed-off-by: vegbir --- pkg/services/quotaturbo/quotaturbo_test.go | 485 +++++++++++++++++++++ 1 file changed, 485 insertions(+) create mode 100644 pkg/services/quotaturbo/quotaturbo_test.go diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go new file mode 100644 index 0000000..0599050 --- /dev/null +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -0,0 +1,485 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-11 +// Description: This file is used for testing quotaturbo.go + +package quotaturbo + +import ( + "context" + "fmt" + "math" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" + "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/podmanager" + "isula.org/rubik/pkg/services/helper" + "isula.org/rubik/test/try" +) + +func sameQuota(t *testing.T, path string, want int64) bool { + const cfsQuotaUsFileName = "cpu.cfs_quota_us" + data, err := util.ReadSmallFile(filepath.Join(path, cfsQuotaUsFileName)) + if err != nil { + assert.NoError(t, err) + return false + } + quota, err := util.ParseInt64(strings.ReplaceAll(string(data), "\n", "")) + if err != nil { + assert.NoError(t, err) + return false + } + if quota != want { + return false + } + return true +} + +// TestQuotaTurbo_Terminate tests Terminate function +func TestQuotaTurbo_Terminate(t *testing.T) { + const ( + fooContName = "Foo" + barContName = "Bar" + podUID = "testPod1" + wrongPodQuota = "600000" + wrongFooQuota = "300000" + wrongBarQuota = "200000" + podPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/" + fooPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon1" + barPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/testCon2" + ) + + var ( + fooCont = &typedef.ContainerInfo{ + Name: fooContName, + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: make(typedef.ResourceMap), + } + barCont = &typedef.ContainerInfo{ + Name: barContName, + ID: "testCon2", + CgroupPath: "kubepods/testPod1/testCon2", + LimitResources: make(typedef.ResourceMap), + } + pod = &typedef.PodInfo{ + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + IDContainersMap: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + barCont.ID: barCont, + }, + } + tests = []struct { + postfunc func(t *testing.T) + fooCPULimit float64 + barCPULimit float64 + name string + }{ + { + name: "TC1-one unlimited container is existed", + fooCPULimit: 2, + barCPULimit: 0, + postfunc: func(t *testing.T) { + var ( + unlimited int64 = -1 + correctFooQuota int64 = 200000 + ) + assert.True(t, sameQuota(t, podPath, unlimited)) + assert.True(t, sameQuota(t, fooPath, correctFooQuota)) + assert.True(t, sameQuota(t, barPath, unlimited)) + }, + }, + { + name: "TC2-all containers are unlimited", + fooCPULimit: 2, + barCPULimit: 1, + postfunc: func(t *testing.T) { + var ( + correctPodQuota int64 = 300000 + correctFooQuota int64 = 200000 + correctBarQuota int64 = 100000 + ) + assert.True(t, sameQuota(t, podPath, correctPodQuota)) + assert.True(t, sameQuota(t, fooPath, correctFooQuota)) + assert.True(t, sameQuota(t, barPath, correctBarQuota)) + }, + }, + { + name: "TC3-all containers are limited", + fooCPULimit: 0, + barCPULimit: 0, + postfunc: func(t *testing.T) { + var unLimited int64 = -1 + assert.True(t, sameQuota(t, podPath, unLimited)) + assert.True(t, sameQuota(t, fooPath, unLimited)) + assert.True(t, sameQuota(t, barPath, unLimited)) + }, + }, + } + ) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + pm = &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + podUID: pod, + }, + }, + } + qt = &QuotaTurbo{ + Viewer: pm, + } + ) + + try.MkdirAll(fooPath, constant.DefaultDirMode) + try.MkdirAll(barPath, constant.DefaultDirMode) + defer func() { + try.RemoveAll(podPath) + }() + + fooCont.LimitResources[typedef.ResourceCPU] = tt.fooCPULimit + barCont.LimitResources[typedef.ResourceCPU] = tt.barCPULimit + + assert.NoError(t, pod.SetCgroupAttr(cpuQuotaKey, wrongPodQuota)) + assert.NoError(t, fooCont.SetCgroupAttr(cpuQuotaKey, wrongFooQuota)) + assert.NoError(t, barCont.SetCgroupAttr(cpuQuotaKey, wrongBarQuota)) + qt.Terminate(pm) + tt.postfunc(t) + }) + } +} + +// TestQuotaTurbo_PreStart tests PreStart +func TestQuotaTurbo_PreStart(t *testing.T) { + const ( + fooContName = "Foo" + podUID = "testPod1" + podPath = "/sys/fs/cgroup/cpu/kubepods/testPod1/" + ) + var ( + fooCont = &typedef.ContainerInfo{ + Name: fooContName, + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: make(typedef.ResourceMap), + } + pm = &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + podUID: { + UID: podUID, + CgroupPath: "kubepods/testPod1", + IDContainersMap: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + }, + }, + }, + }, + } + name = "quotaturbo" + qt = NewQuotaTurbo(name) + ) + testName := "TC1- test Prestart" + t.Run(testName, func(t *testing.T) { + qt.PreStart(pm) + try.MkdirAll(podPath, constant.DefaultDirMode) + defer try.RemoveAll(podPath) + qt.PreStart(pm) + pm.Pods.Pods[podUID].IDContainersMap[fooCont.ID]. + LimitResources[typedef.ResourceCPU] = math.Min(1, float64(runtime.NumCPU())-1) + qt.PreStart(pm) + }) +} + +// TestConfig_Validate test Validate function +func TestConfig_Validate(t *testing.T) { + type fields struct { + HighWaterMark int + AlarmWaterMark int + SyncInterval int + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "TC1-alarmWaterMark is less or equal to highWaterMark", + fields: fields{ + HighWaterMark: 100, + AlarmWaterMark: 60, + SyncInterval: 1000, + }, + wantErr: true, + }, + { + name: "TC2-highWater mark exceed the max quota turbo water mark(100)", + fields: fields{ + HighWaterMark: 1000, + AlarmWaterMark: 100000, + SyncInterval: 1000, + }, + wantErr: true, + }, + { + name: "TC3-sync interval out of range(100-10000)", + fields: fields{ + HighWaterMark: 60, + AlarmWaterMark: 80, + SyncInterval: 1, + }, + wantErr: true, + }, + { + name: "TC4-normal case", + fields: fields{ + HighWaterMark: 60, + AlarmWaterMark: 100, + SyncInterval: 1000, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf := &Config{ + HighWaterMark: tt.fields.HighWaterMark, + AlarmWaterMark: tt.fields.AlarmWaterMark, + SyncInterval: tt.fields.SyncInterval, + } + if err := conf.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Config.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestQuotaTurbo_SetConfig tests SetConfig +func TestQuotaTurbo_SetConfig(t *testing.T) { + const name = "quotaturbo" + type args struct { + f helper.ConfigHandler + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "TC1-error function", + args: args{ + f: func(configName string, d interface{}) error { return fmt.Errorf("error occures") }, + }, + wantErr: true, + }, + { + name: "TC2-success", + args: args{ + f: func(configName string, d interface{}) error { return nil }, + }, + wantErr: false, + }, + { + name: "TC3-invalid config", + args: args{ + f: func(configName string, d interface{}) error { + c := d.(*Config) + c.AlarmWaterMark = 101 + return nil + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qt := NewQuotaTurbo(name) + if err := qt.SetConfig(tt.args.f); (err != nil) != tt.wantErr { + t.Errorf("QuotaTurbo.SetConfig() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestQuotaTurbo_Other tests other function +func TestQuotaTurbo_Other(t *testing.T) { + const name = "quotaturbo" + tests := []struct { + name string + want bool + }{ + { + name: "TC1-test other", + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &QuotaTurboFactory{ObjName: name} + f.Name() + instance, err := f.NewObj() + assert.NoError(t, err) + qt := instance.(*QuotaTurbo) + if got := qt.IsRunner(); got != tt.want { + t.Errorf("QuotaTurbo.IsRunner() = %v, want %v", got, tt.want) + } + qt.ID() + + }) + } +} + +// TestQuotaTurbo_AdjustQuota tests AdjustQuota +func TestQuotaTurbo_AdjustQuota(t *testing.T) { + const ( + name = "quotaturbo" + cpuPeriodFile = "cpu.cfs_period_us" + cpuQuotaFile = "cpu.cfs_quota_us" + cpuUsageFile = "cpuacct.usage" + cpuStatFile = "cpu.stat" + stat = `nr_periods 1 + nr_throttled 1 + throttled_time 1 + ` + quota = "200000" + period = "100000" + usage = "1234567" + ) + var ( + fooCont = &typedef.ContainerInfo{ + Name: "Foo", + ID: "testCon1", + CgroupPath: "kubepods/testPod1/testCon1", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: math.Min(2, float64(runtime.NumCPU()-1)), + }, + } + barCont = &typedef.ContainerInfo{ + Name: "Bar", + ID: "testCon2", + CgroupPath: "kubepods/testPod2/testCon2", + LimitResources: typedef.ResourceMap{ + typedef.ResourceCPU: math.Min(2, float64(runtime.NumCPU()-1)), + }, + } + preEnv = func(contPath string) { + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuPeriodFile), period) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuQuotaFile), quota) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpuacct", contPath, cpuUsageFile), usage) + try.WriteFile(filepath.Join(constant.TmpTestDir, "cpu", contPath, cpuStatFile), stat) + } + ) + type args struct { + conts map[string]*typedef.ContainerInfo + } + tests := []struct { + name string + args args + pre func(t *testing.T, qt *QuotaTurbo) + post func(t *testing.T) + }{ + { + name: "TC1-fail add foo container & remove bar container successfully", + args: args{ + conts: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + }, + }, + pre: func(t *testing.T, qt *QuotaTurbo) { + preEnv(barCont.CgroupPath) + assert.NoError(t, qt.client.AddCgroup(barCont.CgroupPath, barCont.LimitResources[typedef.ResourceCPU])) + assert.Equal(t, 1, len(qt.client.GetAllCgroup())) + }, + post: func(t *testing.T) { + try.RemoveAll(constant.TmpTestDir) + }, + }, + { + name: "TC2-no container add & remove bar container successfully", + args: args{ + conts: map[string]*typedef.ContainerInfo{ + fooCont.ID: fooCont, + barCont.ID: barCont, + }, + }, + pre: func(t *testing.T, qt *QuotaTurbo) { + preEnv(barCont.CgroupPath) + preEnv(fooCont.CgroupPath) + assert.NoError(t, qt.client.AddCgroup(barCont.CgroupPath, barCont.LimitResources[typedef.ResourceCPU])) + assert.NoError(t, qt.client.AddCgroup(fooCont.CgroupPath, fooCont.LimitResources[typedef.ResourceCPU])) + assert.Equal(t, 2, len(qt.client.GetAllCgroup())) + }, + post: func(t *testing.T) { + try.RemoveAll(constant.TmpTestDir) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qt := NewQuotaTurbo(name) + qt.client.SetCgroupRoot(constant.TmpTestDir) + if tt.pre != nil { + tt.pre(t, qt) + } + qt.AdjustQuota(tt.args.conts) + if tt.post != nil { + tt.post(t) + } + }) + } +} + +// TestQuotaTurbo_Run tests run +func TestQuotaTurbo_Run(t *testing.T) { + const name = "quotaturbo" + var fooPod = &typedef.PodInfo{ + Name: "Foo", + UID: "testPod1", + CgroupPath: "kubepods/testPod1", + Annotations: map[string]string{ + constant.QuotaAnnotationKey: "true", + }, + } + tests := []struct { + name string + }{ + { + name: "TC1-run and cancel", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qt := NewQuotaTurbo(name) + pm := &podmanager.PodManager{ + Pods: &podmanager.PodCache{ + Pods: map[string]*typedef.PodInfo{ + fooPod.UID: fooPod, + }, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + qt.Viewer = pm + go qt.Run(ctx) + time.Sleep(time.Millisecond * 200) + cancel() + }) + } +} -- Gitee From 779529a73c943477407430c79d35a9dba9c378a2 Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Sat, 11 Mar 2023 22:11:39 +0800 Subject: [PATCH 60/73] fix: fix config not be transported to service Signed-off-by: DCCooper <1866858@gmail.com> --- pkg/services/dynCache/dynCache.go | 50 ++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index 042ab96..72bb0e9 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -127,6 +127,26 @@ func (i DynCacheFactory) NewObj() (interface{}, error) { return NewDynCache(i.ObjName), nil } +func newConfig() *Config { + return &Config{ + DefaultLimitMode: modeStatic, + DefaultResctrlDir: defaultResctrlDir, + DefaultPidNameSpace: defaultPidNameSpace, + AdjustInterval: defaultAdInt, + PerfDuration: defaultPerfDur, + L3Percent: MultiLvlPercent{ + Low: defaultLowL3, + Mid: defaultMidL3, + High: defaultHighL3, + }, + MemBandPercent: MultiLvlPercent{ + Low: defaultLowMB, + Mid: defaultMidMB, + High: defaultHighMB, + }, + } +} + // NewDynCache return cache limit instance with default settings func NewDynCache(name string) *DynCache { return &DynCache{ @@ -136,26 +156,15 @@ func NewDynCache(name string) *DynCache { MaxMiss: defaultMaxMiss, MinMiss: defaultMinMiss, }, - config: &Config{ - DefaultLimitMode: modeStatic, - DefaultResctrlDir: defaultResctrlDir, - DefaultPidNameSpace: defaultPidNameSpace, - AdjustInterval: defaultAdInt, - PerfDuration: defaultPerfDur, - L3Percent: MultiLvlPercent{ - Low: defaultLowL3, - Mid: defaultMidL3, - High: defaultHighL3, - }, - MemBandPercent: MultiLvlPercent{ - Low: defaultLowMB, - Mid: defaultMidMB, - High: defaultHighMB, - }, - }, + config: newConfig(), } } +// IsRunner returns true that tells other dynCache is a persistent service +func (c *DynCache) IsRunner() bool { + return true +} + // PreStart will do some pre-setting actions func (c *DynCache) PreStart(viewer api.Viewer) error { c.Viewer = viewer @@ -177,15 +186,16 @@ func (c *DynCache) Run(ctx context.Context) { wait.Until(c.StartDynamic, time.Millisecond*time.Duration(c.config.AdjustInterval), ctx.Done()) } +// SetConfig sets and checks Config func (c *DynCache) SetConfig(f helper.ConfigHandler) error { - var config Config - if err := f(c.Name, &config); err != nil { + config := newConfig() + if err := f(c.Name, config); err != nil { return err } if err := config.Validate(); err != nil { return err } - c.config = &config + c.config = config return nil } -- Gitee From 6d8d79a37da2f9363fa4572c6452de4db4d629b8 Mon Sep 17 00:00:00 2001 From: hanchao Date: Sun, 12 Mar 2023 16:53:34 +0800 Subject: [PATCH 61/73] bugfix: fix the configuration name of iocost --- pkg/services/iocost/iocost.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 11fe078..beff2f2 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -58,7 +58,7 @@ type IOCostConfig struct { // NodeConfig define the config of node, include iocost type NodeConfig struct { NodeName string `json:"nodeName,omitempty"` - IOCostConfig []IOCostConfig `json:"iocostConfig,omitempty"` + IOCostConfig []IOCostConfig `json:"config,omitempty"` } // IOCost for iocost class @@ -115,6 +115,10 @@ func (io *IOCost) ID() string { // SetConfig to config nodeConfig configure func (io *IOCost) SetConfig(f helper.ConfigHandler) error { + if f == nil { + return fmt.Errorf("config handler function callback is nil") + } + var nodeConfigs []NodeConfig var nodeConfig *NodeConfig if err := f(io.name, &nodeConfigs); err != nil { @@ -148,7 +152,6 @@ func (io *IOCost) loadConfig(nodeConfig *NodeConfig) error { io.configIOCost(nodeConfig.IOCostConfig) return nil - } // PreStart is the pre-start action -- Gitee From 24b2f7f6fa7b9ed18a4983158bf4336edb3b8547 Mon Sep 17 00:00:00 2001 From: wujing Date: Mon, 13 Mar 2023 17:28:00 +0800 Subject: [PATCH 62/73] refactor: optimize the log of the preemption module Signed-off-by: wujing --- pkg/services/preemption/preemption.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index 4c91537..410049e 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -80,7 +80,7 @@ func (q *Preemption) SetConfig(f helper.ConfigHandler) error { func (q *Preemption) PreStart(viewer api.Viewer) error { for _, pod := range viewer.ListPodsWithOptions() { if err := q.SetQoSLevel(pod); err != nil { - log.Errorf("error prestart pod %v: %v", pod.Name, err) + log.Errorf("failed to set the qos level for the previously started pod %v: %v", pod.Name, err) } } return nil @@ -104,11 +104,11 @@ func (q *Preemption) UpdatePod(old, new *typedef.PodInfo) error { case newQos == oldQos: return nil case newQos > oldQos: - return fmt.Errorf("not support change qos level from low to high") + return fmt.Errorf("does not support pod qos level setting from low to high") default: if err := q.ValidateConfig(new); err != nil { if err := q.SetQoSLevel(new); err != nil { - return fmt.Errorf("update qos for pod %s(%s) failed: %v", new.Name, new.UID, err) + return fmt.Errorf("failed to update the qos level of pod %s(%s): %v", new.Name, new.UID, err) } } } @@ -126,11 +126,11 @@ func (q *Preemption) ValidateConfig(pod *typedef.PodInfo) error { targetLevel := getQoSLevel(pod) for _, r := range q.config.Resource { if err := pod.GetCgroupAttr(supportCgroupTypes[r]).Expect(targetLevel); err != nil { - return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) + return fmt.Errorf("failed to validate the qos level configuration of pod %s: %v", pod.Name, err) } for _, container := range pod.IDContainersMap { if err := container.GetCgroupAttr(supportCgroupTypes[r]).Expect(targetLevel); err != nil { - return fmt.Errorf("failed to validate pod %s: %v", pod.Name, err) + return fmt.Errorf("failed to validate the qos level configuration of container %s: %v", pod.Name, err) } } } @@ -140,11 +140,11 @@ func (q *Preemption) ValidateConfig(pod *typedef.PodInfo) error { // SetQoSLevel set pod and all containers' qos level within it func (q *Preemption) SetQoSLevel(pod *typedef.PodInfo) error { if pod == nil { - return fmt.Errorf("pod info is empty") + return fmt.Errorf("empty pod info") } qosLevel := getQoSLevel(pod) if qosLevel == constant.Online { - log.Debugf("pod %s already online", pod.Name) + log.Infof("pod %s has already been set to online", pod.Name) return nil } @@ -158,7 +158,7 @@ func (q *Preemption) SetQoSLevel(pod *typedef.PodInfo) error { } } } - log.Debugf("set pod %s(%s) qos level %d ok", pod.Name, pod.UID, qosLevel) + log.Infof("the qos level of pod %s(%s) is set to %d successfully", pod.Name, pod.UID, qosLevel) return nil } @@ -187,7 +187,7 @@ func (conf *PreemptionConfig) Validate() error { } for _, r := range conf.Resource { if _, ok := supportCgroupTypes[r]; !ok { - return fmt.Errorf("not support sub system %s", r) + return fmt.Errorf("does not support setting the %s subsystem", r) } } return nil -- Gitee From 99d90ac2869e081b7996a36b3080d2a6372245b8 Mon Sep 17 00:00:00 2001 From: vegbir Date: Wed, 15 Mar 2023 20:43:08 +0800 Subject: [PATCH 63/73] optimize: log optimize & rubik exit when prestart failed 1. rubik exits if any service fail to prestart, and terminate prestarted services 2. print all parameters of each service before starting, including user-defined parameters and default parameters. 3. add container adding/removing logs for quotaTurbo Signed-off-by: vegbir --- pkg/config/jsonparser.go | 12 ++++++ pkg/config/parserfactory.go | 1 + pkg/rubik/servicemanager.go | 53 ++++++++++++++++++--------- pkg/services/dynCache/dynCache.go | 5 +++ pkg/services/helper/service_base.go | 5 +++ pkg/services/quotaturbo/quotaturbo.go | 9 +++++ pkg/services/service.go | 2 + 7 files changed, 70 insertions(+), 17 deletions(-) diff --git a/pkg/config/jsonparser.go b/pkg/config/jsonparser.go index 17fa586..93c8ccb 100644 --- a/pkg/config/jsonparser.go +++ b/pkg/config/jsonparser.go @@ -57,3 +57,15 @@ func (p *jsonParser) UnmarshalSubConfig(data interface{}, v interface{}) error { // 2. convert json string to struct return json.Unmarshal(jsonString, v) } + +// MarshalIndent serializes interface to string +func (p *jsonParser) MarshalIndent(data interface{}, prefix, indent string) (string, error) { + if data == nil { + return "", nil + } + res, err := json.MarshalIndent(data, prefix, indent) + if err != nil { + return "", err + } + return string(res), nil +} diff --git a/pkg/config/parserfactory.go b/pkg/config/parserfactory.go index 8a533da..d7a74c7 100644 --- a/pkg/config/parserfactory.go +++ b/pkg/config/parserfactory.go @@ -23,6 +23,7 @@ type ( ConfigParser interface { ParseConfig(data []byte) (map[string]interface{}, error) UnmarshalSubConfig(data interface{}, v interface{}) error + MarshalIndent(v interface{}, prefix, indent string) (string, error) } ) diff --git a/pkg/rubik/servicemanager.go b/pkg/rubik/servicemanager.go index 35aeff3..55e6882 100644 --- a/pkg/rubik/servicemanager.go +++ b/pkg/rubik/servicemanager.go @@ -71,9 +71,20 @@ func (manager *ServiceManager) InitServices(features []string, serviceConfig map return fmt.Errorf("set configuration failed, err:%v", err) } + conf, err := parser.MarshalIndent(s.GetConfig(), "", "\t") + if err != nil { + return fmt.Errorf("fail to get service %v configuration: %v", s.ID(), err) + } + if err := manager.AddRunningService(feature, s); err != nil { return err } + + if len(conf) != 0 { + log.Infof("service %v will run with configuration:\n%v", s.ID(), conf) + } else { + log.Infof("service %v will run", s.ID()) + } } return nil } @@ -114,18 +125,21 @@ func (manager *ServiceManager) EventTypes() []typedef.EventType { } // terminatingRunningServices handles services exits during the setup and exit phases -func (manager *ServiceManager) terminatingRunningServices() error { - for _, s := range manager.RunningServices { +func terminatingServices(serviceMap map[string]services.Service, viewer api.Viewer) { + for name, s := range serviceMap { if s.IsRunner() { if err := s.Stop(); err != nil { - log.Errorf("stop service err:%v", err) + log.Errorf("fail to stop service %v: %v", name, err) + } else { + log.Infof("stop service %v successfully", name) } } - if err := s.Terminate(manager.Viewer); err != nil { - log.Errorf("terminate service err:%v", err) + if err := s.Terminate(viewer); err != nil { + log.Errorf("fail to terminate service %v: %v", name, err) + } else { + log.Infof("terminate service %v successfully", name) } } - return nil } // Setup pre-starts services, such as preparing the environment, etc. @@ -135,20 +149,22 @@ func (manager *ServiceManager) Setup(v api.Viewer) error { return fmt.Errorf("viewer should not be empty") } manager.Viewer = v - manager.Lock() - var unavailable []string + + var preStarted = make(map[string]services.Service, 0) + manager.RLock() for name, s := range manager.RunningServices { + /* + Try to prestart the service. If any service fails, rubik exits + and invokes the terminate function to terminate the prestarted service. + */ if err := s.PreStart(manager.Viewer); err != nil { - log.Errorf("PreStart failed:%v", err) - unavailable = append(unavailable, name) + terminatingServices(preStarted, manager.Viewer) + return fmt.Errorf("fail to preStart service %v: %v", name, err) } + preStarted[name] = s + log.Infof("service %v pre-start successfully", name) } - - for _, name := range unavailable { - delete(manager.RunningServices, name) - log.Infof("service %v is unavailable", name) - } - manager.Unlock() + manager.RUnlock() return nil } @@ -185,7 +201,10 @@ func (manager *ServiceManager) Start(ctx context.Context) { // Stop terminates the running service func (manager *ServiceManager) Stop() error { - return manager.terminatingRunningServices() + manager.RLock() + terminatingServices(manager.RunningServices, manager.Viewer) + manager.RUnlock() + return nil } // addFunc handles pod addition events diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dynCache/dynCache.go index 72bb0e9..2eec701 100644 --- a/pkg/services/dynCache/dynCache.go +++ b/pkg/services/dynCache/dynCache.go @@ -180,6 +180,11 @@ func (c *DynCache) ID() string { return c.Name } +// GetConfig returns Config +func (c *DynCache) GetConfig() interface{} { + return c.config +} + // Run implement service run function func (c *DynCache) Run(ctx context.Context) { go wait.Until(c.SyncCacheLimit, time.Second, ctx.Done()) diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go index 6b71000..e0e4c60 100644 --- a/pkg/services/helper/service_base.go +++ b/pkg/services/helper/service_base.go @@ -76,3 +76,8 @@ func (s *ServiceBase) DeletePod(*typedef.PodInfo) error { log.Warnf("this interface is not implemented.") return nil } + +// Run to start runner +func (s *ServiceBase) GetConfig() interface{} { + return nil +} diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index b6c203d..6dfd5d0 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -110,6 +110,8 @@ func (qt *QuotaTurbo) syncCgroups(conts map[string]*typedef.ContainerInfo) { if _, found := conts[id]; !found { if err := qt.client.RemoveCgroup(path); err != nil { log.Errorf("error removing container %v: %v", id, err) + } else { + log.Infof("remove container %v", id) } } } @@ -124,6 +126,8 @@ func (qt *QuotaTurbo) syncCgroups(conts map[string]*typedef.ContainerInfo) { // add container to quotaturbo if err := qt.client.AddCgroup(cont.CgroupPath, cont.LimitResources[typedef.ResourceCPU]); err != nil { log.Errorf("error adding container %v: %v", cont.Name, err) + } else { + log.Infof("add container %v", id) } } } @@ -185,6 +189,11 @@ func (qt *QuotaTurbo) SetConfig(f helper.ConfigHandler) error { return nil } +// GetConfig returns Config +func (qt *QuotaTurbo) GetConfig() interface{} { + return qt.conf +} + // IsRunner returns true that tells other quotaTurbo is a persistent service func (qt *QuotaTurbo) IsRunner() bool { return true diff --git a/pkg/services/service.go b/pkg/services/service.go index b18f607..9ba07d8 100644 --- a/pkg/services/service.go +++ b/pkg/services/service.go @@ -52,6 +52,8 @@ type Service interface { ID() string // SetConfig is an interface that invoke the ConfigHandler to obtain the corresponding configuration. SetConfig(helper.ConfigHandler) error + // GetConfig is an interface for obtaining service running configurations. + GetConfig() interface{} // PreStarter is an interface for calling a collection of methods when the service is pre-started PreStart(api.Viewer) error // Terminator is an interface that calls a collection of methods when the service terminates -- Gitee From aef1012b808c79250fccdc07091318791dc215e0 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 16 Mar 2023 20:25:47 +0800 Subject: [PATCH 64/73] cleancode: clean rubik's code 1. Eliminate the magic number 2. Correct wrong comments 3. The dynCache package was renamed dyncache, in line with the package name specification 4. Delete redundant code 5. Export variables to add comments 6. Fix long lines of code Signed-off-by: vegbir --- pkg/common/constant/constant.go | 1 - pkg/common/util/conversion.go | 3 +- pkg/common/util/conversion_test.go | 2 +- pkg/core/typedef/cgroup/common.go | 3 ++ pkg/core/typedef/podinfo.go | 4 +-- pkg/core/typedef/rawpod.go | 1 + pkg/feature/feature.go | 15 ++++++--- pkg/rubik/import.go | 2 +- pkg/rubik/rubik.go | 2 +- pkg/rubik/servicemanager.go | 11 ++++--- pkg/services/component.go | 2 +- .../dynache_init.go} | 0 .../{dynCache => dyncache}/dynamic.go | 9 +++-- .../{dynCache => dyncache}/dynamic_test.go | 15 ++++++--- .../dynCache.go => dyncache/dyncache.go} | 0 .../dyncache_init_test.go} | 7 ++-- .../dyncache_test.go} | 33 ++++++++++--------- pkg/services/{dynCache => dyncache}/sync.go | 0 .../{dynCache => dyncache}/sync_test.go | 18 ++++++---- pkg/services/helper/service_base.go | 4 +-- pkg/services/iolimit/iolimit.go | 10 ------ pkg/services/preemption/preemption_test.go | 3 +- pkg/services/quotaturbo/quotaturbo_test.go | 11 ++++--- pkg/version/version.go | 7 ++-- 24 files changed, 94 insertions(+), 69 deletions(-) rename pkg/services/{dynCache/dynCache_init.go => dyncache/dynache_init.go} (100%) rename pkg/services/{dynCache => dyncache}/dynamic.go (94%) rename pkg/services/{dynCache => dyncache}/dynamic_test.go (95%) rename pkg/services/{dynCache/dynCache.go => dyncache/dyncache.go} (100%) rename pkg/services/{dynCache/dynCache_init_test.go => dyncache/dyncache_init_test.go} (96%) rename pkg/services/{dynCache/dynCache_test.go => dyncache/dyncache_test.go} (90%) rename pkg/services/{dynCache => dyncache}/sync.go (100%) rename pkg/services/{dynCache => dyncache}/sync_test.go (93%) diff --git a/pkg/common/constant/constant.go b/pkg/common/constant/constant.go index de8282c..c2a5ce4 100644 --- a/pkg/common/constant/constant.go +++ b/pkg/common/constant/constant.go @@ -96,7 +96,6 @@ const ( const ( Offline = -1 Online = 0 - // TODO: add more level ) // cgroup file name diff --git a/pkg/common/util/conversion.go b/pkg/common/util/conversion.go index 4cbc496..b45dd35 100644 --- a/pkg/common/util/conversion.go +++ b/pkg/common/util/conversion.go @@ -105,6 +105,7 @@ func DeepCopy(value interface{}) interface{} { newSlice := reflect.MakeSlice(typ, val.Len(), val.Cap()) reflect.Copy(newSlice, val) return newSlice.Interface() + default: + return value } - return value } diff --git a/pkg/common/util/conversion_test.go b/pkg/common/util/conversion_test.go index 29b7e8c..3cea35f 100644 --- a/pkg/common/util/conversion_test.go +++ b/pkg/common/util/conversion_test.go @@ -179,7 +179,7 @@ func TestDeepCopy(t *testing.T) { oldSlice := []string{"a", "b", "c"} newSlice := DeepCopy(oldSlice).([]string) - for i, _ := range newSlice { + for i := range newSlice { newSlice[i] += "z" } assert.Equal(t, oldSlice, []string{"a", "b", "c"}) diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 396a67a..8856962 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -26,6 +26,7 @@ import ( var rootDir = constant.DefaultCgroupRoot +// AbsoluteCgroupPath returns the absolute path of the cgroup func AbsoluteCgroupPath(elem ...string) string { elem = append([]string{rootDir}, elem...) return filepath.Join(elem...) @@ -150,11 +151,13 @@ func (attr *Attr) CPUStat() (*CPUStat, error) { return NewCPUStat(attr.Value) } +// Hierarchy is used to represent a cgroup path type Hierarchy struct { MountPoint string Path string } +// NewHierarchy creates a Hierarchy instance func NewHierarchy(mountPoint, path string) *Hierarchy { return &Hierarchy{ MountPoint: mountPoint, diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index 7f725f3..7272faf 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -53,13 +53,13 @@ func (pod *PodInfo) DeepCopy() *PodInfo { ) // nil is different from empty value in golang if pod.IDContainersMap != nil { - contMap = make(map[string]*ContainerInfo, len(pod.IDContainersMap)) + contMap = make(map[string]*ContainerInfo) for id, cont := range pod.IDContainersMap { contMap[id] = cont.DeepCopy() } } if pod.Annotations != nil { - annoMap = make(map[string]string, len(pod.Annotations)) + annoMap = make(map[string]string) for k, v := range pod.Annotations { annoMap[k] = v } diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index a96a2e1..59dfb59 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -198,6 +198,7 @@ func (cont *RawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { return iterator(&cont.spec.Resources.Requests), iterator(&cont.spec.Resources.Limits) } +// DeepCopy returns the deep copy object of ResourceMap func (m ResourceMap) DeepCopy() ResourceMap { if m == nil { return nil diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 59c41bf..d5e9e2e 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -15,11 +15,18 @@ package feature const ( + // PreemptionFeature is the Preemption feature name PreemptionFeature = "preemption" - DynCacheFeature = "dynCache" - IOLimitFeature = "ioLimit" - IOCostFeature = "ioCost" - DynMemoryFeature = "dynMemory" + // DynCacheFeature is the DynCache feature name + DynCacheFeature = "dynCache" + // IOLimitFeature is the IOLimit feature name + IOLimitFeature = "ioLimit" + // IOCostFeature is the IOCost feature name + IOCostFeature = "ioCost" + // DynMemoryFeature is the DynMemory feature name + DynMemoryFeature = "dynMemory" + // QuotaBurstFeature is the QuotaBurst feature name QuotaBurstFeature = "quotaBurst" + // QuotaTurboFeature is the QuotaTurbo feature name QuotaTurboFeature = "quotaTurbo" ) diff --git a/pkg/rubik/import.go b/pkg/rubik/import.go index a35ada8..a083f9a 100644 --- a/pkg/rubik/import.go +++ b/pkg/rubik/import.go @@ -16,7 +16,7 @@ package rubik import ( // introduce packages to auto register service - _ "isula.org/rubik/pkg/services/dynCache" + _ "isula.org/rubik/pkg/services/dyncache" _ "isula.org/rubik/pkg/services/iocost" _ "isula.org/rubik/pkg/services/preemption" _ "isula.org/rubik/pkg/services/quotaburst" diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index 3c94bac..f6506d2 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -47,7 +47,7 @@ type Agent struct { func NewAgent(cfg *config.Config) (*Agent, error) { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) serviceManager := NewServiceManager() - if err := serviceManager.InitServices( cfg.Agent.EnabledFeatures, + if err := serviceManager.InitServices(cfg.Agent.EnabledFeatures, cfg.UnwarpServiceConfig(), cfg); err != nil { return nil, err } diff --git a/pkg/rubik/servicemanager.go b/pkg/rubik/servicemanager.go index 55e6882..e4bf6ce 100644 --- a/pkg/rubik/servicemanager.go +++ b/pkg/rubik/servicemanager.go @@ -51,7 +51,8 @@ func NewServiceManager() *ServiceManager { } // InitServices parses the to-be-run services config and loads them to the ServiceManager -func (manager *ServiceManager) InitServices(features []string, serviceConfig map[string]interface{}, parser config.ConfigParser) error { +func (manager *ServiceManager) InitServices(features []string, + serviceConfig map[string]interface{}, parser config.ConfigParser) error { for _, feature := range features { s, err := services.GetServiceComponent(feature) if err != nil { @@ -216,7 +217,7 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { } const retryCount = 5 - runOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + addOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) for i := 0; i < retryCount; i++ { if err := s.AddPod(podInfo); err != nil { @@ -230,7 +231,7 @@ func (manager *ServiceManager) addFunc(event typedef.Event) { manager.RLock() var wg sync.WaitGroup for _, s := range manager.RunningServices { - go runOnce(s, podInfo.DeepCopy(), &wg) + go addOnce(s, podInfo.DeepCopy(), &wg) } wg.Wait() manager.RUnlock() @@ -273,7 +274,7 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { return } - runOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { + deleteOnce := func(s services.Service, podInfo *typedef.PodInfo, wg *sync.WaitGroup) { wg.Add(1) if err := s.DeletePod(podInfo); err != nil { log.Errorf("service %s delete func failed: %v", s.ID(), err) @@ -283,7 +284,7 @@ func (manager *ServiceManager) deleteFunc(event typedef.Event) { manager.RLock() var wg sync.WaitGroup for _, s := range manager.RunningServices { - go runOnce(s, podInfo.DeepCopy(), &wg) + go deleteOnce(s, podInfo.DeepCopy(), &wg) } wg.Wait() manager.RUnlock() diff --git a/pkg/services/component.go b/pkg/services/component.go index 08f28b4..9f6728b 100644 --- a/pkg/services/component.go +++ b/pkg/services/component.go @@ -16,7 +16,7 @@ package services import ( "isula.org/rubik/pkg/feature" - dyncache "isula.org/rubik/pkg/services/dynCache" + "isula.org/rubik/pkg/services/dyncache" "isula.org/rubik/pkg/services/helper" "isula.org/rubik/pkg/services/iocost" "isula.org/rubik/pkg/services/iolimit" diff --git a/pkg/services/dynCache/dynCache_init.go b/pkg/services/dyncache/dynache_init.go similarity index 100% rename from pkg/services/dynCache/dynCache_init.go rename to pkg/services/dyncache/dynache_init.go diff --git a/pkg/services/dynCache/dynamic.go b/pkg/services/dyncache/dynamic.go similarity index 94% rename from pkg/services/dynCache/dynamic.go rename to pkg/services/dyncache/dynamic.go index c199a38..50336c0 100644 --- a/pkg/services/dynCache/dynamic.go +++ b/pkg/services/dyncache/dynamic.go @@ -70,9 +70,12 @@ func getPodCacheMiss(pod *typedef.PodInfo, perfDu int) (int, int) { if err != nil { return 0, 0 } - - return int(100.0 * float64(stat.CacheMisses) / (1.0 + float64(stat.CacheReferences))), - int(100.0 * float64(stat.LLCMiss) / (1.0 + float64(stat.LLCAccess))) + const ( + probability = 100.0 + bias = 1.0 + ) + return int(probability * float64(stat.CacheMisses) / (bias + float64(stat.CacheReferences))), + int(probability * float64(stat.LLCMiss) / (bias + float64(stat.LLCAccess))) } func (c *DynCache) dynamicExist() bool { diff --git a/pkg/services/dynCache/dynamic_test.go b/pkg/services/dyncache/dynamic_test.go similarity index 95% rename from pkg/services/dynCache/dynamic_test.go rename to pkg/services/dyncache/dynamic_test.go index 4a5c12a..e0630f7 100644 --- a/pkg/services/dynCache/dynamic_test.go +++ b/pkg/services/dyncache/dynamic_test.go @@ -65,7 +65,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) c.config.DefaultResctrlDir = resctrlDir c.config.DefaultLimitMode = modeDynamic c.config.PerfDuration = 10 @@ -107,7 +108,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) c.config.DefaultResctrlDir = resctrlDir c.config.DefaultLimitMode = modeDynamic c.config.PerfDuration = 10 @@ -149,7 +151,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) c.config.DefaultResctrlDir = resctrlDir c.config.DefaultLimitMode = modeDynamic c.config.PerfDuration = 10 @@ -191,7 +194,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) c.config.DefaultResctrlDir = resctrlDir c.config.DefaultLimitMode = modeDynamic c.config.PerfDuration = 10 @@ -233,7 +237,8 @@ func TestCacheLimit_StartDynamic(t *testing.T) { setMaskFile(t, resctrlDir, "3ff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) c.config.DefaultResctrlDir = resctrlDir c.config.DefaultLimitMode = modeDynamic c.config.PerfDuration = 10 diff --git a/pkg/services/dynCache/dynCache.go b/pkg/services/dyncache/dyncache.go similarity index 100% rename from pkg/services/dynCache/dynCache.go rename to pkg/services/dyncache/dyncache.go diff --git a/pkg/services/dynCache/dynCache_init_test.go b/pkg/services/dyncache/dyncache_init_test.go similarity index 96% rename from pkg/services/dynCache/dynCache_init_test.go rename to pkg/services/dyncache/dyncache_init_test.go index 28f5c84..71e5a88 100644 --- a/pkg/services/dynCache/dynCache_init_test.go +++ b/pkg/services/dyncache/dyncache_init_test.go @@ -31,7 +31,8 @@ func setMaskFile(t *testing.T, resctrlDir string, data string) { maskFile := filepath.Join(maskDir, "cbm_mask") try.MkdirAll(maskDir, constant.DefaultDirMode).OrDie() try.WriteFile(maskFile, data).OrDie() - try.WriteFile(filepath.Join(resctrlDir, schemataFileName), "L3:0=7fff;1=7fff;2=7fff;3=7fff\nMB:0=100;1=100;2=100;3=100").OrDie() + try.WriteFile(filepath.Join(resctrlDir, schemataFileName), + "L3:0=7fff;1=7fff;2=7fff;3=7fff\nMB:0=100;1=100;2=100;3=100").OrDie() } func genNumaNodes(path string, num int) { @@ -49,7 +50,6 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { Attr *Attr Name string } - // defaultConfig := genDefaultConfig() tests := []struct { name string fields fields @@ -69,7 +69,8 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { setMaskFile(t, c.config.DefaultResctrlDir, "7fff") numaNodeDir := try.GenTestDir().String() c.Attr.NumaNodeDir = numaNodeDir - genNumaNodes(c.Attr.NumaNodeDir, 4) + const numaNode = 4 + genNumaNodes(c.Attr.NumaNodeDir, numaNode) }, postHook: func(t *testing.T, c *DynCache) { resctrlLevelMap := map[string]string{ diff --git a/pkg/services/dynCache/dynCache_test.go b/pkg/services/dyncache/dyncache_test.go similarity index 90% rename from pkg/services/dynCache/dynCache_test.go rename to pkg/services/dyncache/dyncache_test.go index e1d0459..a4c2bce 100644 --- a/pkg/services/dynCache/dynCache_test.go +++ b/pkg/services/dyncache/dyncache_test.go @@ -28,6 +28,7 @@ const ( ) func TestCacheLimit_Validate(t *testing.T) { + const num2 = 2 type fields struct { Config *Config Attr *Attr @@ -48,12 +49,12 @@ func TestCacheLimit_Validate(t *testing.T) { PerfDuration: minPerfDur + 1, L3Percent: MultiLvlPercent{ Low: minPercent + 1, - Mid: maxPercent/2 + 1, + Mid: maxPercent/num2 + 1, High: maxPercent - 1, }, MemBandPercent: MultiLvlPercent{ Low: minPercent + 1, - Mid: maxPercent/2 + 1, + Mid: maxPercent/num2 + 1, High: maxPercent - 1, }, }, @@ -98,7 +99,7 @@ func TestCacheLimit_Validate(t *testing.T) { fields: fields{ Config: &Config{ DefaultLimitMode: modeStatic, - AdjustInterval: maxAdjustInterval/2 + 1, + AdjustInterval: maxAdjustInterval/num2 + 1, PerfDuration: minPerfDur - 1, }, }, @@ -110,7 +111,7 @@ func TestCacheLimit_Validate(t *testing.T) { fields: fields{ Config: &Config{ DefaultLimitMode: modeStatic, - AdjustInterval: maxAdjustInterval/2 + 1, + AdjustInterval: maxAdjustInterval/num2 + 1, PerfDuration: maxPerfDur + 1, }, }, @@ -122,8 +123,8 @@ func TestCacheLimit_Validate(t *testing.T) { fields: fields{ Config: &Config{ DefaultLimitMode: modeStatic, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, + AdjustInterval: maxAdjustInterval/num2 + 1, + PerfDuration: maxPerfDur/num2 + 1, L3Percent: MultiLvlPercent{ Low: minPerfDur - 1, }, @@ -137,17 +138,17 @@ func TestCacheLimit_Validate(t *testing.T) { fields: fields{ Config: &Config{ DefaultLimitMode: modeStatic, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, + AdjustInterval: maxAdjustInterval/num2 + 1, + PerfDuration: maxPerfDur/num2 + 1, L3Percent: MultiLvlPercent{ - Low: minPercent + 2, + Low: minPercent + num2, Mid: minPercent + 1, High: minPercent + 1, }, MemBandPercent: MultiLvlPercent{ Low: minPercent, Mid: minPercent + 1, - High: minPercent + 2, + High: minPercent + num2, }, }, }, @@ -159,17 +160,17 @@ func TestCacheLimit_Validate(t *testing.T) { fields: fields{ Config: &Config{ DefaultLimitMode: modeStatic, - AdjustInterval: maxAdjustInterval/2 + 1, - PerfDuration: maxPerfDur/2 + 1, + AdjustInterval: maxAdjustInterval/num2 + 1, + PerfDuration: maxPerfDur/num2 + 1, L3Percent: MultiLvlPercent{ Low: minPercent, Mid: minPercent + 1, - High: minPercent + 2, + High: minPercent + num2, }, MemBandPercent: MultiLvlPercent{ Low: minPercent, - Mid: maxPercent/2 + 1, - High: maxPercent / 2, + Mid: maxPercent/num2 + 1, + High: maxPercent / num2, }, }, }, @@ -236,6 +237,7 @@ func TestNewCacheLimit(t *testing.T) { } } +// TestCacheLimit_PreStart tests PreStart func TestCacheLimit_PreStart(t *testing.T) { type fields struct { Config *Config @@ -278,6 +280,7 @@ func TestCacheLimit_PreStart(t *testing.T) { } } +// TestCacheLimit_ID tests ID func TestCacheLimit_ID(t *testing.T) { type fields struct { Config *Config diff --git a/pkg/services/dynCache/sync.go b/pkg/services/dyncache/sync.go similarity index 100% rename from pkg/services/dynCache/sync.go rename to pkg/services/dyncache/sync.go diff --git a/pkg/services/dynCache/sync_test.go b/pkg/services/dyncache/sync_test.go similarity index 93% rename from pkg/services/dynCache/sync_test.go rename to pkg/services/dyncache/sync_test.go index 0a12579..118ac38 100644 --- a/pkg/services/dynCache/sync_test.go +++ b/pkg/services/dyncache/sync_test.go @@ -88,7 +88,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager }, @@ -107,7 +108,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager }, @@ -126,7 +128,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { preHook: func(t *testing.T, c *DynCache, fakePods []*try.FakePod) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager c.config.DefaultLimitMode = levelDynamic @@ -147,7 +150,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "invalid" - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager }, @@ -187,7 +191,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager }, @@ -207,7 +212,8 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" - try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") + try.WriteFile(filepath.Join(defaultConfig.DefaultResctrlDir, + resctrlDirPrefix+pod.Annotations[constant.CacheLimitAnnotationKey], "tasks"), "") } c.Viewer = manager }, diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go index e0e4c60..78675c3 100644 --- a/pkg/services/helper/service_base.go +++ b/pkg/services/helper/service_base.go @@ -34,13 +34,13 @@ func (s *ServiceBase) SetConfig(ConfigHandler) error { return nil } -// PreStarter is an interface for calling a collection of methods when the service is pre-started +// PreStart is an interface for calling a collection of methods when the service is pre-started func (s *ServiceBase) PreStart(api.Viewer) error { log.Warnf("this interface is not implemented.") return nil } -// Terminator is an interface that calls a collection of methods when the service terminates +// Terminate is an interface that calls a collection of methods when the service terminates func (s *ServiceBase) Terminate(api.Viewer) error { log.Warnf("this interface is not implemented.") return nil diff --git a/pkg/services/iolimit/iolimit.go b/pkg/services/iolimit/iolimit.go index 376e026..c5b6039 100644 --- a/pkg/services/iolimit/iolimit.go +++ b/pkg/services/iolimit/iolimit.go @@ -24,16 +24,6 @@ type DeviceConfig struct { DeviceValue string `json:"value,omitempty"` } -// annoConfig defines the annotation config of iolimit. -/* -type annoConfig struct { - DeviceReadBps []DeviceConfig `json:"device_read_bps,omitempty"` - DeviceWriteBps []DeviceConfig `json:"device_write_bps,omitempty"` - DeviceReadIops []DeviceConfig `json:"device_read_iops,omitempty"` - DeviceWriteIops []DeviceConfig `json:"device_write_iops,omitempty"` -} -*/ - // IOLimit is the class of IOLimit. type IOLimit struct { helper.ServiceBase diff --git a/pkg/services/preemption/preemption_test.go b/pkg/services/preemption/preemption_test.go index d494ff0..7ac8f5e 100644 --- a/pkg/services/preemption/preemption_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -51,6 +51,7 @@ var getCommonField = func(subSys []string) fields { } func TestPreemptionAddFunc(t *testing.T) { + const containerNum = 3 var addFuncTC = []test{ { name: "TC1-set offline pod qos ok", @@ -69,7 +70,7 @@ func TestPreemptionAddFunc(t *testing.T) { new: try.GenFakeOnlinePod(map[*cgroup.Key]string{ supportCgroupTypes["cpu"]: "0", supportCgroupTypes["memory"]: "0", - }).WithContainers(3), + }).WithContainers(containerNum), }, }, { diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index 0599050..7bac196 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -362,6 +362,7 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { quota = "200000" period = "100000" usage = "1234567" + minCPU = 2 ) var ( fooCont = &typedef.ContainerInfo{ @@ -369,7 +370,7 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { ID: "testCon1", CgroupPath: "kubepods/testPod1/testCon1", LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: math.Min(2, float64(runtime.NumCPU()-1)), + typedef.ResourceCPU: math.Min(minCPU, float64(runtime.NumCPU()-1)), }, } barCont = &typedef.ContainerInfo{ @@ -377,7 +378,7 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { ID: "testCon2", CgroupPath: "kubepods/testPod2/testCon2", LimitResources: typedef.ResourceMap{ - typedef.ResourceCPU: math.Min(2, float64(runtime.NumCPU()-1)), + typedef.ResourceCPU: math.Min(minCPU, float64(runtime.NumCPU()-1)), }, } preEnv = func(contPath string) { @@ -425,7 +426,8 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { preEnv(fooCont.CgroupPath) assert.NoError(t, qt.client.AddCgroup(barCont.CgroupPath, barCont.LimitResources[typedef.ResourceCPU])) assert.NoError(t, qt.client.AddCgroup(fooCont.CgroupPath, fooCont.LimitResources[typedef.ResourceCPU])) - assert.Equal(t, 2, len(qt.client.GetAllCgroup())) + const cgroupLen = 2 + assert.Equal(t, cgroupLen, len(qt.client.GetAllCgroup())) }, post: func(t *testing.T) { try.RemoveAll(constant.TmpTestDir) @@ -478,7 +480,8 @@ func TestQuotaTurbo_Run(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) qt.Viewer = pm go qt.Run(ctx) - time.Sleep(time.Millisecond * 200) + const sleepTime = time.Millisecond * 200 + time.Sleep(sleepTime) cancel() }) } diff --git a/pkg/version/version.go b/pkg/version/version.go index e285a9b..e8163fb 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -21,9 +21,9 @@ import ( ) var ( - // Version represents rubik version + // Version represents rubik Version Version string - // Release represents rubik release number + // Release represents rubik Release number Release string // GitCommit represents git commit number GitCommit string @@ -33,7 +33,8 @@ var ( func init() { var showVersion bool - if len(os.Args) == 2 && os.Args[1] == "-v" { + const maxArgLen = 2 + if len(os.Args) == maxArgLen && os.Args[1] == "-v" { showVersion = true } -- Gitee From c2a1a16cc151523a8605333dfda34814b57c2937 Mon Sep 17 00:00:00 2001 From: wujing Date: Fri, 17 Mar 2023 15:31:08 +0800 Subject: [PATCH 65/73] style: fix typo Signed-off-by: wujing --- pkg/config/config.go | 4 ++-- pkg/informer/informerfactory.go | 4 ++-- pkg/rubik/rubik.go | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4d4b3de..16ab6e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -118,8 +118,8 @@ func (c *Config) filterNonServiceKeys(name string) bool { return ok } -// UnwarpServiceConfig returns service configuration, indexed by service name -func (c *Config) UnwarpServiceConfig() map[string]interface{} { +// UnwrapServiceConfig returns service configuration, indexed by service name +func (c *Config) UnwrapServiceConfig() map[string]interface{} { serviceConfig := make(map[string]interface{}) for name, conf := range c.Fields { if c.filterNonServiceKeys(name) { diff --git a/pkg/informer/informerfactory.go b/pkg/informer/informerfactory.go index 8a3df5e..69de266 100644 --- a/pkg/informer/informerfactory.go +++ b/pkg/informer/informerfactory.go @@ -48,8 +48,8 @@ func (factory *informerFactory) GetInformerCreator(iType informerType) informerC } } -// GetInfomerFactory returns the Informer factory class entity -func GetInfomerFactory() *informerFactory { +// GetInformerFactory returns the Informer factory class entity +func GetInformerFactory() *informerFactory { if defaultInformerFactory == nil { defaultInformerFactory = &informerFactory{} } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index f6506d2..94dc373 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -48,7 +48,7 @@ func NewAgent(cfg *config.Config) (*Agent, error) { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) serviceManager := NewServiceManager() if err := serviceManager.InitServices(cfg.Agent.EnabledFeatures, - cfg.UnwarpServiceConfig(), cfg); err != nil { + cfg.UnwrapServiceConfig(), cfg); err != nil { return nil, err } a := &Agent{ @@ -77,7 +77,7 @@ func (a *Agent) Run(ctx context.Context) error { // startInformer starts informer to obtain external data func (a *Agent) startInformer(ctx context.Context) error { publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) - informer, err := informer.GetInfomerFactory().GetInformerCreator(informer.APISERVER)(publisher) + informer, err := informer.GetInformerFactory().GetInformerCreator(informer.APISERVER)(publisher) if err != nil { return fmt.Errorf("fail to set informer: %v", err) } @@ -89,7 +89,7 @@ func (a *Agent) startInformer(ctx context.Context) error { return nil } -// stopInformer stops the infomer +// stopInformer stops the informer func (a *Agent) stopInformer() { a.informer.Unsubscribe(a.podManager) } @@ -162,7 +162,7 @@ func Run() int { ctx, cancel := context.WithCancel(context.Background()) // 2. handle external signals - go handelSignals(cancel) + go handleSignals(cancel) // 3. run rubik-agent if err := runAgent(ctx); err != nil { @@ -172,7 +172,7 @@ func Run() int { return constant.NormalExitCode } -func handelSignals(cancel context.CancelFunc) { +func handleSignals(cancel context.CancelFunc) { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT) for sig := range signalChan { -- Gitee From be8c53157361e87321db622d6ad2204f83121f61 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 17 Mar 2023 14:49:06 +0800 Subject: [PATCH 66/73] optimize: remove useless hashicorp/go-multierror We implemented a simple error overlay mechanism by ourselves without using additional packages Signed-off-by: vegbir --- go.mod | 1 - go.sum | 4 - pkg/lib/cpu/quotaturbo/client.go | 6 +- pkg/lib/cpu/quotaturbo/cpuquota.go | 22 +- pkg/lib/cpu/quotaturbo/statusstore.go | 6 +- pkg/lib/cpu/quotaturbo/statusstore_test.go | 1 - vendor/github.com/hashicorp/errwrap/LICENSE | 354 ------------------ vendor/github.com/hashicorp/errwrap/README.md | 89 ----- .../github.com/hashicorp/errwrap/errwrap.go | 169 --------- vendor/github.com/hashicorp/errwrap/go.mod | 1 - .../hashicorp/go-multierror/.travis.yml | 12 - .../hashicorp/go-multierror/LICENSE | 353 ----------------- .../hashicorp/go-multierror/Makefile | 31 -- .../hashicorp/go-multierror/README.md | 97 ----- .../hashicorp/go-multierror/append.go | 41 -- .../hashicorp/go-multierror/flatten.go | 26 -- .../hashicorp/go-multierror/format.go | 27 -- .../github.com/hashicorp/go-multierror/go.mod | 3 - .../github.com/hashicorp/go-multierror/go.sum | 4 - .../hashicorp/go-multierror/multierror.go | 51 --- .../hashicorp/go-multierror/prefix.go | 37 -- .../hashicorp/go-multierror/sort.go | 16 - vendor/modules.txt | 5 - 23 files changed, 20 insertions(+), 1336 deletions(-) delete mode 100644 vendor/github.com/hashicorp/errwrap/LICENSE delete mode 100644 vendor/github.com/hashicorp/errwrap/README.md delete mode 100644 vendor/github.com/hashicorp/errwrap/errwrap.go delete mode 100644 vendor/github.com/hashicorp/errwrap/go.mod delete mode 100644 vendor/github.com/hashicorp/go-multierror/.travis.yml delete mode 100644 vendor/github.com/hashicorp/go-multierror/LICENSE delete mode 100644 vendor/github.com/hashicorp/go-multierror/Makefile delete mode 100644 vendor/github.com/hashicorp/go-multierror/README.md delete mode 100644 vendor/github.com/hashicorp/go-multierror/append.go delete mode 100644 vendor/github.com/hashicorp/go-multierror/flatten.go delete mode 100644 vendor/github.com/hashicorp/go-multierror/format.go delete mode 100644 vendor/github.com/hashicorp/go-multierror/go.mod delete mode 100644 vendor/github.com/hashicorp/go-multierror/go.sum delete mode 100644 vendor/github.com/hashicorp/go-multierror/multierror.go delete mode 100644 vendor/github.com/hashicorp/go-multierror/prefix.go delete mode 100644 vendor/github.com/hashicorp/go-multierror/sort.go diff --git a/go.mod b/go.mod index 6297396..d14af5e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/cyphar/filepath-securejoin v0.0.0-00010101000000-000000000000 github.com/google/uuid v1.1.2 - github.com/hashicorp/go-multierror v1.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.6.1 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd diff --git a/go.sum b/go.sum index 770db0b..4962dc5 100644 --- a/go.sum +++ b/go.sum @@ -125,10 +125,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/pkg/lib/cpu/quotaturbo/client.go b/pkg/lib/cpu/quotaturbo/client.go index a9e40ba..a1e3277 100644 --- a/pkg/lib/cpu/quotaturbo/client.go +++ b/pkg/lib/cpu/quotaturbo/client.go @@ -16,8 +16,6 @@ package quotaturbo import ( "fmt" - - "github.com/hashicorp/go-multierror" ) // Client is quotaTurbo client @@ -44,11 +42,11 @@ func (c *Client) AdjustQuota() error { } var errs error if err := c.updateCPUQuotas(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } c.adjustQuota(c.StatusStore) if err := c.writeQuota(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } return errs } diff --git a/pkg/lib/cpu/quotaturbo/cpuquota.go b/pkg/lib/cpu/quotaturbo/cpuquota.go index 4d89d73..601edfa 100644 --- a/pkg/lib/cpu/quotaturbo/cpuquota.go +++ b/pkg/lib/cpu/quotaturbo/cpuquota.go @@ -19,8 +19,6 @@ import ( "path" "time" - "github.com/hashicorp/go-multierror" - "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -99,16 +97,16 @@ func NewCPUQuota(h *cgroup.Hierarchy, cpuLimit float64) (*CPUQuota, error) { func (c *CPUQuota) update() error { var errs error if err := c.updatePeriod(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } if err := c.updateThrottle(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } if err := c.updateQuota(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } if err := c.updateUsage(); err != nil { - errs = multierror.Append(errs, err) + errs = appendErr(errs, err) } if errs != nil { return errs @@ -251,3 +249,15 @@ func (c *CPUQuota) recoverQuota() error { c.nextQuota = int64(c.cpuLimit * float64(c.period)) return c.writeQuota() } + +func appendErr(errs error, err error) error { + if errs == nil { + return err + } + if err == nil { + return errs + } + errStr1 := errs.Error() + errStr2 := err.Error() + return fmt.Errorf("%s \n* %s", errStr1, errStr2) +} diff --git a/pkg/lib/cpu/quotaturbo/statusstore.go b/pkg/lib/cpu/quotaturbo/statusstore.go index 82422a6..7c9f263 100644 --- a/pkg/lib/cpu/quotaturbo/statusstore.go +++ b/pkg/lib/cpu/quotaturbo/statusstore.go @@ -21,8 +21,6 @@ import ( "sync" "time" - "github.com/hashicorp/go-multierror" - "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -160,7 +158,7 @@ func (store *StatusStore) updateCPUQuotas() error { var errs error for id, c := range store.cpuQuotas { if err := c.update(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error updating cpu quota %v: %v", id, err)) + errs = appendErr(errs, fmt.Errorf("error updating cpu quota %v: %v", id, err)) } } return errs @@ -171,7 +169,7 @@ func (store *StatusStore) writeQuota() error { var errs error for id, c := range store.cpuQuotas { if err := c.writeQuota(); err != nil { - errs = multierror.Append(errs, fmt.Errorf("error writing cgroup quota %v: %v", id, err)) + errs = appendErr(errs, fmt.Errorf("error writing cgroup quota %v: %v", id, err)) } } return errs diff --git a/pkg/lib/cpu/quotaturbo/statusstore_test.go b/pkg/lib/cpu/quotaturbo/statusstore_test.go index a69fac4..49ccddc 100644 --- a/pkg/lib/cpu/quotaturbo/statusstore_test.go +++ b/pkg/lib/cpu/quotaturbo/statusstore_test.go @@ -286,7 +286,6 @@ func TestStatusStore_AddCgroup(t *testing.T) { type args struct { cgroupPath string cpuLimit float64 - cpuRequest float64 } tests := []struct { name string diff --git a/vendor/github.com/hashicorp/errwrap/LICENSE b/vendor/github.com/hashicorp/errwrap/LICENSE deleted file mode 100644 index c33dcc7..0000000 --- a/vendor/github.com/hashicorp/errwrap/LICENSE +++ /dev/null @@ -1,354 +0,0 @@ -Mozilla Public License, version 2.0 - -1. Definitions - -1.1. “Contributor” - - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. - -1.2. “Contributor Version” - - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” - - means Covered Software of a particular Contributor. - -1.4. “Covered Software” - - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. - -1.5. “Incompatible With Secondary Licenses” - means - - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or - - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. - -1.6. “Executable Form” - - means any form of the work other than Source Code Form. - -1.7. “Larger Work” - - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. - -1.8. “License” - - means this document. - -1.9. “Licensable” - - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. - -1.10. “Modifications” - - means any of the following: - - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or - - b. any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor - - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. - -1.12. “Secondary License” - - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. - -1.13. “Source Code Form” - - means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) - - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. - - -2. License Grants and Conditions - -2.1. Grants - - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. - diff --git a/vendor/github.com/hashicorp/errwrap/README.md b/vendor/github.com/hashicorp/errwrap/README.md deleted file mode 100644 index 444df08..0000000 --- a/vendor/github.com/hashicorp/errwrap/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# errwrap - -`errwrap` is a package for Go that formalizes the pattern of wrapping errors -and checking if an error contains another error. - -There is a common pattern in Go of taking a returned `error` value and -then wrapping it (such as with `fmt.Errorf`) before returning it. The problem -with this pattern is that you completely lose the original `error` structure. - -Arguably the _correct_ approach is that you should make a custom structure -implementing the `error` interface, and have the original error as a field -on that structure, such [as this example](http://golang.org/pkg/os/#PathError). -This is a good approach, but you have to know the entire chain of possible -rewrapping that happens, when you might just care about one. - -`errwrap` formalizes this pattern (it doesn't matter what approach you use -above) by giving a single interface for wrapping errors, checking if a specific -error is wrapped, and extracting that error. - -## Installation and Docs - -Install using `go get github.com/hashicorp/errwrap`. - -Full documentation is available at -http://godoc.org/github.com/hashicorp/errwrap - -## Usage - -#### Basic Usage - -Below is a very basic example of its usage: - -```go -// A function that always returns an error, but wraps it, like a real -// function might. -func tryOpen() error { - _, err := os.Open("/i/dont/exist") - if err != nil { - return errwrap.Wrapf("Doesn't exist: {{err}}", err) - } - - return nil -} - -func main() { - err := tryOpen() - - // We can use the Contains helpers to check if an error contains - // another error. It is safe to do this with a nil error, or with - // an error that doesn't even use the errwrap package. - if errwrap.Contains(err, "does not exist") { - // Do something - } - if errwrap.ContainsType(err, new(os.PathError)) { - // Do something - } - - // Or we can use the associated `Get` functions to just extract - // a specific error. This would return nil if that specific error doesn't - // exist. - perr := errwrap.GetType(err, new(os.PathError)) -} -``` - -#### Custom Types - -If you're already making custom types that properly wrap errors, then -you can get all the functionality of `errwraps.Contains` and such by -implementing the `Wrapper` interface with just one function. Example: - -```go -type AppError { - Code ErrorCode - Err error -} - -func (e *AppError) WrappedErrors() []error { - return []error{e.Err} -} -``` - -Now this works: - -```go -err := &AppError{Err: fmt.Errorf("an error")} -if errwrap.ContainsType(err, fmt.Errorf("")) { - // This will work! -} -``` diff --git a/vendor/github.com/hashicorp/errwrap/errwrap.go b/vendor/github.com/hashicorp/errwrap/errwrap.go deleted file mode 100644 index a733bef..0000000 --- a/vendor/github.com/hashicorp/errwrap/errwrap.go +++ /dev/null @@ -1,169 +0,0 @@ -// Package errwrap implements methods to formalize error wrapping in Go. -// -// All of the top-level functions that take an `error` are built to be able -// to take any error, not just wrapped errors. This allows you to use errwrap -// without having to type-check and type-cast everywhere. -package errwrap - -import ( - "errors" - "reflect" - "strings" -) - -// WalkFunc is the callback called for Walk. -type WalkFunc func(error) - -// Wrapper is an interface that can be implemented by custom types to -// have all the Contains, Get, etc. functions in errwrap work. -// -// When Walk reaches a Wrapper, it will call the callback for every -// wrapped error in addition to the wrapper itself. Since all the top-level -// functions in errwrap use Walk, this means that all those functions work -// with your custom type. -type Wrapper interface { - WrappedErrors() []error -} - -// Wrap defines that outer wraps inner, returning an error type that -// can be cleanly used with the other methods in this package, such as -// Contains, GetAll, etc. -// -// This function won't modify the error message at all (the outer message -// will be used). -func Wrap(outer, inner error) error { - return &wrappedError{ - Outer: outer, - Inner: inner, - } -} - -// Wrapf wraps an error with a formatting message. This is similar to using -// `fmt.Errorf` to wrap an error. If you're using `fmt.Errorf` to wrap -// errors, you should replace it with this. -// -// format is the format of the error message. The string '{{err}}' will -// be replaced with the original error message. -func Wrapf(format string, err error) error { - outerMsg := "" - if err != nil { - outerMsg = err.Error() - } - - outer := errors.New(strings.Replace( - format, "{{err}}", outerMsg, -1)) - - return Wrap(outer, err) -} - -// Contains checks if the given error contains an error with the -// message msg. If err is not a wrapped error, this will always return -// false unless the error itself happens to match this msg. -func Contains(err error, msg string) bool { - return len(GetAll(err, msg)) > 0 -} - -// ContainsType checks if the given error contains an error with -// the same concrete type as v. If err is not a wrapped error, this will -// check the err itself. -func ContainsType(err error, v interface{}) bool { - return len(GetAllType(err, v)) > 0 -} - -// Get is the same as GetAll but returns the deepest matching error. -func Get(err error, msg string) error { - es := GetAll(err, msg) - if len(es) > 0 { - return es[len(es)-1] - } - - return nil -} - -// GetType is the same as GetAllType but returns the deepest matching error. -func GetType(err error, v interface{}) error { - es := GetAllType(err, v) - if len(es) > 0 { - return es[len(es)-1] - } - - return nil -} - -// GetAll gets all the errors that might be wrapped in err with the -// given message. The order of the errors is such that the outermost -// matching error (the most recent wrap) is index zero, and so on. -func GetAll(err error, msg string) []error { - var result []error - - Walk(err, func(err error) { - if err.Error() == msg { - result = append(result, err) - } - }) - - return result -} - -// GetAllType gets all the errors that are the same type as v. -// -// The order of the return value is the same as described in GetAll. -func GetAllType(err error, v interface{}) []error { - var result []error - - var search string - if v != nil { - search = reflect.TypeOf(v).String() - } - Walk(err, func(err error) { - var needle string - if err != nil { - needle = reflect.TypeOf(err).String() - } - - if needle == search { - result = append(result, err) - } - }) - - return result -} - -// Walk walks all the wrapped errors in err and calls the callback. If -// err isn't a wrapped error, this will be called once for err. If err -// is a wrapped error, the callback will be called for both the wrapper -// that implements error as well as the wrapped error itself. -func Walk(err error, cb WalkFunc) { - if err == nil { - return - } - - switch e := err.(type) { - case *wrappedError: - cb(e.Outer) - Walk(e.Inner, cb) - case Wrapper: - cb(err) - - for _, err := range e.WrappedErrors() { - Walk(err, cb) - } - default: - cb(err) - } -} - -// wrappedError is an implementation of error that has both the -// outer and inner errors. -type wrappedError struct { - Outer error - Inner error -} - -func (w *wrappedError) Error() string { - return w.Outer.Error() -} - -func (w *wrappedError) WrappedErrors() []error { - return []error{w.Outer, w.Inner} -} diff --git a/vendor/github.com/hashicorp/errwrap/go.mod b/vendor/github.com/hashicorp/errwrap/go.mod deleted file mode 100644 index c9b8402..0000000 --- a/vendor/github.com/hashicorp/errwrap/go.mod +++ /dev/null @@ -1 +0,0 @@ -module github.com/hashicorp/errwrap diff --git a/vendor/github.com/hashicorp/go-multierror/.travis.yml b/vendor/github.com/hashicorp/go-multierror/.travis.yml deleted file mode 100644 index 304a835..0000000 --- a/vendor/github.com/hashicorp/go-multierror/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -sudo: false - -language: go - -go: - - 1.x - -branches: - only: - - master - -script: make test testrace diff --git a/vendor/github.com/hashicorp/go-multierror/LICENSE b/vendor/github.com/hashicorp/go-multierror/LICENSE deleted file mode 100644 index 82b4de9..0000000 --- a/vendor/github.com/hashicorp/go-multierror/LICENSE +++ /dev/null @@ -1,353 +0,0 @@ -Mozilla Public License, version 2.0 - -1. Definitions - -1.1. “Contributor” - - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. - -1.2. “Contributor Version” - - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” - - means Covered Software of a particular Contributor. - -1.4. “Covered Software” - - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. - -1.5. “Incompatible With Secondary Licenses” - means - - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or - - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. - -1.6. “Executable Form” - - means any form of the work other than Source Code Form. - -1.7. “Larger Work” - - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. - -1.8. “License” - - means this document. - -1.9. “Licensable” - - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. - -1.10. “Modifications” - - means any of the following: - - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or - - b. any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor - - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. - -1.12. “Secondary License” - - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. - -1.13. “Source Code Form” - - means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) - - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. - - -2. License Grants and Conditions - -2.1. Grants - - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: - - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and - - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. - -2.2. Effective Date - - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. - -2.3. Limitations on Grant Scope - - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: - - a. for any code that a Contributor has removed from Covered Software; or - - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. - - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). - -2.4. Subsequent Licenses - - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). - -2.5. Representation - - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. - -2.6. Fair Use - - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. - - -3. Responsibilities - -3.1. Distribution of Source Form - - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - - If You distribute Covered Software in Executable Form then: - - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and - - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). - -3.4. Notices - - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. - -6. Disclaimer of Warranty - - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation - - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. - - -10. Versions of the License - -10.1. New Versions - - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. - -10.2. Effect of New Versions - - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. - -10.3. Modified Versions - - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/hashicorp/go-multierror/Makefile b/vendor/github.com/hashicorp/go-multierror/Makefile deleted file mode 100644 index b97cd6e..0000000 --- a/vendor/github.com/hashicorp/go-multierror/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -TEST?=./... - -default: test - -# test runs the test suite and vets the code. -test: generate - @echo "==> Running tests..." - @go list $(TEST) \ - | grep -v "/vendor/" \ - | xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS} - -# testrace runs the race checker -testrace: generate - @echo "==> Running tests (race)..." - @go list $(TEST) \ - | grep -v "/vendor/" \ - | xargs -n1 go test -timeout=60s -race ${TESTARGS} - -# updatedeps installs all the dependencies needed to run and build. -updatedeps: - @sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'" - -# generate runs `go generate` to build the dynamically generated source files. -generate: - @echo "==> Generating..." - @find . -type f -name '.DS_Store' -delete - @go list ./... \ - | grep -v "/vendor/" \ - | xargs -n1 go generate - -.PHONY: default test testrace updatedeps generate diff --git a/vendor/github.com/hashicorp/go-multierror/README.md b/vendor/github.com/hashicorp/go-multierror/README.md deleted file mode 100644 index ead5830..0000000 --- a/vendor/github.com/hashicorp/go-multierror/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# go-multierror - -[![Build Status](http://img.shields.io/travis/hashicorp/go-multierror.svg?style=flat-square)][travis] -[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] - -[travis]: https://travis-ci.org/hashicorp/go-multierror -[godocs]: https://godoc.org/github.com/hashicorp/go-multierror - -`go-multierror` is a package for Go that provides a mechanism for -representing a list of `error` values as a single `error`. - -This allows a function in Go to return an `error` that might actually -be a list of errors. If the caller knows this, they can unwrap the -list and access the errors. If the caller doesn't know, the error -formats to a nice human-readable format. - -`go-multierror` implements the -[errwrap](https://github.com/hashicorp/errwrap) interface so that it can -be used with that library, as well. - -## Installation and Docs - -Install using `go get github.com/hashicorp/go-multierror`. - -Full documentation is available at -http://godoc.org/github.com/hashicorp/go-multierror - -## Usage - -go-multierror is easy to use and purposely built to be unobtrusive in -existing Go applications/libraries that may not be aware of it. - -**Building a list of errors** - -The `Append` function is used to create a list of errors. This function -behaves a lot like the Go built-in `append` function: it doesn't matter -if the first argument is nil, a `multierror.Error`, or any other `error`, -the function behaves as you would expect. - -```go -var result error - -if err := step1(); err != nil { - result = multierror.Append(result, err) -} -if err := step2(); err != nil { - result = multierror.Append(result, err) -} - -return result -``` - -**Customizing the formatting of the errors** - -By specifying a custom `ErrorFormat`, you can customize the format -of the `Error() string` function: - -```go -var result *multierror.Error - -// ... accumulate errors here, maybe using Append - -if result != nil { - result.ErrorFormat = func([]error) string { - return "errors!" - } -} -``` - -**Accessing the list of errors** - -`multierror.Error` implements `error` so if the caller doesn't know about -multierror, it will work just fine. But if you're aware a multierror might -be returned, you can use type switches to access the list of errors: - -```go -if err := something(); err != nil { - if merr, ok := err.(*multierror.Error); ok { - // Use merr.Errors - } -} -``` - -**Returning a multierror only if there are errors** - -If you build a `multierror.Error`, you can use the `ErrorOrNil` function -to return an `error` implementation only if there are errors to return: - -```go -var result *multierror.Error - -// ... accumulate errors here - -// Return the `error` only if errors were added to the multierror, otherwise -// return nil since there are no errors. -return result.ErrorOrNil() -``` diff --git a/vendor/github.com/hashicorp/go-multierror/append.go b/vendor/github.com/hashicorp/go-multierror/append.go deleted file mode 100644 index 775b6e7..0000000 --- a/vendor/github.com/hashicorp/go-multierror/append.go +++ /dev/null @@ -1,41 +0,0 @@ -package multierror - -// Append is a helper function that will append more errors -// onto an Error in order to create a larger multi-error. -// -// If err is not a multierror.Error, then it will be turned into -// one. If any of the errs are multierr.Error, they will be flattened -// one level into err. -func Append(err error, errs ...error) *Error { - switch err := err.(type) { - case *Error: - // Typed nils can reach here, so initialize if we are nil - if err == nil { - err = new(Error) - } - - // Go through each error and flatten - for _, e := range errs { - switch e := e.(type) { - case *Error: - if e != nil { - err.Errors = append(err.Errors, e.Errors...) - } - default: - if e != nil { - err.Errors = append(err.Errors, e) - } - } - } - - return err - default: - newErrs := make([]error, 0, len(errs)+1) - if err != nil { - newErrs = append(newErrs, err) - } - newErrs = append(newErrs, errs...) - - return Append(&Error{}, newErrs...) - } -} diff --git a/vendor/github.com/hashicorp/go-multierror/flatten.go b/vendor/github.com/hashicorp/go-multierror/flatten.go deleted file mode 100644 index aab8e9a..0000000 --- a/vendor/github.com/hashicorp/go-multierror/flatten.go +++ /dev/null @@ -1,26 +0,0 @@ -package multierror - -// Flatten flattens the given error, merging any *Errors together into -// a single *Error. -func Flatten(err error) error { - // If it isn't an *Error, just return the error as-is - if _, ok := err.(*Error); !ok { - return err - } - - // Otherwise, make the result and flatten away! - flatErr := new(Error) - flatten(err, flatErr) - return flatErr -} - -func flatten(err error, flatErr *Error) { - switch err := err.(type) { - case *Error: - for _, e := range err.Errors { - flatten(e, flatErr) - } - default: - flatErr.Errors = append(flatErr.Errors, err) - } -} diff --git a/vendor/github.com/hashicorp/go-multierror/format.go b/vendor/github.com/hashicorp/go-multierror/format.go deleted file mode 100644 index 47f13c4..0000000 --- a/vendor/github.com/hashicorp/go-multierror/format.go +++ /dev/null @@ -1,27 +0,0 @@ -package multierror - -import ( - "fmt" - "strings" -) - -// ErrorFormatFunc is a function callback that is called by Error to -// turn the list of errors into a string. -type ErrorFormatFunc func([]error) string - -// ListFormatFunc is a basic formatter that outputs the number of errors -// that occurred along with a bullet point list of the errors. -func ListFormatFunc(es []error) string { - if len(es) == 1 { - return fmt.Sprintf("1 error occurred:\n\t* %s\n\n", es[0]) - } - - points := make([]string, len(es)) - for i, err := range es { - points[i] = fmt.Sprintf("* %s", err) - } - - return fmt.Sprintf( - "%d errors occurred:\n\t%s\n\n", - len(es), strings.Join(points, "\n\t")) -} diff --git a/vendor/github.com/hashicorp/go-multierror/go.mod b/vendor/github.com/hashicorp/go-multierror/go.mod deleted file mode 100644 index 2534331..0000000 --- a/vendor/github.com/hashicorp/go-multierror/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/hashicorp/go-multierror - -require github.com/hashicorp/errwrap v1.0.0 diff --git a/vendor/github.com/hashicorp/go-multierror/go.sum b/vendor/github.com/hashicorp/go-multierror/go.sum deleted file mode 100644 index 85b1f8f..0000000 --- a/vendor/github.com/hashicorp/go-multierror/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/vendor/github.com/hashicorp/go-multierror/multierror.go b/vendor/github.com/hashicorp/go-multierror/multierror.go deleted file mode 100644 index 89b1422..0000000 --- a/vendor/github.com/hashicorp/go-multierror/multierror.go +++ /dev/null @@ -1,51 +0,0 @@ -package multierror - -import ( - "fmt" -) - -// Error is an error type to track multiple errors. This is used to -// accumulate errors in cases and return them as a single "error". -type Error struct { - Errors []error - ErrorFormat ErrorFormatFunc -} - -func (e *Error) Error() string { - fn := e.ErrorFormat - if fn == nil { - fn = ListFormatFunc - } - - return fn(e.Errors) -} - -// ErrorOrNil returns an error interface if this Error represents -// a list of errors, or returns nil if the list of errors is empty. This -// function is useful at the end of accumulation to make sure that the value -// returned represents the existence of errors. -func (e *Error) ErrorOrNil() error { - if e == nil { - return nil - } - if len(e.Errors) == 0 { - return nil - } - - return e -} - -func (e *Error) GoString() string { - return fmt.Sprintf("*%#v", *e) -} - -// WrappedErrors returns the list of errors that this Error is wrapping. -// It is an implementation of the errwrap.Wrapper interface so that -// multierror.Error can be used with that library. -// -// This method is not safe to be called concurrently and is no different -// than accessing the Errors field directly. It is implemented only to -// satisfy the errwrap.Wrapper interface. -func (e *Error) WrappedErrors() []error { - return e.Errors -} diff --git a/vendor/github.com/hashicorp/go-multierror/prefix.go b/vendor/github.com/hashicorp/go-multierror/prefix.go deleted file mode 100644 index 5c477ab..0000000 --- a/vendor/github.com/hashicorp/go-multierror/prefix.go +++ /dev/null @@ -1,37 +0,0 @@ -package multierror - -import ( - "fmt" - - "github.com/hashicorp/errwrap" -) - -// Prefix is a helper function that will prefix some text -// to the given error. If the error is a multierror.Error, then -// it will be prefixed to each wrapped error. -// -// This is useful to use when appending multiple multierrors -// together in order to give better scoping. -func Prefix(err error, prefix string) error { - if err == nil { - return nil - } - - format := fmt.Sprintf("%s {{err}}", prefix) - switch err := err.(type) { - case *Error: - // Typed nils can reach here, so initialize if we are nil - if err == nil { - err = new(Error) - } - - // Wrap each of the errors - for i, e := range err.Errors { - err.Errors[i] = errwrap.Wrapf(format, e) - } - - return err - default: - return errwrap.Wrapf(format, err) - } -} diff --git a/vendor/github.com/hashicorp/go-multierror/sort.go b/vendor/github.com/hashicorp/go-multierror/sort.go deleted file mode 100644 index fecb14e..0000000 --- a/vendor/github.com/hashicorp/go-multierror/sort.go +++ /dev/null @@ -1,16 +0,0 @@ -package multierror - -// Len implements sort.Interface function for length -func (err Error) Len() int { - return len(err.Errors) -} - -// Swap implements sort.Interface function for swapping elements -func (err Error) Swap(i, j int) { - err.Errors[i], err.Errors[j] = err.Errors[j], err.Errors[i] -} - -// Less implements sort.Interface function for determining order -func (err Error) Less(i, j int) bool { - return err.Errors[i].Error() < err.Errors[j].Error() -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 209b861..2f895a3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -32,11 +32,6 @@ github.com/google/uuid github.com/googleapis/gnostic/compiler github.com/googleapis/gnostic/extensions github.com/googleapis/gnostic/openapiv2 -# github.com/hashicorp/errwrap v1.0.0 -github.com/hashicorp/errwrap -# github.com/hashicorp/go-multierror v1.0.0 -## explicit -github.com/hashicorp/go-multierror # github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru -- Gitee From 020d8c41beb71d211ec5699badcf658803b4eb69 Mon Sep 17 00:00:00 2001 From: Lu Jingxiao Date: Mon, 20 Mar 2023 11:59:57 +0800 Subject: [PATCH 67/73] optimize: optimize ServiceBase and logs Extract the Name field as an internal variable of ServiceBase 1. Enable ServiceBase.go to get the name of the caller 2. Reduce the duplicate logic of ID 3. Optimize log format and level Signed-off-by: Lu Jingxiao --- pkg/common/log/log.go | 4 +-- pkg/lib/cpu/quotaturbo/cpuquota.go | 4 +-- pkg/rubik/servicemanager.go | 4 +-- pkg/services/dyncache/dynamic_test.go | 6 ++++- pkg/services/dyncache/dyncache.go | 10 +++----- pkg/services/dyncache/dyncache_init_test.go | 11 +++++--- pkg/services/dyncache/dyncache_test.go | 18 ++++++++++--- pkg/services/dyncache/sync_test.go | 6 ++++- pkg/services/helper/service_base.go | 28 ++++++++++++++------- pkg/services/iocost/iocost.go | 11 +++----- pkg/services/iolimit/iolimit.go | 10 +++----- pkg/services/preemption/preemption.go | 10 ++------ pkg/services/preemption/preemption_test.go | 16 +++++++++--- pkg/services/quotaburst/quotaburst.go | 8 +----- pkg/services/quotaburst/quotaburst_test.go | 9 ++++--- pkg/services/quotaturbo/quotaturbo.go | 12 +++------ pkg/services/quotaturbo/quotaturbo_test.go | 11 ++++++-- 17 files changed, 98 insertions(+), 80 deletions(-) diff --git a/pkg/common/log/log.go b/pkg/common/log/log.go index 1fefc3b..d9a5cc7 100644 --- a/pkg/common/log/log.go +++ b/pkg/common/log/log.go @@ -224,7 +224,7 @@ func output(level string, format string, args ...interface{}) { // Warnf log warn level func Warnf(format string, args ...interface{}) { if logWarn >= logLevel { - output(levelToString(logInfo), format, args...) + output(levelToString(logWarn), format, args...) } } @@ -279,7 +279,7 @@ func (e *Entry) Warnf(f string, args ...interface{}) { if logInfo < logLevel { return } - output(e.level(logInfo), f, args...) + output(e.level(logWarn), f, args...) } // Infof write logs diff --git a/pkg/lib/cpu/quotaturbo/cpuquota.go b/pkg/lib/cpu/quotaturbo/cpuquota.go index 601edfa..cb8fad7 100644 --- a/pkg/lib/cpu/quotaturbo/cpuquota.go +++ b/pkg/lib/cpu/quotaturbo/cpuquota.go @@ -257,7 +257,5 @@ func appendErr(errs error, err error) error { if err == nil { return errs } - errStr1 := errs.Error() - errStr2 := err.Error() - return fmt.Errorf("%s \n* %s", errStr1, errStr2) + return fmt.Errorf("%s \n* %s", errs.Error(), err.Error()) } diff --git a/pkg/rubik/servicemanager.go b/pkg/rubik/servicemanager.go index e4bf6ce..ba63b18 100644 --- a/pkg/rubik/servicemanager.go +++ b/pkg/rubik/servicemanager.go @@ -132,13 +132,13 @@ func terminatingServices(serviceMap map[string]services.Service, viewer api.View if err := s.Stop(); err != nil { log.Errorf("fail to stop service %v: %v", name, err) } else { - log.Infof("stop service %v successfully", name) + log.Infof("service %v stop successfully", name) } } if err := s.Terminate(viewer); err != nil { log.Errorf("fail to terminate service %v: %v", name, err) } else { - log.Infof("terminate service %v successfully", name) + log.Infof("service %v terminate successfully", name) } } } diff --git a/pkg/services/dyncache/dynamic_test.go b/pkg/services/dyncache/dynamic_test.go index e0630f7..e538437 100644 --- a/pkg/services/dyncache/dynamic_test.go +++ b/pkg/services/dyncache/dynamic_test.go @@ -23,9 +23,11 @@ import ( "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/perf" "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" ) +// TestCacheLimit_StartDynamic tests StartDynamic of CacheLimit func TestCacheLimit_StartDynamic(t *testing.T) { if !perf.Support() { t.Skipf("%s only run on physical machine", t.Name()) @@ -264,7 +266,9 @@ func TestCacheLimit_StartDynamic(t *testing.T) { c := &DynCache{ config: tt.fields.Config, Attr: tt.fields.Attr, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } if tt.preHook != nil { tt.preHook(t, c, tt.fields.FakePods) diff --git a/pkg/services/dyncache/dyncache.go b/pkg/services/dyncache/dyncache.go index 2eec701..3204d90 100644 --- a/pkg/services/dyncache/dyncache.go +++ b/pkg/services/dyncache/dyncache.go @@ -91,7 +91,6 @@ type DynCache struct { config *Config Attr *Attr Viewer api.Viewer - Name string `json:"-"` } // Attr is cache limit attribute differ from config @@ -150,7 +149,9 @@ func newConfig() *Config { // NewDynCache return cache limit instance with default settings func NewDynCache(name string) *DynCache { return &DynCache{ - Name: name, + ServiceBase: helper.ServiceBase{ + Name: name, + }, Attr: &Attr{ NumaNodeDir: defaultNumaNodeDir, MaxMiss: defaultMaxMiss, @@ -175,11 +176,6 @@ func (c *DynCache) PreStart(viewer api.Viewer) error { return nil } -// ID returns service's name -func (c *DynCache) ID() string { - return c.Name -} - // GetConfig returns Config func (c *DynCache) GetConfig() interface{} { return c.config diff --git a/pkg/services/dyncache/dyncache_init_test.go b/pkg/services/dyncache/dyncache_init_test.go index 71e5a88..9836748 100644 --- a/pkg/services/dyncache/dyncache_init_test.go +++ b/pkg/services/dyncache/dyncache_init_test.go @@ -21,8 +21,10 @@ import ( "testing" "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/perf" + "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" ) @@ -41,6 +43,7 @@ func genNumaNodes(path string, num int) { } } +// TestCacheLimit_InitCacheLimitDir tests InitCacheLimitDir of CacheLimit func TestCacheLimit_InitCacheLimitDir(t *testing.T) { if !perf.Support() { t.Skipf("%s only run on physical machine", t.Name()) @@ -101,7 +104,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") - os.Symlink(pidNameSpaceFileOri.String(), pidNameSpace) + assert.NoError(t, os.Symlink(pidNameSpaceFileOri.String(), pidNameSpace)) c.config.DefaultPidNameSpace = pidNameSpace }, postHook: func(t *testing.T, c *DynCache) { @@ -120,7 +123,7 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { pidNameSpaceFileOri := try.WriteFile(filepath.Join(pidNameSpaceDir, "pid:[4026531836x]"), "") pidNameSpace := filepath.Join(pidNameSpaceDir, "pid") - os.Link(pidNameSpaceFileOri.String(), pidNameSpace) + assert.NoError(t, os.Link(pidNameSpaceFileOri.String(), pidNameSpace)) c.config.DefaultPidNameSpace = pidNameSpace }, postHook: func(t *testing.T, c *DynCache) { @@ -208,7 +211,9 @@ func TestCacheLimit_InitCacheLimitDir(t *testing.T) { c := &DynCache{ config: tt.fields.Config, Attr: tt.fields.Attr, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } if tt.preHook != nil { tt.preHook(t, c) diff --git a/pkg/services/dyncache/dyncache_test.go b/pkg/services/dyncache/dyncache_test.go index a4c2bce..7921653 100644 --- a/pkg/services/dyncache/dyncache_test.go +++ b/pkg/services/dyncache/dyncache_test.go @@ -21,12 +21,14 @@ import ( "testing" "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/services/helper" ) const ( moduleName = "dynCache" ) +// TestCacheLimit_StartDynamic tests StartDynamic of CacheLimit func TestCacheLimit_Validate(t *testing.T) { const num2 = 2 type fields struct { @@ -181,7 +183,9 @@ func TestCacheLimit_Validate(t *testing.T) { c := &DynCache{ config: tt.fields.Config, Attr: tt.fields.Attr, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } err := c.config.Validate() if (err != nil) != tt.wantErr { @@ -202,7 +206,9 @@ func TestNewCacheLimit(t *testing.T) { { name: "TC-do nothing", want: &DynCache{ - Name: moduleName, + ServiceBase: helper.ServiceBase{ + Name: moduleName, + }, Attr: &Attr{ NumaNodeDir: defaultNumaNodeDir, MaxMiss: defaultMaxMiss, @@ -271,7 +277,9 @@ func TestCacheLimit_PreStart(t *testing.T) { config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } if err := c.PreStart(tt.args.viewer); (err != nil) != tt.wantErr { t.Errorf("CacheLimit.PreStart() error = %v, wantErr %v", err, tt.wantErr) @@ -307,7 +315,9 @@ func TestCacheLimit_ID(t *testing.T) { config: tt.fields.Config, Attr: tt.fields.Attr, Viewer: tt.fields.Viewer, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } if got := c.ID(); got != tt.want { t.Errorf("CacheLimit.ID() = %v, want %v", got, tt.want) diff --git a/pkg/services/dyncache/sync_test.go b/pkg/services/dyncache/sync_test.go index 118ac38..67ff07d 100644 --- a/pkg/services/dyncache/sync_test.go +++ b/pkg/services/dyncache/sync_test.go @@ -22,6 +22,7 @@ import ( "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/podmanager" + "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" ) @@ -55,6 +56,7 @@ func cleanFakePods(fakePods []*try.FakePod) { } } +// TestCacheLimit_SyncCacheLimit tests SyncCacheLimit of CacheLimit func TestCacheLimit_SyncCacheLimit(t *testing.T) { resctrlDir := try.GenTestDir().String() defer try.RemoveAll(resctrlDir) @@ -244,7 +246,9 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { c := &DynCache{ config: tt.fields.Config, Attr: tt.fields.Attr, - Name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, } if tt.preHook != nil { tt.preHook(t, c, tt.fields.FakePods) diff --git a/pkg/services/helper/service_base.go b/pkg/services/helper/service_base.go index 78675c3..3015e89 100644 --- a/pkg/services/helper/service_base.go +++ b/pkg/services/helper/service_base.go @@ -16,7 +16,6 @@ package helper import ( "context" - "fmt" "isula.org/rubik/pkg/api" "isula.org/rubik/pkg/common/log" @@ -24,7 +23,16 @@ import ( ) // ServiceBase is the basic class of a service. -type ServiceBase struct{} +type ServiceBase struct { + Name string +} + +// NewServiceBase returns the instance of service +func NewServiceBase(serviceName string) *ServiceBase { + return &ServiceBase{ + Name: serviceName, + } +} // ConfigHandler is that obtains the configured callback function. type ConfigHandler func(configName string, d interface{}) error @@ -36,16 +44,21 @@ func (s *ServiceBase) SetConfig(ConfigHandler) error { // PreStart is an interface for calling a collection of methods when the service is pre-started func (s *ServiceBase) PreStart(api.Viewer) error { - log.Warnf("this interface is not implemented.") + log.Warnf("%v: PreStart interface is not implemented", s.Name) return nil } // Terminate is an interface that calls a collection of methods when the service terminates func (s *ServiceBase) Terminate(api.Viewer) error { - log.Warnf("this interface is not implemented.") + log.Warnf("%v: Terminate interface is not implemented", s.Name) return nil } +// ID is an interface that calls a collection of methods returning service's ID +func (s *ServiceBase) ID() string { + return s.Name +} + // IsRunner to Confirm whether it is a runner func (s *ServiceBase) IsRunner() bool { return false @@ -56,28 +69,25 @@ func (s *ServiceBase) Run(context.Context) {} // Stop to stop runner func (s *ServiceBase) Stop() error { - return fmt.Errorf("this interface is not implemented") + return nil } // AddPod to deal the event of adding a pod. func (s *ServiceBase) AddPod(*typedef.PodInfo) error { - log.Warnf("this interface is not implemented.") return nil } // UpdatePod to deal the pod update event. func (S *ServiceBase) UpdatePod(old, new *typedef.PodInfo) error { - log.Warnf("this interface is not implemented.") return nil } // DeletePod to deal the pod deletion event. func (s *ServiceBase) DeletePod(*typedef.PodInfo) error { - log.Warnf("this interface is not implemented.") return nil } -// Run to start runner +// GetConfig returns the config of service func (s *ServiceBase) GetConfig() interface{} { return nil } diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index beff2f2..3fccb9d 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -64,7 +64,6 @@ type NodeConfig struct { // IOCost for iocost class type IOCost struct { helper.ServiceBase - name string } var ( @@ -85,7 +84,7 @@ func (i IOCostFactory) Name() string { func (i IOCostFactory) NewObj() (interface{}, error) { if ioCostSupport() { nodeName = os.Getenv(constant.NodeNameEnvKey) - return &IOCost{name: i.ObjName}, nil + return &IOCost{ServiceBase: helper.ServiceBase{Name: i.ObjName}}, nil } return nil, fmt.Errorf("this machine not support iocost") } @@ -108,11 +107,6 @@ func ioCostSupport() bool { return util.PathExist(qosFile) && util.PathExist(modelFile) } -// ID for get the name of iocost -func (io *IOCost) ID() string { - return io.name -} - // SetConfig to config nodeConfig configure func (io *IOCost) SetConfig(f helper.ConfigHandler) error { if f == nil { @@ -121,7 +115,7 @@ func (io *IOCost) SetConfig(f helper.ConfigHandler) error { var nodeConfigs []NodeConfig var nodeConfig *NodeConfig - if err := f(io.name, &nodeConfigs); err != nil { + if err := f(io.Name, &nodeConfigs); err != nil { return err } @@ -159,6 +153,7 @@ func (io *IOCost) PreStart(viewer api.Viewer) error { return io.dealExistedPods(viewer) } +// Terminate is the terminating action func (b *IOCost) Terminate(viewer api.Viewer) error { if err := b.clearIOCost(); err != nil { return err diff --git a/pkg/services/iolimit/iolimit.go b/pkg/services/iolimit/iolimit.go index c5b6039..dcc70d2 100644 --- a/pkg/services/iolimit/iolimit.go +++ b/pkg/services/iolimit/iolimit.go @@ -27,7 +27,6 @@ type DeviceConfig struct { // IOLimit is the class of IOLimit. type IOLimit struct { helper.ServiceBase - name string } // IOLimitFactory is the factory of IOLimit. @@ -42,10 +41,7 @@ func (i IOLimitFactory) Name() string { // NewObj to create object of IOLimit. func (i IOLimitFactory) NewObj() (interface{}, error) { - return &IOLimit{name: i.ObjName}, nil -} - -// ID to get the name of IOLimit. -func (i *IOLimit) ID() string { - return i.name + return &IOLimit{ + ServiceBase: *helper.NewServiceBase(i.ObjName), + }, nil } diff --git a/pkg/services/preemption/preemption.go b/pkg/services/preemption/preemption.go index 410049e..ce436a3 100644 --- a/pkg/services/preemption/preemption.go +++ b/pkg/services/preemption/preemption.go @@ -34,7 +34,6 @@ var supportCgroupTypes = map[string]*cgroup.Key{ // Preemption define service which related to qos level setting type Preemption struct { helper.ServiceBase - name string config PreemptionConfig } @@ -55,18 +54,13 @@ func (i PreemptionFactory) Name() string { // NewObj to create object of Preemption. func (i PreemptionFactory) NewObj() (interface{}, error) { - return &Preemption{name: i.ObjName}, nil -} - -// ID return qos service name -func (q *Preemption) ID() string { - return q.name + return &Preemption{ServiceBase: helper.ServiceBase{Name: i.ObjName}}, nil } // SetConfig to config Preemption configure func (q *Preemption) SetConfig(f helper.ConfigHandler) error { var c PreemptionConfig - if err := f(q.name, &c); err != nil { + if err := f(q.Name, &c); err != nil { return err } if err := c.Validate(); err != nil { diff --git a/pkg/services/preemption/preemption_test.go b/pkg/services/preemption/preemption_test.go index 7ac8f5e..5c716d4 100644 --- a/pkg/services/preemption/preemption_test.go +++ b/pkg/services/preemption/preemption_test.go @@ -19,6 +19,7 @@ import ( "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" ) @@ -50,6 +51,7 @@ var getCommonField = func(subSys []string) fields { } } +// TestPreemptionAddFunc tests AddFunc of Preemption func TestPreemptionAddFunc(t *testing.T) { const containerNum = 3 var addFuncTC = []test{ @@ -107,7 +109,9 @@ func TestPreemptionAddFunc(t *testing.T) { for _, tt := range addFuncTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, config: tt.fields.Config, } if tt.preHook != nil { @@ -124,6 +128,7 @@ func TestPreemptionAddFunc(t *testing.T) { } } +// TestPreemptionUpdatePod tests UpdatePod of Preemption func TestPreemptionUpdatePod(t *testing.T) { var updateFuncTC = []test{ { @@ -132,7 +137,6 @@ func TestPreemptionUpdatePod(t *testing.T) { args: args{old: try.GenFakeOnlinePod(map[*cgroup.Key]string{supportCgroupTypes["cpu"]: "0"}).WithContainers(3)}, preHook: func(pod *try.FakePod) *try.FakePod { newPod := pod.DeepCopy() - // TODO: need fix pod.DeepCopy newAnnotation := make(map[string]string, 0) newAnnotation[constant.PriorityAnnotationKey] = "true" newPod.Annotations = newAnnotation @@ -165,7 +169,9 @@ func TestPreemptionUpdatePod(t *testing.T) { for _, tt := range updateFuncTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, config: tt.fields.Config, } if tt.preHook != nil { @@ -206,7 +212,9 @@ func TestPreemptionValidate(t *testing.T) { for _, tt := range validateTC { t.Run(tt.name, func(t *testing.T) { q := &Preemption{ - name: tt.fields.Name, + ServiceBase: helper.ServiceBase{ + Name: tt.fields.Name, + }, config: tt.fields.Config, } if err := q.config.Validate(); (err != nil) != tt.wantErr { diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index 75f9bbf..88c3004 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -34,7 +34,6 @@ import ( // Burst is used to control cpu burst type Burst struct { helper.ServiceBase - name string } // BurstFactory is the factory os Burst. @@ -49,12 +48,7 @@ func (i BurstFactory) Name() string { // NewObj to create object of Burst. func (i BurstFactory) NewObj() (interface{}, error) { - return &Burst{name: i.ObjName}, nil -} - -// ID returns the module name -func (b *Burst) ID() string { - return b.name + return &Burst{ServiceBase: helper.ServiceBase{Name: i.ObjName}}, nil } // AddPod implement add function when pod is added in k8s diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go index 0e2ec08..08e4659 100644 --- a/pkg/services/quotaburst/quotaburst_test.go +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -24,6 +24,7 @@ import ( "isula.org/rubik/pkg/core/typedef" "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/podmanager" + "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" ) @@ -155,7 +156,7 @@ func TestBurst_AddPod(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := Burst{name: moduleName} + conf := Burst{ServiceBase: helper.ServiceBase{Name: moduleName}} if tt.args.burst != "" { tt.args.pod.Annotations[constant.QuotaBurstAnnotationKey] = tt.args.burst } @@ -172,7 +173,7 @@ func TestBurst_AddPod(t *testing.T) { func TestOther(t *testing.T) { const tcName = "TC1-test Other" t.Run(tcName, func(t *testing.T) { - got := Burst{name: moduleName} + got := Burst{ServiceBase: helper.ServiceBase{Name: moduleName}} assert.NoError(t, got.DeletePod(&typedef.PodInfo{})) assert.Equal(t, moduleName, got.ID()) }) @@ -220,7 +221,7 @@ func TestBurst_UpdatePod(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := Burst{name: moduleName} + conf := Burst{ServiceBase: helper.ServiceBase{Name: moduleName}} if err := conf.UpdatePod(tt.args.oldPod, tt.args.newPod); (err != nil) != tt.wantErr { t.Errorf("Burst.UpdatePod() error = %v, wantErr %v", err, tt.wantErr) } @@ -254,7 +255,7 @@ func TestBurst_PreStart(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - conf := Burst{name: moduleName} + conf := Burst{ServiceBase: helper.ServiceBase{Name: moduleName}} if err := conf.PreStart(tt.args.viewer); (err != nil) != tt.wantErr { t.Errorf("Burst.PreStart() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index 6dfd5d0..0442d85 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -76,7 +76,6 @@ func NewConfig() *Config { // QuotaTurbo manages all container CPU quota data on the current node. type QuotaTurbo struct { - name string conf *Config client *quotaturbo.Client Viewer api.Viewer @@ -86,17 +85,14 @@ type QuotaTurbo struct { // NewQuotaTurbo generate quota turbo objects func NewQuotaTurbo(n string) *QuotaTurbo { return &QuotaTurbo{ - name: n, + ServiceBase: helper.ServiceBase{ + Name: n, + }, conf: NewConfig(), client: quotaturbo.NewClient(), } } -// ID returns the module name -func (qt *QuotaTurbo) ID() string { - return qt.name -} - // syncCgroups updates the cgroup in cilent according to the current whitelist pod list func (qt *QuotaTurbo) syncCgroups(conts map[string]*typedef.ContainerInfo) { var ( @@ -179,7 +175,7 @@ func (conf *Config) Validate() error { // SetConfig sets and checks Config func (qt *QuotaTurbo) SetConfig(f helper.ConfigHandler) error { var conf = NewConfig() - if err := f(qt.name, conf); err != nil { + if err := f(qt.Name, conf); err != nil { return err } if err := conf.Validate(); err != nil { diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index 7bac196..e2d910f 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" @@ -301,7 +302,10 @@ func TestQuotaTurbo_SetConfig(t *testing.T) { name: "TC3-invalid config", args: args{ f: func(configName string, d interface{}) error { - c := d.(*Config) + c, ok := d.(*Config) + if !ok { + t.Error("fial to convert config") + } c.AlarmWaterMark = 101 return nil }, @@ -337,7 +341,10 @@ func TestQuotaTurbo_Other(t *testing.T) { f.Name() instance, err := f.NewObj() assert.NoError(t, err) - qt := instance.(*QuotaTurbo) + qt, ok := instance.(*QuotaTurbo) + if !ok { + t.Error("fial to convert QuotaTurbo") + } if got := qt.IsRunner(); got != tt.want { t.Errorf("QuotaTurbo.IsRunner() = %v, want %v", got, tt.want) } -- Gitee From 5d559a9fa181f864b42b8108dcda38fe1240d019 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 23 Mar 2023 16:26:22 +0800 Subject: [PATCH 68/73] test:add pkg/core/typedef/cgroup DT test coverage: 100.0% of statements ok isula.org/rubik/pkg/core/typedef/cgroup 0.006s Signed-off-by: vegbir --- pkg/core/typedef/cgroup/common_test.go | 632 +++++++++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 pkg/core/typedef/cgroup/common_test.go diff --git a/pkg/core/typedef/cgroup/common_test.go b/pkg/core/typedef/cgroup/common_test.go new file mode 100644 index 0000000..6138594 --- /dev/null +++ b/pkg/core/typedef/cgroup/common_test.go @@ -0,0 +1,632 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-03-23 +// Description: This file test common function of cgroup + +// Package cgroup uses map to manage cgroup parameters and provides a friendly and simple cgroup usage method +package cgroup + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/util" +) + +// TestReadCgroupFile tests ReadCgroupFile +func TestReadCgroupFile(t *testing.T) { + InitMountDir(constant.TmpTestDir) + defer InitMountDir(constant.DefaultCgroupRoot) + pathElems := []string{"cpu", "kubepods", "PodXXX", "ContYYY", "cpu.cfs_quota_us"} + const value = "-1\n" + tests := []struct { + name string + args []string + pre func(t *testing.T) + post func(t *testing.T) + want []byte + wantErr bool + }{ + { + name: "TC1-non existed path", + args: pathElems, + wantErr: true, + want: nil, + }, + { + name: "TC2-successfully", + args: pathElems, + pre: func(t *testing.T) { + assert.NoError(t, util.WriteFile( + filepath.Join(constant.TmpTestDir, filepath.Join(pathElems...)), + value)) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: false, + want: []byte(value), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre(t) + } + got, err := ReadCgroupFile(tt.args...) + if (err != nil) != tt.wantErr { + t.Errorf("ReadCgroupFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadCgroupFile() = %v, want %v", got, tt.want) + } + if tt.post != nil { + tt.post(t) + } + }) + } +} + +// TestWriteCgroupFile tests WriteCgroupFile +func TestWriteCgroupFile(t *testing.T) { + InitMountDir(constant.TmpTestDir) + defer InitMountDir(constant.DefaultCgroupRoot) + pathElems := []string{"cpu", "kubepods", "PodXXX", "ContYYY", "cpu.cfs_quota_us"} + const value = "-1\n" + type args struct { + content string + elem []string + } + tests := []struct { + name string + args args + wantErr bool + pre func(t *testing.T) + post func(t *testing.T) + }{ + { + name: "TC1-non existed path", + args: args{ + content: value, + elem: pathElems, + }, + wantErr: true, + }, + { + name: "TC2-successfully", + args: args{ + content: value, + elem: pathElems, + }, + pre: func(t *testing.T) { + assert.NoError(t, util.WriteFile( + filepath.Join(constant.TmpTestDir, filepath.Join(pathElems...)), + value)) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre(t) + } + if err := WriteCgroupFile(tt.args.content, tt.args.elem...); (err != nil) != tt.wantErr { + t.Errorf("WriteCgroupFile() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post(t) + } + }) + } +} + +// TestAttr_Expect tests Expect of Attr +func TestAttr_Expect(t *testing.T) { + const ( + intValue int = 1 + int64Value int64 = 1 + stringValue string = "rubik" + float64Value float64 = 1.0 + ) + type fields struct { + Value string + Err error + } + type args struct { + arg interface{} + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "TC1-attribute error", + fields: fields{ + Value: "", + Err: fmt.Errorf("fail to get value"), + }, + wantErr: true, + }, + { + name: "TC2.1-expect int 1(parse fail)", + fields: fields{ + Value: "", + }, + args: args{ + arg: intValue, + }, + wantErr: true, + }, + { + name: "TC2.2-expect int 1(not equal value)", + fields: fields{ + Value: "100", + }, + args: args{ + arg: intValue, + }, + wantErr: true, + }, + { + name: "TC2.3-expect int 1(success)", + fields: fields{ + Value: "1", + }, + args: args{ + arg: intValue, + }, + wantErr: false, + }, + { + name: "TC3.1-expect int64 1(parse fail)", + fields: fields{ + Value: "", + }, + args: args{ + arg: int64Value, + }, + wantErr: true, + }, + { + name: "TC3.2-expect int64 1(not equal value)", + fields: fields{ + Value: "100", + }, + args: args{ + arg: int64Value, + }, + wantErr: true, + }, + { + name: "TC3.3-expect int64 1(success)", + fields: fields{ + Value: "1", + }, + args: args{ + arg: int64Value, + }, + wantErr: false, + }, + { + name: "TC4.1-expect string rubik(not equal value)", + fields: fields{ + Value: "-1", + }, + args: args{ + arg: stringValue, + }, + wantErr: true, + }, + { + name: "TC4.2-expect string rubik(success)", + fields: fields{ + Value: stringValue, + }, + args: args{ + arg: stringValue, + }, + wantErr: false, + }, + { + name: "TC5-expect float64(undefined type)", + args: args{ + arg: float64Value, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attr := &Attr{ + Value: tt.fields.Value, + Err: tt.fields.Err, + } + if err := attr.Expect(tt.args.arg); (err != nil) != tt.wantErr { + t.Errorf("Attr.Expect() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestAttr_Int64 tests Int64 of Attr +func TestAttr_Int64(t *testing.T) { + const int64Value int64 = 1 + type fields struct { + Value string + Err error + } + tests := []struct { + name string + fields fields + want int64 + wantErr bool + }{ + { + name: "TC1-attribute error", + fields: fields{ + Err: fmt.Errorf("fail to get value"), + }, + wantErr: true, + }, + { + name: "TC2-expect int64 1(success)", + fields: fields{ + Value: "1", + }, + want: int64Value, + wantErr: false, + }, + { + name: "TC3-expect int64 1(error parse)", + fields: fields{ + Value: "rubik", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attr := &Attr{ + Value: tt.fields.Value, + Err: tt.fields.Err, + } + got, err := attr.Int64() + if (err != nil) != tt.wantErr { + t.Errorf("Attr.Int64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Attr.Int64() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestAttr_Int tests Int of Attr +func TestAttr_Int(t *testing.T) { + const intValue int = 1 + type fields struct { + Value string + Err error + } + tests := []struct { + name string + fields fields + want int + wantErr bool + }{ + { + name: "TC1-attribute error", + fields: fields{ + Err: fmt.Errorf("fail to get value"), + }, + wantErr: true, + }, + { + name: "TC2-expect int 1(success)", + fields: fields{ + Value: "1", + }, + want: intValue, + wantErr: false, + }, + { + name: "TC3-expect int 1(error parse)", + fields: fields{ + Value: "rubik", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attr := &Attr{ + Value: tt.fields.Value, + Err: tt.fields.Err, + } + got, err := attr.Int() + if (err != nil) != tt.wantErr { + t.Errorf("Attr.Int() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Attr.Int() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestAttr_Int64Map tests Int64Map of Attr +func TestAttr_Int64Map(t *testing.T) { + m := map[string]int64{"a": 1, "b": 2} + type fields struct { + Value string + Err error + } + tests := []struct { + name string + fields fields + want map[string]int64 + wantErr bool + }{ + + { + name: "TC1-attribute error", + fields: fields{ + Err: fmt.Errorf("fail to get value"), + }, + wantErr: true, + }, + { + name: "TC2-expect int 1(success)", + fields: fields{ + Value: `a 1 + b 2`, + }, + want: m, + wantErr: false, + }, + { + name: "TC3-expect int 1(error parse)", + fields: fields{ + Value: "rubik", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attr := &Attr{ + Value: tt.fields.Value, + Err: tt.fields.Err, + } + got, err := attr.Int64Map() + if (err != nil) != tt.wantErr { + t.Errorf("Attr.Int64Map() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Attr.Int64Map() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestAttr_CPUStat tests CPUStat of Attr +func TestAttr_CPUStat(t *testing.T) { + res := &CPUStat{ + NrPeriods: 1, + NrThrottled: 1, + ThrottledTime: 1, + } + type fields struct { + Value string + Err error + } + tests := []struct { + name string + fields fields + want *CPUStat + wantErr bool + }{ + { + name: "TC1-attribute error", + fields: fields{ + Err: fmt.Errorf("fail to get value"), + }, + wantErr: true, + }, + { + name: "TC2-expect int 1(success)", + fields: fields{ + Value: `nr_periods 1 + nr_throttled 1 + throttled_time 1`, + }, + want: res, + wantErr: false, + }, + { + name: "TC3-expect int 1(error parse)", + fields: fields{ + Value: "rubik", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attr := &Attr{ + Value: tt.fields.Value, + Err: tt.fields.Err, + } + got, err := attr.CPUStat() + if (err != nil) != tt.wantErr { + t.Errorf("Attr.CPUStat() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Attr.CPUStat() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestHierarchy_SetCgroupAttr tests SetCgroupAttr of Hierarchy +func TestHierarchy_SetCgroupAttr(t *testing.T) { + type args struct { + key *Key + value string + } + tests := []struct { + name string + path string + args args + wantErr bool + }{ + { + name: "TC1.1-empty key", + args: args{}, + wantErr: true, + }, + { + name: "TC1.2-empty Subsys", + args: args{ + key: &Key{}, + }, + wantErr: true, + }, + { + name: "TC2-", + args: args{ + key: &Key{ + SubSys: "cpu", + FileName: "cpu.cfs_quota_us", + }, + value: "1", + }, + path: "kubepods/PodXXX/ContXXX", + wantErr: true, + }, + } + defer os.RemoveAll(constant.TmpTestDir) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := NewHierarchy(constant.TmpTestDir, tt.path) + if err := h.SetCgroupAttr(tt.args.key, tt.args.value); (err != nil) != tt.wantErr { + t.Errorf("Hierarchy.SetCgroupAttr() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestHierarchy_GetCgroupAttr tests GetCgroupAttr of Hierarchy +func TestHierarchy_GetCgroupAttr(t *testing.T) { + const ( + contPath = "kubepods/PodXXX/ContXXX" + value = " 1\n" + ) + var quotaKey = &Key{ + SubSys: "cpu", + FileName: "cpu.cfs_quota_us", + } + tests := []struct { + name string + path string + args *Key + want string + wantErr bool + pre func(t *testing.T) + post func(t *testing.T) + }{ + { + name: "TC1.1-empty key", + args: nil, + wantErr: true, + }, + { + name: "TC2-empty path", + args: quotaKey, + path: contPath, + wantErr: true, + }, + { + name: "TC3-success", + args: quotaKey, + path: contPath, + pre: func(t *testing.T) { + assert.NoError(t, util.WriteFile( + filepath.Join(constant.TmpTestDir, quotaKey.SubSys, contPath, quotaKey.FileName), + value)) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + want: "1", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := NewHierarchy(constant.TmpTestDir, tt.path) + if tt.pre != nil { + tt.pre(t) + } + got := h.GetCgroupAttr(tt.args) + if (got.Err != nil) != tt.wantErr { + t.Errorf("Hierarchy.GetCgroupAttr() error = %v, wantErr %v", got.Err, tt.wantErr) + } + if got.Err == nil { + if got.Value != tt.want { + t.Errorf("Hierarchy.GetCgroupAttr() = %v, want %v", got, tt.want) + } + } + if tt.post != nil { + tt.post(t) + } + }) + } +} + +// TestAbsoluteCgroupPath tests AbsoluteCgroupPath +func TestAbsoluteCgroupPath(t *testing.T) { + tests := []struct { + name string + args []string + want string + }{ + { + name: "TC1-AbsoluteCgroupPath", + args: []string{"a", "b"}, + want: GetMountDir() + "/a/b", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := AbsoluteCgroupPath(tt.args...); got != tt.want { + t.Errorf("AbsoluteCgroupPath() = %v, want %v", got, tt.want) + } + }) + } +} -- Gitee From 7fc3100e878a6f618168bc98238eb220891629b0 Mon Sep 17 00:00:00 2001 From: vegbir Date: Thu, 23 Mar 2023 20:33:04 +0800 Subject: [PATCH 69/73] test:add subscriber & publisher DT tests coverage: 100.0% of statements Signed-off-by: vegbir --- pkg/core/publisher/genericpublisher_test.go | 143 ++++++++++++++++++ pkg/core/publisher/publisherfactory_test.go | 46 ++++++ pkg/core/subscriber/genericsubscriber_test.go | 62 ++++++++ 3 files changed, 251 insertions(+) create mode 100644 pkg/core/publisher/genericpublisher_test.go create mode 100644 pkg/core/publisher/publisherfactory_test.go create mode 100644 pkg/core/subscriber/genericsubscriber_test.go diff --git a/pkg/core/publisher/genericpublisher_test.go b/pkg/core/publisher/genericpublisher_test.go new file mode 100644 index 0000000..16cd832 --- /dev/null +++ b/pkg/core/publisher/genericpublisher_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-03-23 +// Description: This file tests pod publisher + +// Package publisher implement publisher interface +package publisher + +import ( + "testing" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/core/typedef" +) + +// mockSubscriber is used to mock a subscriber +type mockSubscriber struct { + name string +} + +// ID returns the unique ID of the subscriber +func (s *mockSubscriber) ID() string { + return s.name +} + +// NotifyFunc notifys subscriber event +func (s *mockSubscriber) NotifyFunc(eventType typedef.EventType, event typedef.Event) {} + +// TopicsFunc returns the topics that the subscriber is interested in +func (s *mockSubscriber) TopicsFunc() []typedef.EventType { + return []typedef.EventType{typedef.RAWPODADD} +} + +// Test_genericPublisher_Subscribe tests Subscribe of genericPublisher +func Test_genericPublisher_Subscribe(t *testing.T) { + const subID = "ID" + type fields struct { + topicSubscribersMap map[typedef.EventType]subscriberIDs + subscribers map[string]NotifyFunc + } + type args struct { + s api.Subscriber + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "TC1-subscriber existed", + fields: fields{ + topicSubscribersMap: map[typedef.EventType]subscriberIDs{ + typedef.INFOADD: map[string]struct{}{subID: {}}, + }, + subscribers: map[string]NotifyFunc{subID: nil}, + }, + args: args{ + s: &mockSubscriber{name: subID}, + }, + wantErr: true, + }, + { + name: "TC2-subscriber is not existed", + fields: fields{ + topicSubscribersMap: make(map[typedef.EventType]subscriberIDs), + subscribers: make(map[string]NotifyFunc), + }, + args: args{ + s: &mockSubscriber{name: subID}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pub := newGenericPublisher() + pub.subscribers = tt.fields.subscribers + pub.topicSubscribersMap = tt.fields.topicSubscribersMap + if err := pub.Subscribe(tt.args.s); (err != nil) != tt.wantErr { + t.Errorf("genericPublisher.Subscribe() error = %v, wantErr %v", err, tt.wantErr) + } + pub.Publish(typedef.RAWPODADD, "a") + }) + } +} + +// Test_genericPublisher_Unsubscribe tests Unsubscribe of genericPublisher +func Test_genericPublisher_Unsubscribe(t *testing.T) { + const subID = "ID" + type fields struct { + topicSubscribersMap map[typedef.EventType]subscriberIDs + subscribers map[string]NotifyFunc + } + type args struct { + s api.Subscriber + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "TC1-subscriber existed", + fields: fields{ + topicSubscribersMap: map[typedef.EventType]subscriberIDs{ + typedef.INFOADD: map[string]struct{}{subID: {}}, + }, + subscribers: map[string]NotifyFunc{subID: nil}, + }, + args: args{ + s: &mockSubscriber{name: subID}, + }, + }, + { + name: "TC2-subscriber is not existed", + fields: fields{ + topicSubscribersMap: make(map[typedef.EventType]subscriberIDs), + subscribers: make(map[string]NotifyFunc), + }, + args: args{ + s: &mockSubscriber{name: subID}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pub := &genericPublisher{ + topicSubscribersMap: tt.fields.topicSubscribersMap, + subscribers: tt.fields.subscribers, + } + pub.Unsubscribe(tt.args.s) + }) + } +} diff --git a/pkg/core/publisher/publisherfactory_test.go b/pkg/core/publisher/publisherfactory_test.go new file mode 100644 index 0000000..b84fde1 --- /dev/null +++ b/pkg/core/publisher/publisherfactory_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-03-23 +// Description: This file tests publisher factory + +// Package publisher implement publisher interface +package publisher + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestGetPublisherFactory tests GetPublisherFactory +func TestGetPublisherFactory(t *testing.T) { + tests := []struct { + name string + want *Factory + }{ + { + name: "TC1-success", + want: &Factory{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetPublisherFactory() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetPublisherFactory() = %v, want %v", got, tt.want) + } + assert.NotNil(t, got.GetPublisher(GENERIC)) + const typ publihserType = 100 + assert.Nil(t, got.GetPublisher(typ)) + }) + } +} diff --git a/pkg/core/subscriber/genericsubscriber_test.go b/pkg/core/subscriber/genericsubscriber_test.go new file mode 100644 index 0000000..65bfe91 --- /dev/null +++ b/pkg/core/subscriber/genericsubscriber_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Create: 2023-03-23 +// Description: This file tests the generic subscriber functionality + +// Package subscriber implements generic subscriber interface +package subscriber + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "isula.org/rubik/pkg/api" + "isula.org/rubik/pkg/core/typedef" +) + +type mockEventHandler struct{} + +// HandleEvent handles the event from publisher +func (h *mockEventHandler) HandleEvent(eventType typedef.EventType, event typedef.Event) {} + +// EventTypes returns the intersted event types +func (h *mockEventHandler) EventTypes() []typedef.EventType { + return nil +} + +// TestNewGenericSubscriber tests NewGenericSubscriber +func TestNewGenericSubscriber(t *testing.T) { + type args struct { + handler api.EventHandler + id string + } + tests := []struct { + name string + args args + }{ + { + name: "TC1-NewGenericSubscriber/ID/NotifyFunc/TopicsFunc", + args: args{ + handler: &mockEventHandler{}, + id: "rubik", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewGenericSubscriber(tt.args.handler, tt.args.id) + assert.Equal(t, "rubik", got.ID()) + got.NotifyFunc(typedef.INFOADD, nil) + got.TopicsFunc() + }) + } +} -- Gitee From 8efe86a76da7344ae88c6853c9f2e96be5d0f7f3 Mon Sep 17 00:00:00 2001 From: vegbir Date: Fri, 24 Mar 2023 15:18:26 +0800 Subject: [PATCH 70/73] test: add DT for isula.org/rubik/pkg/common/util coverage: 94.0% of statements Signed-off-by: vegbir --- pkg/common/util/conversion_test.go | 78 +++++++++++ pkg/common/util/file.go | 10 +- pkg/common/util/file_test.go | 206 ++++++++++++++++++++++++++++- pkg/common/util/math.go | 2 +- pkg/common/util/math_test.go | 92 +++++++++++++ 5 files changed, 374 insertions(+), 14 deletions(-) create mode 100644 pkg/common/util/math_test.go diff --git a/pkg/common/util/conversion_test.go b/pkg/common/util/conversion_test.go index 3cea35f..66a52d1 100644 --- a/pkg/common/util/conversion_test.go +++ b/pkg/common/util/conversion_test.go @@ -16,6 +16,7 @@ package util import ( "math" + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -185,3 +186,80 @@ func TestDeepCopy(t *testing.T) { assert.Equal(t, oldSlice, []string{"a", "b", "c"}) assert.Equal(t, newSlice, []string{"az", "bz", "cz"}) } + +// TestParseInt64Map tests ParseInt64Map +func TestParseInt64Map(t *testing.T) { + type args struct { + data string + } + tests := []struct { + name string + args args + want map[string]int64 + wantErr bool + }{ + // TODO: Add test cases. + { + name: "TC1-length of fields is 3", + args: args{ + data: `a 10 10`, + }, + wantErr: true, + }, + { + name: "TC2-the second field is string type", + args: args{ + data: `a a`, + }, + wantErr: true, + }, + { + name: "TC3-success", + args: args{ + data: `a 10`, + }, + want: map[string]int64{"a": 10}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseInt64Map(tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("ParseInt64Map() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseInt64Map() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestPercentageToDecimal tests PercentageToDecimal +func TestPercentageToDecimal(t *testing.T) { + type args struct { + num float64 + } + tests := []struct { + name string + args args + want float64 + }{ + // TODO: Add test cases. + { + name: "TC1-1% to 0.01", + args: args{ + num: 1.0, + }, + want: 0.01, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PercentageToDecimal(tt.args.num); got != tt.want { + t.Errorf("PercentageToDecimal() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/common/util/file.go b/pkg/common/util/file.go index b9b2e0b..970a181 100644 --- a/pkg/common/util/file.go +++ b/pkg/common/util/file.go @@ -11,6 +11,7 @@ // Create: 2021-04-17 // Description: filepath related common functions +// Package util is common utilitization package util import ( @@ -112,9 +113,6 @@ func ReadFile(path string) ([]byte, error) { if IsDir(path) { return nil, fmt.Errorf("%v is not a file", path) } - if !PathExist(path) { - return nil, fmt.Errorf("%v: No such file or directory", path) - } return ioutil.ReadFile(path) } @@ -135,12 +133,12 @@ func WriteFile(path, content string) error { // AppendFile appends content to the file func AppendFile(path, content string) error { - if IsDir(path) { - return fmt.Errorf("%v is not a file", path) - } if !PathExist(path) { return fmt.Errorf("%v: No such file or directory", path) } + if IsDir(path) { + return fmt.Errorf("%v is not a file", path) + } f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, constant.DefaultFileMode) defer func() { if err != f.Close() { diff --git a/pkg/common/util/file_test.go b/pkg/common/util/file_test.go index 62a7de9..5620fed 100644 --- a/pkg/common/util/file_test.go +++ b/pkg/common/util/file_test.go @@ -11,24 +11,27 @@ // Create: 2021-04-17 // Description: filepath related common functions testing +// Package util is common utilitization package util import ( + "fmt" "io/ioutil" "os" "path/filepath" + "reflect" "testing" "github.com/stretchr/testify/assert" - "isula.org/rubik/pkg/common/constant" ) -// TestIsDirectory is IsDirectory function test -func TestIsDirectory(t *testing.T) { +// TestIsDir is IsDir function test +func TestIsDir(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) directory, err := ioutil.TempDir(constant.TmpTestDir, t.Name()) assert.NoError(t, err) - defer os.RemoveAll(directory) filePath, err := ioutil.TempFile(directory, t.Name()) assert.NoError(t, err) @@ -71,9 +74,10 @@ func TestIsDirectory(t *testing.T) { // TestPathIsExist is PathExist function test func TestPathIsExist(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) filePath, err := ioutil.TempDir(constant.TmpTestDir, "file_exist") assert.NoError(t, err) - defer os.RemoveAll(filePath) type args struct { path string @@ -105,9 +109,10 @@ func TestPathIsExist(t *testing.T) { // TestReadSmallFile is test for read file func TestReadSmallFile(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) filePath, err := ioutil.TempDir(constant.TmpTestDir, "read_file") assert.NoError(t, err) - defer os.RemoveAll(filePath) // case1: ok err = ioutil.WriteFile(filepath.Join(filePath, "ok"), []byte{}, constant.DefaultFileMode) @@ -139,7 +144,6 @@ func TestCreateLockFile(t *testing.T) { UnlockFile(lock) assert.NoError(t, lock.Close()) assert.NoError(t, os.Remove(lockFile)) - } // TestLockFail is CreateLockFile fail test @@ -170,3 +174,191 @@ func TestLockFail(t *testing.T) { err = os.RemoveAll(lockFile) assert.NoError(t, err) } + +// TestReadFile tests ReadFile +func TestReadFile(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) + type args struct { + path string + } + tests := []struct { + name string + args args + pre func(t *testing.T) + post func(t *testing.T) + want []byte + wantErr bool + }{ + { + name: "TC1-path is dir", + args: args{ + path: constant.TmpTestDir, + }, + pre: func(t *testing.T) { + _, err := ioutil.TempDir(constant.TmpTestDir, "TC1") + assert.NoError(t, err) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre(t) + } + got, err := ReadFile(tt.args.path) + if (err != nil) != tt.wantErr { + t.Errorf("ReadFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadFile() = %v, want %v", got, tt.want) + } + if tt.post != nil { + tt.post(t) + } + + }) + } +} + +// TestWriteFile tests WriteFile +func TestWriteFile(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) + var filePath = filepath.Join(constant.TmpTestDir, "cpu", "kubepods", "PodXXX") + type args struct { + path string + content string + } + tests := []struct { + name string + args args + pre func(t *testing.T) + post func(t *testing.T) + wantErr bool + }{ + { + name: "TC1-path is dir", + args: args{ + path: constant.TmpTestDir, + }, + pre: func(t *testing.T) { + _, err := ioutil.TempDir(constant.TmpTestDir, "TC1") + assert.NoError(t, err) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: true, + }, + { + name: "TC2-create dir & write file", + args: args{ + path: filePath, + content: "1", + }, + pre: func(t *testing.T) { + assert.NoError(t, os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode)) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre(t) + } + if err := WriteFile(tt.args.path, tt.args.content); (err != nil) != tt.wantErr { + t.Errorf("WriteFile() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.post != nil { + tt.post(t) + } + }) + } +} + +func TestAppendFile(t *testing.T) { + os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode) + defer os.RemoveAll(constant.TmpTestDir) + var ( + dirPath = filepath.Join(constant.TmpTestDir, "cpu", "kubepods", "PodXXX") + filePath = filepath.Join(dirPath, "cpu.cfs_quota_us") + ) + type args struct { + path string + content string + } + tests := []struct { + name string + args args + pre func(t *testing.T) + post func(t *testing.T) + wantErr bool + }{ + { + name: "TC1-path is dir", + args: args{ + path: constant.TmpTestDir, + }, + pre: func(t *testing.T) { + _, err := ioutil.TempDir(constant.TmpTestDir, "TC1") + assert.NoError(t, err) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(filepath.Join(constant.TmpTestDir, "TC1"))) + }, + wantErr: true, + }, + { + name: "TC2-empty path", + args: args{ + path: dirPath, + }, + pre: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: true, + }, + { + name: "TC3-write file success", + args: args{ + path: filePath, + content: "1", + }, + pre: func(t *testing.T) { + assert.NoError(t, os.MkdirAll(dirPath, constant.DefaultDirMode)) + assert.NoError(t, ioutil.WriteFile(filePath, []byte(""), constant.DefaultFileMode)) + }, + post: func(t *testing.T) { + assert.NoError(t, os.RemoveAll(constant.TmpTestDir)) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pre != nil { + tt.pre(t) + } + err := AppendFile(tt.args.path, tt.args.content) + if (err != nil) != tt.wantErr { + t.Errorf("AppendFile() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + fmt.Printf("error: %v\n", err) + } + if tt.post != nil { + tt.post(t) + } + }) + } +} diff --git a/pkg/common/util/math.go b/pkg/common/util/math.go index 854a3ec..c0031b3 100644 --- a/pkg/common/util/math.go +++ b/pkg/common/util/math.go @@ -54,7 +54,7 @@ func Div(dividend, divisor float64, args ...interface{}) float64 { return maxValue } ans := dividend / divisor - if len(format) == 0 { + if len(format) != 0 { if value, err := ParseFloat64(fmt.Sprintf(format, ans)); err == nil { ans = value } diff --git a/pkg/common/util/math_test.go b/pkg/common/util/math_test.go new file mode 100644 index 0000000..5ac2be3 --- /dev/null +++ b/pkg/common/util/math_test.go @@ -0,0 +1,92 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2023-03-24 +// Description: This file is used for testing math + +// Package util is common utilitization +package util + +import ( + "testing" +) + +// TestDiv tests Div +func TestDiv(t *testing.T) { + const ( + dividend float64 = 100.0 + divisor float64 = 1.0 + maxValue float64 = 70.0 + accuracy float64 = 2.0 + format string = "%.2f" + ) + type args struct { + dividend float64 + divisor float64 + args []interface{} + } + tests := []struct { + name string + args args + want float64 + }{ + { + name: "TC1-dividend: 100, divisor: 1, default arguments", + args: args{ + dividend: dividend, + divisor: divisor, + }, + want: dividend, + }, + { + name: "TC2-dividend: 100, divisor: 0, maxValue: 70", + args: args{ + dividend: dividend, + divisor: 0, + args: []interface{}{ + maxValue, + }, + }, + want: maxValue, + }, + { + name: "TC3-dividend: 100, divisor: 1, maxValue: 70, accuracy: 2", + args: args{ + dividend: dividend, + divisor: divisor, + args: []interface{}{ + maxValue, + accuracy, + }, + }, + want: maxValue, + }, + { + name: `TC4-dividend: 3, divisor: 8, format: %.2f`, + args: args{ + dividend: 3, + divisor: 8, + args: []interface{}{ + maxValue, + accuracy, + format, + }, + }, + want: 0.38, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Div(tt.args.dividend, tt.args.divisor, tt.args.args...); got != tt.want { + t.Errorf("Div() = %v, want %v", got, tt.want) + } + }) + } +} -- Gitee From 45a91ce0ed906c98adda3771fcd0861ad5bd59a4 Mon Sep 17 00:00:00 2001 From: hanchao Date: Sat, 25 Mar 2023 13:46:52 +0800 Subject: [PATCH 71/73] rubik: fix the issue that iocost configuration parsing fails Reason: current parser reports an error when parsing an json array --- hack/rubik-daemonset.yaml | 6 ++++++ pkg/config/jsonparser.go | 11 ++--------- pkg/services/iocost/iocost.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hack/rubik-daemonset.yaml b/hack/rubik-daemonset.yaml index b706018..3cbd93a 100644 --- a/hack/rubik-daemonset.yaml +++ b/hack/rubik-daemonset.yaml @@ -100,6 +100,9 @@ spec: - name: sysfs mountPath: /sys/fs readOnly: false + - name: devfs + mountPath: /dev + readOnly: false - name: config-volume mountPath: /var/lib/rubik terminationGracePeriodSeconds: 30 @@ -113,6 +116,9 @@ spec: - name: sysfs hostPath: path: /sys/fs + - name: devfs + hostPath: + path: /dev - name: config-volume configMap: name: rubik-config diff --git a/pkg/config/jsonparser.go b/pkg/config/jsonparser.go index 93c8ccb..7cbc6b2 100644 --- a/pkg/config/jsonparser.go +++ b/pkg/config/jsonparser.go @@ -16,7 +16,6 @@ package config import ( "encoding/json" - "fmt" ) // defaultJsonParser is globally unique json parser @@ -44,17 +43,11 @@ func (parser *jsonParser) ParseConfig(data []byte) (map[string]interface{}, erro // UnmarshalSubConfig deserializes interface to structure func (p *jsonParser) UnmarshalSubConfig(data interface{}, v interface{}) error { - // 1. convert map[string]interface to json string - val, ok := data.(map[string]interface{}) - if !ok { - fmt.Printf("invalid type %T\n", data) - return fmt.Errorf("invalid type %T", data) - } - jsonString, err := json.Marshal(val) + jsonString, err := json.Marshal(data) if err != nil { return err } - // 2. convert json string to struct + // 1. convert json string to struct return json.Unmarshal(jsonString, v) } diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 3fccb9d..4c4bd98 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -114,11 +114,11 @@ func (io *IOCost) SetConfig(f helper.ConfigHandler) error { } var nodeConfigs []NodeConfig - var nodeConfig *NodeConfig if err := f(io.Name, &nodeConfigs); err != nil { return err } + var nodeConfig *NodeConfig for _, config := range nodeConfigs { if config.NodeName == nodeName { nodeConfig = &config -- Gitee From 2f5f76c45d82715b1be2744890d7e575796275b2 Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 25 Mar 2023 14:56:09 +0800 Subject: [PATCH 72/73] optmize: replace CgroupPath with the cgroup.Hierarchy structure Podinfo and ContainerInfo implement `SetCgroupAttr`/`GetCgroupAttr` respectively, which can be replaced by `SetCgroupAttr`/`GetCgroupAttr` of `cgroup.Hierarchy` to remove redundant code. If MountPoint is empty, the global default mount point is used as the mount point of /cgroup. Signed-off-by: vegbir --- pkg/core/typedef/cgroup/common.go | 16 ++++++-- pkg/core/typedef/containerinfo.go | 36 +---------------- pkg/core/typedef/podinfo.go | 30 ++------------- pkg/services/dyncache/dynamic.go | 2 +- pkg/services/dyncache/sync.go | 2 +- pkg/services/dyncache/sync_test.go | 2 +- pkg/services/iocost/iocost.go | 2 +- pkg/services/quotaburst/quotaburst.go | 4 +- pkg/services/quotaturbo/quotaturbo.go | 2 +- pkg/services/quotaturbo/quotaturbo_test.go | 45 +++++++++++----------- test/try/pod.go | 10 ++--- test/try/pod_test.go | 40 +++++++++++-------- 12 files changed, 77 insertions(+), 114 deletions(-) diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 8856962..237015a 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -153,8 +153,8 @@ func (attr *Attr) CPUStat() (*CPUStat, error) { // Hierarchy is used to represent a cgroup path type Hierarchy struct { - MountPoint string - Path string + MountPoint string `json:"mountPoint,omitempty"` + Path string `json:"cgroupPath"` } // NewHierarchy creates a Hierarchy instance @@ -170,7 +170,11 @@ func (h *Hierarchy) SetCgroupAttr(key *Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - return writeCgroupFile(filepath.Join(h.MountPoint, key.SubSys, h.Path, key.FileName), value) + var mountPoint = rootDir + if len(h.MountPoint) > 0 { + mountPoint = h.MountPoint + } + return writeCgroupFile(filepath.Join(mountPoint, key.SubSys, h.Path, key.FileName), value) } // GetCgroupAttr gets cgroup file content @@ -178,7 +182,11 @@ func (h *Hierarchy) GetCgroupAttr(key *Key) *Attr { if err := validateCgroupKey(key); err != nil { return &Attr{Err: err} } - data, err := readCgroupFile(filepath.Join(h.MountPoint, key.SubSys, h.Path, key.FileName)) + var mountPoint = rootDir + if len(h.MountPoint) > 0 { + mountPoint = h.MountPoint + } + data, err := readCgroupFile(filepath.Join(mountPoint, key.SubSys, h.Path, key.FileName)) if err != nil { return &Attr{Err: err} } diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 7ab9d3c..39cb2cc 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -15,7 +15,6 @@ package typedef import ( - "fmt" "path/filepath" "strings" "sync" @@ -63,9 +62,9 @@ func (engine *ContainerEngineType) Prefix() string { // ContainerInfo contains the interested information of container type ContainerInfo struct { + cgroup.Hierarchy Name string `json:"name"` ID string `json:"id"` - CgroupPath string `json:"cgroupPath"` RequestResources ResourceMap `json:"requests,omitempty"` LimitResources ResourceMap `json:"limits,omitempty"` } @@ -76,7 +75,7 @@ func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *Con return &ContainerInfo{ Name: rawContainer.status.Name, ID: id, - CgroupPath: filepath.Join(podCgroupPath, id), + Hierarchy: cgroup.Hierarchy{Path: filepath.Join(podCgroupPath, id)}, RequestResources: requests, LimitResources: limits, } @@ -99,34 +98,3 @@ func (cont *ContainerInfo) DeepCopy() *ContainerInfo { copyObject.RequestResources = cont.RequestResources.DeepCopy() return ©Object } - -// SetCgroupAttr sets the container cgroup file -func (cont *ContainerInfo) SetCgroupAttr(key *cgroup.Key, value string) error { - if err := validateCgroupKey(key); err != nil { - return err - } - return cgroup.WriteCgroupFile(value, key.SubSys, cont.CgroupPath, key.FileName) -} - -// GetCgroupAttr gets container cgroup file content -func (cont *ContainerInfo) GetCgroupAttr(key *cgroup.Key) *cgroup.Attr { - if err := validateCgroupKey(key); err != nil { - return &cgroup.Attr{Err: err} - } - data, err := cgroup.ReadCgroupFile(key.SubSys, cont.CgroupPath, key.FileName) - if err != nil { - return &cgroup.Attr{Err: err} - } - return &cgroup.Attr{Value: strings.TrimSpace(string(data)), Err: nil} -} - -// validateCgroupKey is used to verify the validity of the cgroup key -func validateCgroupKey(key *cgroup.Key) error { - if key == nil { - return fmt.Errorf("key cannot be empty") - } - if len(key.SubSys) == 0 || len(key.FileName) == 0 { - return fmt.Errorf("invalid key") - } - return nil -} diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index 7272faf..907f02b 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -15,18 +15,16 @@ package typedef import ( - "strings" - "isula.org/rubik/pkg/core/typedef/cgroup" ) // PodInfo represents pod type PodInfo struct { - IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` + cgroup.Hierarchy Name string `json:"name"` UID string `json:"uid"` - CgroupPath string `json:"cgroupPath"` Namespace string `json:"namespace"` + IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` Annotations map[string]string `json:"annotations,omitempty"` } @@ -36,7 +34,7 @@ func NewPodInfo(pod *RawPod) *PodInfo { Name: pod.Name, Namespace: pod.Namespace, UID: pod.ID(), - CgroupPath: pod.CgroupPath(), + Hierarchy: cgroup.Hierarchy{Path: pod.CgroupPath()}, IDContainersMap: pod.ExtractContainerInfos(), Annotations: pod.DeepCopy().Annotations, } @@ -68,29 +66,9 @@ func (pod *PodInfo) DeepCopy() *PodInfo { return &PodInfo{ Name: pod.Name, UID: pod.UID, - CgroupPath: pod.CgroupPath, + Hierarchy: pod.Hierarchy, Namespace: pod.Namespace, Annotations: annoMap, IDContainersMap: contMap, } } - -// SetCgroupAttr sets the container cgroup file -func (pod *PodInfo) SetCgroupAttr(key *cgroup.Key, value string) error { - if err := validateCgroupKey(key); err != nil { - return err - } - return cgroup.WriteCgroupFile(value, key.SubSys, pod.CgroupPath, key.FileName) -} - -// GetCgroupAttr gets container cgroup file content -func (pod *PodInfo) GetCgroupAttr(key *cgroup.Key) *cgroup.Attr { - if err := validateCgroupKey(key); err != nil { - return &cgroup.Attr{Err: err} - } - data, err := cgroup.ReadCgroupFile(key.SubSys, pod.CgroupPath, key.FileName) - if err != nil { - return &cgroup.Attr{Err: err} - } - return &cgroup.Attr{Value: strings.TrimSpace(string(data)), Err: nil} -} diff --git a/pkg/services/dyncache/dynamic.go b/pkg/services/dyncache/dynamic.go index 50336c0..09bde4c 100644 --- a/pkg/services/dyncache/dynamic.go +++ b/pkg/services/dyncache/dynamic.go @@ -61,7 +61,7 @@ func (c *DynCache) StartDynamic() { } func getPodCacheMiss(pod *typedef.PodInfo, perfDu int) (int, int) { - cgroupPath := cgroup.AbsoluteCgroupPath("perf_event", pod.CgroupPath, "") + cgroupPath := cgroup.AbsoluteCgroupPath("perf_event", pod.Path, "") if !util.PathExist(cgroupPath) { return 0, 0 } diff --git a/pkg/services/dyncache/sync.go b/pkg/services/dyncache/sync.go index a9b2fb3..8307c41 100644 --- a/pkg/services/dyncache/sync.go +++ b/pkg/services/dyncache/sync.go @@ -61,7 +61,7 @@ func (c *DynCache) SyncCacheLimit() { // writeTasksToResctrl will write tasks running in containers into resctrl group func (c *DynCache) writeTasksToResctrl(pod *typedef.PodInfo) error { - if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", pod.CgroupPath, "")) { + if !util.PathExist(cgroup.AbsoluteCgroupPath("cpu", pod.Path, "")) { // just return since pod maybe deleted return nil } diff --git a/pkg/services/dyncache/sync_test.go b/pkg/services/dyncache/sync_test.go index 67ff07d..8002118 100644 --- a/pkg/services/dyncache/sync_test.go +++ b/pkg/services/dyncache/sync_test.go @@ -173,7 +173,7 @@ func TestCacheLimit_SyncCacheLimit(t *testing.T) { manager := genPodManager(fakePods) for _, pod := range manager.Pods.Pods { pod.Annotations[constant.CacheLimitAnnotationKey] = "low" - try.RemoveAll(cgroup.AbsoluteCgroupPath("cpu", pod.CgroupPath, "")) + try.RemoveAll(cgroup.AbsoluteCgroupPath("cpu", pod.Path, "")) } c.Viewer = manager }, diff --git a/pkg/services/iocost/iocost.go b/pkg/services/iocost/iocost.go index 4c4bd98..9d9eb59 100644 --- a/pkg/services/iocost/iocost.go +++ b/pkg/services/iocost/iocost.go @@ -237,7 +237,7 @@ func (b *IOCost) configPodIOCostWeight(podInfo *typedef.PodInfo) error { weight = onlineWeight } for _, container := range podInfo.IDContainersMap { - if err := ConfigContainerIOCostWeight(container.CgroupPath, weight); err != nil { + if err := ConfigContainerIOCostWeight(container.Path, weight); err != nil { return err } } diff --git a/pkg/services/quotaburst/quotaburst.go b/pkg/services/quotaburst/quotaburst.go index 88c3004..b59cccd 100644 --- a/pkg/services/quotaburst/quotaburst.go +++ b/pkg/services/quotaburst/quotaburst.go @@ -92,7 +92,7 @@ func setPodQuotaBurst(podInfo *typedef.PodInfo) error { const subsys = "cpu" // 1. Try to write container burst value firstly for _, c := range podInfo.IDContainersMap { - cgpath := cgroup.AbsoluteCgroupPath(subsys, c.CgroupPath, "") + cgpath := cgroup.AbsoluteCgroupPath(subsys, c.Path, "") if err := setQuotaBurst(burst, cgpath); err != nil { log.Errorf("set container quota burst failed: %v", err) continue @@ -105,7 +105,7 @@ func setPodQuotaBurst(podInfo *typedef.PodInfo) error { podBurst += burst } // 2. Try to write pod burst value - podPath := cgroup.AbsoluteCgroupPath(subsys, podInfo.CgroupPath, "") + podPath := cgroup.AbsoluteCgroupPath(subsys, podInfo.Path, "") if err := setQuotaBurst(podBurst, podPath); err != nil { log.Errorf("set pod quota burst failed: %v", err) } diff --git a/pkg/services/quotaturbo/quotaturbo.go b/pkg/services/quotaturbo/quotaturbo.go index 0442d85..974b2a1 100644 --- a/pkg/services/quotaturbo/quotaturbo.go +++ b/pkg/services/quotaturbo/quotaturbo.go @@ -120,7 +120,7 @@ func (qt *QuotaTurbo) syncCgroups(conts map[string]*typedef.ContainerInfo) { continue } // add container to quotaturbo - if err := qt.client.AddCgroup(cont.CgroupPath, cont.LimitResources[typedef.ResourceCPU]); err != nil { + if err := qt.client.AddCgroup(cont.Path, cont.LimitResources[typedef.ResourceCPU]); err != nil { log.Errorf("error adding container %v: %v", cont.Name, err) } else { log.Infof("add container %v", id) diff --git a/pkg/services/quotaturbo/quotaturbo_test.go b/pkg/services/quotaturbo/quotaturbo_test.go index e2d910f..49ff033 100644 --- a/pkg/services/quotaturbo/quotaturbo_test.go +++ b/pkg/services/quotaturbo/quotaturbo_test.go @@ -28,6 +28,7 @@ import ( "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/common/util" "isula.org/rubik/pkg/core/typedef" + "isula.org/rubik/pkg/core/typedef/cgroup" "isula.org/rubik/pkg/podmanager" "isula.org/rubik/pkg/services/helper" "isula.org/rubik/test/try" @@ -69,18 +70,18 @@ func TestQuotaTurbo_Terminate(t *testing.T) { fooCont = &typedef.ContainerInfo{ Name: fooContName, ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1/testCon1"}, LimitResources: make(typedef.ResourceMap), } barCont = &typedef.ContainerInfo{ Name: barContName, ID: "testCon2", - CgroupPath: "kubepods/testPod1/testCon2", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1/testCon2"}, LimitResources: make(typedef.ResourceMap), } pod = &typedef.PodInfo{ - UID: "testPod1", - CgroupPath: "kubepods/testPod1", + UID: "testPod1", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1"}, IDContainersMap: map[string]*typedef.ContainerInfo{ fooCont.ID: fooCont, barCont.ID: barCont, @@ -179,15 +180,15 @@ func TestQuotaTurbo_PreStart(t *testing.T) { fooCont = &typedef.ContainerInfo{ Name: fooContName, ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1/testCon1"}, LimitResources: make(typedef.ResourceMap), } pm = &podmanager.PodManager{ Pods: &podmanager.PodCache{ Pods: map[string]*typedef.PodInfo{ podUID: { - UID: podUID, - CgroupPath: "kubepods/testPod1", + UID: podUID, + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1"}, IDContainersMap: map[string]*typedef.ContainerInfo{ fooCont.ID: fooCont, }, @@ -373,17 +374,17 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { ) var ( fooCont = &typedef.ContainerInfo{ - Name: "Foo", - ID: "testCon1", - CgroupPath: "kubepods/testPod1/testCon1", + Name: "Foo", + ID: "testCon1", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1/testCon1"}, LimitResources: typedef.ResourceMap{ typedef.ResourceCPU: math.Min(minCPU, float64(runtime.NumCPU()-1)), }, } barCont = &typedef.ContainerInfo{ - Name: "Bar", - ID: "testCon2", - CgroupPath: "kubepods/testPod2/testCon2", + Name: "Bar", + ID: "testCon2", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod2/testCon2"}, LimitResources: typedef.ResourceMap{ typedef.ResourceCPU: math.Min(minCPU, float64(runtime.NumCPU()-1)), }, @@ -412,8 +413,8 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { }, }, pre: func(t *testing.T, qt *QuotaTurbo) { - preEnv(barCont.CgroupPath) - assert.NoError(t, qt.client.AddCgroup(barCont.CgroupPath, barCont.LimitResources[typedef.ResourceCPU])) + preEnv(barCont.Path) + assert.NoError(t, qt.client.AddCgroup(barCont.Path, barCont.LimitResources[typedef.ResourceCPU])) assert.Equal(t, 1, len(qt.client.GetAllCgroup())) }, post: func(t *testing.T) { @@ -429,10 +430,10 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { }, }, pre: func(t *testing.T, qt *QuotaTurbo) { - preEnv(barCont.CgroupPath) - preEnv(fooCont.CgroupPath) - assert.NoError(t, qt.client.AddCgroup(barCont.CgroupPath, barCont.LimitResources[typedef.ResourceCPU])) - assert.NoError(t, qt.client.AddCgroup(fooCont.CgroupPath, fooCont.LimitResources[typedef.ResourceCPU])) + preEnv(barCont.Path) + preEnv(fooCont.Path) + assert.NoError(t, qt.client.AddCgroup(barCont.Path, barCont.LimitResources[typedef.ResourceCPU])) + assert.NoError(t, qt.client.AddCgroup(fooCont.Path, fooCont.LimitResources[typedef.ResourceCPU])) const cgroupLen = 2 assert.Equal(t, cgroupLen, len(qt.client.GetAllCgroup())) }, @@ -460,9 +461,9 @@ func TestQuotaTurbo_AdjustQuota(t *testing.T) { func TestQuotaTurbo_Run(t *testing.T) { const name = "quotaturbo" var fooPod = &typedef.PodInfo{ - Name: "Foo", - UID: "testPod1", - CgroupPath: "kubepods/testPod1", + Name: "Foo", + UID: "testPod1", + Hierarchy: cgroup.Hierarchy{Path: "kubepods/testPod1"}, Annotations: map[string]string{ constant.QuotaAnnotationKey: "true", }, diff --git a/test/try/pod.go b/test/try/pod.go index 5916a94..18cb0ec 100644 --- a/test/try/pod.go +++ b/test/try/pod.go @@ -43,7 +43,7 @@ func GenFakeContainerInfo(pod *FakePod) *typedef.ContainerInfo { var fakeContainer = &typedef.ContainerInfo{ Name: fmt.Sprintf("fakeContainer-%s", containerID[:idLen]), ID: containerID, - CgroupPath: filepath.Join(pod.CgroupPath, containerID), + Hierarchy: cgroup.Hierarchy{Path: filepath.Join(pod.Path, containerID)}, RequestResources: make(typedef.ResourceMap, 0), LimitResources: make(typedef.ResourceMap, 0), } @@ -58,7 +58,7 @@ func GenFakePodInfo(qosClass corev1.PodQOSClass) *typedef.PodInfo { Name: fmt.Sprintf("fakepod-%s", podID[:idLen]), Namespace: "test", UID: constant.PodCgroupNamePrefix + podID, - CgroupPath: genRelativeCgroupPath(qosClass, podID), + Hierarchy: cgroup.Hierarchy{Path: genRelativeCgroupPath(qosClass, podID)}, Annotations: make(map[string]string, 0), } return fakePod @@ -81,7 +81,7 @@ func (pod *FakePod) genFakePodCgroupPath() Ret { // generate fake cgroup path for key, value := range pod.Keys { // generate pod absolute cgroup path - podCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, pod.CgroupPath, key.FileName) + podCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, pod.Path, key.FileName) if err := WriteFile(podCGFilePath, value); err.err != nil { return err } @@ -97,7 +97,7 @@ func (pod *FakePod) genFakeContainersCgroupPath() Ret { for key, value := range pod.Keys { for _, container := range pod.IDContainersMap { // generate container absolute cgroup path - containerCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, container.CgroupPath, key.FileName) + containerCGFilePath := cgroup.AbsoluteCgroupPath(key.SubSys, container.Path, key.FileName) if err := WriteFile(containerCGFilePath, value); err.err != nil { return err } @@ -123,7 +123,7 @@ func (pod *FakePod) CleanPath() Ret { return newRet(nil) } for key := range pod.Keys { - path := cgroup.AbsoluteCgroupPath(key.SubSys, pod.CgroupPath, key.FileName) + path := cgroup.AbsoluteCgroupPath(key.SubSys, pod.Path, key.FileName) if len(key.FileName) != 0 { path = filepath.Dir(path) } diff --git a/test/try/pod_test.go b/test/try/pod_test.go index b65ce2d..45b3e06 100644 --- a/test/try/pod_test.go +++ b/test/try/pod_test.go @@ -48,10 +48,13 @@ func TestNewFakePod(t *testing.T) { want: &FakePod{ Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, PodInfo: &typedef.PodInfo{ - Name: "fakepod-" + id[:idLen], - UID: id, - Namespace: "test", - CgroupPath: filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBestEffort)), id), + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + Hierarchy: cgroup.Hierarchy{ + Path: filepath.Join(constant.KubepodsCgroup, + strings.ToLower(string(corev1.PodQOSBestEffort)), + id)}, }, }, }, @@ -64,10 +67,12 @@ func TestNewFakePod(t *testing.T) { want: &FakePod{ Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, PodInfo: &typedef.PodInfo{ - Name: "fakepod-" + id[:idLen], - UID: id, - Namespace: "test", - CgroupPath: filepath.Join(constant.KubepodsCgroup, id), + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + Hierarchy: cgroup.Hierarchy{ + Path: filepath.Join(constant.KubepodsCgroup, id), + }, }, }, }, @@ -80,10 +85,13 @@ func TestNewFakePod(t *testing.T) { want: &FakePod{ Keys: map[*cgroup.Key]string{{SubSys: "cpu", FileName: constant.CPUCgroupFileName}: "0"}, PodInfo: &typedef.PodInfo{ - Name: "fakepod-" + id[:idLen], - UID: id, - Namespace: "test", - CgroupPath: filepath.Join(constant.KubepodsCgroup, strings.ToLower(string(corev1.PodQOSBurstable)), id), + Name: "fakepod-" + id[:idLen], + UID: id, + Namespace: "test", + Hierarchy: cgroup.Hierarchy{ + Path: filepath.Join(constant.KubepodsCgroup, + strings.ToLower(string(corev1.PodQOSBurstable)), + id)}, }, }, }, @@ -94,7 +102,7 @@ func TestNewFakePod(t *testing.T) { assert.Equal(t, fakePod.Namespace, tt.want.Namespace) assert.Equal(t, len(fakePod.Name), len(tt.want.Name)) assert.Equal(t, len(fakePod.UID), len(tt.want.UID)) - assert.Equal(t, len(fakePod.CgroupPath), len(tt.want.CgroupPath)) + assert.Equal(t, len(fakePod.Path), len(tt.want.Path)) }) } } @@ -130,11 +138,11 @@ func TestGenFakePod(t *testing.T) { fakePod := GenFakePod(tt.args.keys, tt.args.qosClass) if tt.args.qosClass != corev1.PodQOSGuaranteed { // guaranteed pod does not have path prefix like "guaranteed/podxxx" - assert.Equal(t, true, strings.Contains(fakePod.CgroupPath, strings.ToLower(string(corev1.PodQOSBestEffort)))) + assert.Equal(t, true, strings.Contains(fakePod.Path, strings.ToLower(string(corev1.PodQOSBestEffort)))) } for key, val := range tt.args.keys { - podCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, fakePod.CgroupPath, key.FileName) + podCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, fakePod.Path, key.FileName) assert.Equal(t, true, util.PathExist(podCgroupFile)) ret := ReadFile(podCgroupFile) assert.NoError(t, ret.err) @@ -144,7 +152,7 @@ func TestGenFakePod(t *testing.T) { fakePod.WithContainers(tt.args.containerNum) for key, val := range tt.args.keys { for _, c := range fakePod.IDContainersMap { - containerCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, c.CgroupPath, key.FileName) + containerCgroupFile := cgroup.AbsoluteCgroupPath(key.SubSys, c.Path, key.FileName) assert.Equal(t, true, util.PathExist(containerCgroupFile)) ret := ReadFile(containerCgroupFile) assert.NoError(t, ret.err) -- Gitee From 49437e9d934844c73ec64a651f2062c0fc34e70f Mon Sep 17 00:00:00 2001 From: hanchao Date: Sat, 25 Mar 2023 13:50:19 +0800 Subject: [PATCH 73/73] test:add service and iocost DT --- pkg/services/iocost/iocost_test.go | 479 +++++++++++++++++++++++++++++ pkg/services/service_test.go | 71 +++++ 2 files changed, 550 insertions(+) create mode 100644 pkg/services/iocost/iocost_test.go create mode 100644 pkg/services/service_test.go diff --git a/pkg/services/iocost/iocost_test.go b/pkg/services/iocost/iocost_test.go new file mode 100644 index 0000000..a906d85 --- /dev/null +++ b/pkg/services/iocost/iocost_test.go @@ -0,0 +1,479 @@ +package iocost + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "unicode" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/core/typedef/cgroup" + "isula.org/rubik/pkg/services/helper" + "isula.org/rubik/test/try" +) + +const ( + sysDevBlock = "/sys/dev/block" + objName = "ioCost" + paramsLen = 2 +) + +type ResultItem struct { + testName string + devno string + qosCheck bool + modelCheck bool + qosParam string + modelParam string +} + +var ( + iocostConfigTestItems []IOCostConfig + resultItmes []ResultItem +) + +func createTestIteam(dev, devno string, enable bool, val int64) (*IOCostConfig, *ResultItem) { + qosStr := devno + " enable=0" + name := "Test iocost disable" + if enable { + qosStr = devno + " enable=1" + name = fmt.Sprintf("Test iocost enable: val=%v", val) + } + + cfg := IOCostConfig{ + Dev: dev, + Enable: enable, + Model: "linear", + Param: LinearParam{ + Rbps: val, Rseqiops: val, Rrandiops: val, + Wbps: val, Wseqiops: val, Wrandiops: val, + }, + } + res := ResultItem{ + testName: name, + devno: devno, + qosCheck: true, + modelCheck: enable, + qosParam: qosStr, + modelParam: fmt.Sprintf("%v ctrl=user model=linear rbps=%v rseqiops=%v rrandiops=%v wbps=%v wseqiops=%v wrandiops=%v", + devno, val, val, val, val, val, val), + } + return &cfg, &res +} +func createIOCostConfigTestItems() { + devs, err := getAllBlockDevice() + if err != nil { + panic("get blkck devices error") + } + for dev, devno := range devs { + for _, val := range []int64{600, 700, 800} { + for _, e := range []bool{true, false, true} { + cfg, res := createTestIteam(dev, devno, e, val) + iocostConfigTestItems = append(iocostConfigTestItems, *cfg) + resultItmes = append(resultItmes, *res) + } + } + } + + var dev, devno string + for k, v := range devs { + dev = k + devno = v + break + } + + cfg, res := createTestIteam(dev, devno, true, 900) + iocostConfigTestItems = append(iocostConfigTestItems, *cfg) + resultItmes = append(resultItmes, *res) + + /* + cfg, res = createTestIteam(dev, devno, true, 1000) + res.testName = "Test iocost config no dev" + cfg.Dev = "XXX" + */ +} + +func TestIOCostSupport(t *testing.T) { + assert.Equal(t, ioCostSupport(), true) + cgroup.InitMountDir("/var/tmp/rubik") + assert.Equal(t, ioCostSupport(), false) + cgroup.InitMountDir(constant.DefaultCgroupRoot) +} + +func TestIOCostID(t *testing.T) { + obj := IOCost{ServiceBase: helper.ServiceBase{Name: objName}} + assert.Equal(t, obj.ID(), objName) +} + +func TestIOCostSetConfig(t *testing.T) { + obj := IOCost{ServiceBase: helper.ServiceBase{Name: objName}} + err := obj.SetConfig(nil) + assert.Error(t, err) + + err = obj.SetConfig(func(configName string, d interface{}) error { + return fmt.Errorf("config handler error test") + }) + assert.Error(t, err) + assert.EqualError(t, err, "config handler error test") + + for i, item := range iocostConfigTestItems { + nodeConfig := NodeConfig{ + NodeName: "global", + IOCostConfig: []IOCostConfig{item}, + } + + t.Run(resultItmes[i].testName, func(t *testing.T) { + var nodeConfigs []NodeConfig + nodeConfigs = append(nodeConfigs, nodeConfig) + cfgStr, err := json.Marshal(nodeConfigs) + assert.NoError(t, err) + err = obj.SetConfig(func(configName string, d interface{}) error { + assert.Equal(t, configName, objName) + return json.Unmarshal(cfgStr, d) + }) + assert.NoError(t, err) + checkResult(t, &resultItmes[i]) + }) + } +} + +func TestConfigIOCost(t *testing.T) { + obj := IOCost{ServiceBase: helper.ServiceBase{Name: objName}} + assert.Equal(t, obj.ID(), objName) + + var devname, devno string + devs, err := getAllBlockDevice() + assert.NoError(t, err) + assert.NotEmpty(t, devs) + + for k, v := range devs { + devname = k + devno = v + break + } + + testItems := []struct { + name string + config IOCostConfig + qosCheck bool + modelCheck bool + qosParam string + modelParam string + }{ + { + name: "Test iocost enable", + config: IOCostConfig{ + Dev: devname, + Enable: true, + Model: "linear", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 600, + }, + }, + qosCheck: true, + modelCheck: true, + qosParam: devno + " enable=1", + modelParam: devno + " ctrl=user model=linear " + + "rbps=600 rseqiops=600 rrandiops=600 " + + "wbps=600 wseqiops=600 wrandiops=600", + }, + { + name: "Test iocost disable", + config: IOCostConfig{ + Dev: devname, + Enable: false, + Model: "linear", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 600, + }, + }, + qosCheck: true, + modelCheck: false, + qosParam: devno + " enable=0", + }, + { + name: "Test modifying iocost linear parameters", + config: IOCostConfig{ + Dev: devname, + Enable: true, + Model: "linear", + Param: LinearParam{ + Rbps: 500, Rseqiops: 500, Rrandiops: 500, + Wbps: 500, Wseqiops: 500, Wrandiops: 500, + }, + }, + qosCheck: true, + modelCheck: true, + qosParam: devno + " enable=1", + modelParam: devno + " ctrl=user model=linear " + + "rbps=500 rseqiops=500 rrandiops=500 " + + "wbps=500 wseqiops=500 wrandiops=500", + }, + { + name: "Test iocost disable", + config: IOCostConfig{ + Dev: devname, + Enable: false, + Model: "linear", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 600, + }, + }, + qosCheck: true, + modelCheck: false, + qosParam: devno + " enable=0", + }, + { + name: "Test iocost no dev error", + config: IOCostConfig{ + Dev: "xxx", + Enable: true, + Model: "linear", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 600, + }, + }, + qosCheck: true, + modelCheck: false, + qosParam: devno + " enable=0", + }, + { + name: "Test iocost non-linear error", + config: IOCostConfig{ + Dev: devname, + Enable: true, + Model: "linearx", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 600, + }, + }, + qosCheck: true, + modelCheck: false, + qosParam: devno + " enable=0", + }, + { + name: "Test iocost param error", + config: IOCostConfig{ + Dev: devname, + Enable: true, + Model: "linear", + Param: LinearParam{ + Rbps: 600, Rseqiops: 600, Rrandiops: 600, + Wbps: 600, Wseqiops: 600, Wrandiops: 0, + }, + }, + qosCheck: true, + modelCheck: false, + qosParam: devno + " enable=0", + }, + } + + for _, tt := range testItems { + t.Run(tt.name, func(t *testing.T) { + params := []IOCostConfig{ + tt.config, + } + obj.configIOCost(params) + if tt.qosCheck { + qos, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostQosFile) + assert.NoError(t, err) + qosParams := strings.Split(string(qos), "\n") + for _, qosParam := range qosParams { + paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) + if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { + assert.Equal(t, tt.qosParam, qosParam[:len(tt.qosParam)]) + break + } + } + } + if tt.modelCheck { + modelParamByte, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostModelFile) + assert.NoError(t, err) + modelParams := strings.Split(string(modelParamByte), "\n") + for _, modelParam := range modelParams { + paramList := strings.FieldsFunc(modelParam, unicode.IsSpace) + if len(paramList) >= paramsLen && strings.Compare(paramList[0], devno) == 0 { + assert.Equal(t, tt.modelParam, modelParam[:len(tt.modelParam)]) + break + } + } + } + }) + } +} + +func TestClearIOcost(t *testing.T) { + obj := IOCost{ServiceBase: helper.ServiceBase{Name: objName}} + assert.Equal(t, obj.ID(), objName) + + devs, err := getAllBlockDevice() + assert.NoError(t, err) + for _, devno := range devs { + qosStr := fmt.Sprintf("%v enable=1", devno) + err := cgroup.WriteCgroupFile(qosStr, blkcgRootDir, iocostQosFile) + assert.NoError(t, err) + } + + err = obj.Terminate(nil) + assert.NoError(t, err) + qosParamByte, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostQosFile) + assert.NoError(t, err) + qosParams := strings.Split(string(qosParamByte), "\n") + for _, qosParam := range qosParams { + paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) + if len(paramList) >= paramsLen { + assert.Equal(t, paramList[1], "enable=0") + } + } +} + +func TestSetPodWeight(t *testing.T) { + // deploy enviroment + const podCgroupPath = "/rubik-podtest" + rubikBlkioTestPath := cgroup.AbsoluteCgroupPath(blkcgRootDir, podCgroupPath) + rubikMemTestPath := cgroup.AbsoluteCgroupPath(memcgRootDir, podCgroupPath) + try.MkdirAll(rubikBlkioTestPath, constant.DefaultDirMode) + try.MkdirAll(rubikMemTestPath, constant.DefaultDirMode) + //defer try.RemoveAll(rubikBlkioTestPath) + //defer try.RemoveAll(rubikMemTestPath) + containerPath := podCgroupPath + "/container" + strconv.Itoa(0) + try.MkdirAll(cgroup.AbsoluteCgroupPath(memcgRootDir, containerPath), constant.DefaultDirMode) + try.MkdirAll(cgroup.AbsoluteCgroupPath(blkcgRootDir, containerPath), constant.DefaultDirMode) + + tests := []struct { + name string + cgroupPath string + weight int + wantErr bool + want string + errMsg string + }{ + { + name: "Test online qos level", + cgroupPath: containerPath, + weight: onlineWeight, + wantErr: false, + want: "default 1000\n", + }, + { + name: "Test offline qos level", + cgroupPath: containerPath, + weight: offlineWeight, + wantErr: false, + want: "default 10\n", + }, + { + name: "Test error cgroup path", + cgroupPath: "/var/log/rubik/rubik-test", + weight: offlineWeight, + wantErr: true, + errMsg: "no such file or diretory", + }, + { + name: "Test error value", + cgroupPath: containerPath, + weight: 100000, + wantErr: true, + errMsg: "invalid argument", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ConfigContainerIOCostWeight(tt.cgroupPath, uint64(tt.weight)) + if tt.wantErr { + assert.Contains(t, err.Error(), tt.errMsg) + return + } + assert.NoError(t, err) + + // check weight + weight, err := cgroup.ReadCgroupFile(blkcgRootDir, tt.cgroupPath, iocostWeightFile) + assert.NoError(t, err) + assert.Equal(t, string(weight), tt.want) + + // check cgroup writeback + ino, err := getDirInode(cgroup.AbsoluteCgroupPath(blkcgRootDir, tt.cgroupPath)) + assert.NoError(t, err) + result := fmt.Sprintf("wb_blkio_path:%v\nwb_blkio_ino:%v\n", tt.cgroupPath, ino) + blkioInoStr, err := cgroup.ReadCgroupFile(memcgRootDir, tt.cgroupPath, wbBlkioinoFile) + assert.NoError(t, err) + assert.Equal(t, result, string(blkioInoStr)) + }) + } +} + +func getAllBlockDevice() (map[string]string, error) { + files, err := os.ReadDir(sysDevBlock) + if err != nil { + return nil, err + } + + devs := make(map[string]string) + for _, f := range files { + if f.Type()&os.ModeSymlink != 0 { + fullName := filepath.Join(sysDevBlock, f.Name()) + realPath, err := os.Readlink(fullName) + if err != nil { + continue + } + vs := strings.Split(realPath, "/") + const penultimate = 2 + if len(vs) > penultimate && strings.Compare(vs[len(vs)-penultimate], "block") == 0 { + const excludeBlock = "dm-" + dmLen := len(excludeBlock) + if len(vs[len(vs)-1]) > dmLen && strings.Compare(vs[len(vs)-1][:dmLen], excludeBlock) == 0 { + continue + } + devs[vs[len(vs)-1]] = f.Name() + } + } + } + return devs, nil +} + +func checkResult(t *testing.T, result *ResultItem) { + if result.qosCheck { + qos, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostQosFile) + assert.NoError(t, err) + qosParams := strings.Split(string(qos), "\n") + for _, qosParam := range qosParams { + paramList := strings.FieldsFunc(qosParam, unicode.IsSpace) + if len(paramList) >= paramsLen && strings.Compare(paramList[0], result.devno) == 0 { + assert.Equal(t, result.qosParam, qosParam[:len(result.qosParam)]) + break + } + } + } + if result.modelCheck { + modelParamByte, err := cgroup.ReadCgroupFile(blkcgRootDir, iocostModelFile) + assert.NoError(t, err) + modelParams := strings.Split(string(modelParamByte), "\n") + for _, modelParam := range modelParams { + paramList := strings.FieldsFunc(modelParam, unicode.IsSpace) + if len(paramList) >= paramsLen && strings.Compare(paramList[0], result.devno) == 0 { + assert.Equal(t, result.modelParam, modelParam[:len(result.modelParam)]) + break + } + } + } +} + +func TestMain(m *testing.M) { + if !ioCostSupport() { + fmt.Println("this machine not support iocost") + return + } + createIOCostConfigTestItems() + m.Run() +} diff --git a/pkg/services/service_test.go b/pkg/services/service_test.go new file mode 100644 index 0000000..5b7b608 --- /dev/null +++ b/pkg/services/service_test.go @@ -0,0 +1,71 @@ +package services + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "isula.org/rubik/pkg/feature" +) + +var defaultFeature = []FeatureSpec{ + { + Name: feature.PreemptionFeature, + Default: true, + }, + { + Name: feature.DynCacheFeature, + Default: true, + }, + { + Name: feature.IOLimitFeature, + Default: true, + }, + { + Name: feature.IOCostFeature, + Default: true, + }, + { + Name: feature.DynMemoryFeature, + Default: true, + }, + { + Name: feature.QuotaBurstFeature, + Default: true, + }, + { + Name: feature.QuotaTurboFeature, + Default: true, + }, +} + +func TestErrorInitServiceComponents(t *testing.T) { + errFeatures := []FeatureSpec{ + { + Name: "testFeature", + Default: true, + }, + { + Name: feature.QuotaTurboFeature, + Default: false, + }, + } + + InitServiceComponents(errFeatures) + for _, feature := range errFeatures { + _, err := GetServiceComponent(feature.Name) + assert.Contains(t, err.Error(), "get service failed") + } +} + +func TestInitServiceComponents(t *testing.T) { + InitServiceComponents(defaultFeature) + for _, feature := range defaultFeature { + s, err := GetServiceComponent(feature.Name) + if err != nil { + assert.Contains(t, err.Error(), "this machine not support") + continue + } + assert.NoError(t, err) + assert.Equal(t, s.ID(), feature.Name) + } +} -- Gitee