From 5e784b8cd9d493f9a76f78431a193304267f7c95 Mon Sep 17 00:00:00 2001 From: gongzhengtang Date: Sun, 27 Sep 2020 10:12:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BE=9D=E8=B5=96=E5=9B=BE=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=A7=A3=E6=9E=90=E4=BC=98=E5=8C=96=EF=BC=8C=E4=B8=BB?= =?UTF-8?q?=E8=A6=81=E9=92=88=E5=AF=B9=E6=95=B0=E6=8D=AE=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E9=81=8D=E5=8E=86=E3=80=81=E5=9D=90=E6=A0=87?= =?UTF-8?q?=E7=9A=84=E7=94=9F=E6=88=90=E3=80=81redis=E7=9A=84=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E7=AD=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packageship/packageship/__init__.py | 16 + .../apps/dependinfo/function/graphcache.py | 47 ++- .../apps/dependinfo/function/singlegraph.py | 360 ++++++++++-------- packageship/packageship/libs/conf/__init__.py | 124 ++++++ .../packageship/libs/conf/global_config.py | 111 ++++++ packageship/packageship/package.ini | 18 +- packageship/setup.py | 1 + 7 files changed, 499 insertions(+), 178 deletions(-) create mode 100755 packageship/packageship/libs/conf/__init__.py create mode 100755 packageship/packageship/libs/conf/global_config.py diff --git a/packageship/packageship/__init__.py b/packageship/packageship/__init__.py index e69de29b..8d516bd6 100644 --- a/packageship/packageship/__init__.py +++ b/packageship/packageship/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +""" +The root path of the project +""" +import os +import sys + +if "SETTINGS_FILE_PATH" not in os.environ: + os.environ["SETTINGS_FILE_PATH"] = '/etc/pkgship/package.ini' + + +# The root directory where the system is running +if getattr(sys, 'frozen', False): + BASE_PATH = os.path.dirname(os.path.realpath(sys.argv[0])) +else: + BASE_PATH = os.path.abspath(os.path.dirname(__file__)) diff --git a/packageship/packageship/application/apps/dependinfo/function/graphcache.py b/packageship/packageship/application/apps/dependinfo/function/graphcache.py index b0e7d86a..3ea54e67 100644 --- a/packageship/packageship/application/apps/dependinfo/function/graphcache.py +++ b/packageship/packageship/application/apps/dependinfo/function/graphcache.py @@ -20,28 +20,38 @@ import json from importlib import import_module from functools import partial from redis import Redis, ConnectionPool +from packageship.libs.conf import configuration REDIS_CONN = Redis(connection_pool=ConnectionPool( - host='192.168.205.128', port=6379, max_connections=10, decode_responses=True)) + host=configuration.REDIS_HOST, + port=configuration.REDIS_PORT, + max_connections=configuration.REDIS_MAX_CONNECTIONS, + decode_responses=True)) lock = threading.Lock() def hash_key(encrypt_obj): """ After sorting the values of the dictionary type, calculate the md5 encrypted hash value + + Args: + encrypt_obj:Dictionaries that need to be computed by hash values """ if isinstance(encrypt_obj, dict): - encrypt_obj = sorted(encrypt_obj) + encrypt_obj = {key: encrypt_obj[key] for key in sorted(encrypt_obj)} md5 = hashlib.md5() md5.update(str(encrypt_obj).encode('utf-8')) return md5.hexdigest() -def import_string(module_path): +def get_module(module_path): """ Import a dotted module path and return the attribute/class designated by the last name in the path. Raise ImportError if the import failed. + + Args: + module_path:Module path """ try: module_path, class_name = module_path.rsplit('.', 1) @@ -61,30 +71,43 @@ def import_string(module_path): def _query_depend(query_parameter, depend_relation_str): """ - Query package dependencies + Retrieves dependency data in the RedIS cache or queries + dependencies from the database and saves them in the RedIS cache + + Args: + query_parameter:The condition of the query is a dictionary-type parameter + depend_relation_str:A module string of dependencies + Returns: + A dictionary form of dependency representation """ depend_relation_key = hash_key(query_parameter) - if REDIS_CONN.exists(depend_relation_key): + + def _get_redis_value(): depend_relation = REDIS_CONN.get(depend_relation_key) if depend_relation: depend_relation = json.loads(depend_relation, encoding='utf-8') return depend_relation - # Use thread lock to safely write data in redis + + if REDIS_CONN.exists(depend_relation_key): + return _get_redis_value() + lock.acquire() if not REDIS_CONN.exists(depend_relation_key + '_flag'): REDIS_CONN.set(depend_relation_key + '_flag', 'True') else: REDIS_CONN.set(depend_relation_key + '_flag', 'False') REDIS_CONN.expire(depend_relation_key + '_flag', 600) - flag = REDIS_CONN.get(depend_relation_key + '_flag') lock.release() - if flag != 'True': - return "LOADING" + while not REDIS_CONN.exists(depend_relation_key) and \ + REDIS_CONN.get(depend_relation_key + '_flag') == 'False': + pass + if REDIS_CONN.exists(depend_relation_key): + return _get_redis_value() # query depend relation try: - depend_relation = import_string(depend_relation_str) - except ImportError: - pass + depend_relation = get_module(depend_relation_str) + except ImportError as err: + raise ImportError(err) else: _depend_result = depend_relation.query_depend_relation(query_parameter) REDIS_CONN.set(depend_relation_key, json.dumps(_depend_result)) diff --git a/packageship/packageship/application/apps/dependinfo/function/singlegraph.py b/packageship/packageship/application/apps/dependinfo/function/singlegraph.py index 4349064a..f9c7008a 100644 --- a/packageship/packageship/application/apps/dependinfo/function/singlegraph.py +++ b/packageship/packageship/application/apps/dependinfo/function/singlegraph.py @@ -14,7 +14,6 @@ Data analysis of dependency graph """ import random -from retrying import retry from packageship.application.apps.package.function.searchdb import db_priority from packageship.application.apps.package.function.constants import ResponseCode from packageship.application.apps.package.serialize import BeDependSchema @@ -25,17 +24,23 @@ from packageship.application.apps.package.function.self_depend import SelfDepend from packageship.application.apps.package.function.install_depend import InstallDepend from packageship.application.apps.package.function.build_depend import BuildDepend from packageship.application.apps.package.function.be_depend import BeDepend +from packageship.libs.log import Log from .graphcache import self_build, bedepend, build_depend, install_depend LEVEL_RADIUS = 30 -NODE_SIZE = 60 +NODE_SIZE = 25 PACKAGE_NAME = 0 +LOGGER = Log(__name__) class SelfBuildDep: """ - Self-compilation dependent data query analysis + Description: Self-compilation dependent data query analysis + + Attributes: + graph:Diagram of an underlying operation instance + query_parameter:Parameters for a dependency query """ def __init__(self, graph): @@ -48,7 +53,8 @@ class SelfBuildDep: 'withsubpack': self.graph.withsubpack } - def _validate(self): + def validate(self): + """Verify the validity of the data""" depend = SelfDependSchema().validate(self.query_parameter) if depend: return False @@ -57,15 +63,21 @@ class SelfBuildDep: @staticmethod def query_depend_relation(query_parameter): """ - Query dependency data + Self-compile dependent relational queries + + Args: + query_parameter:Parameters for a dependency query """ + db_list = query_parameter['db_list'] + packagename = query_parameter['packagename'] + selfbuild = int(query_parameter['selfbuild']) + withsubpack = int(query_parameter['withsubpack']) + packtype = query_parameter['packtype'] _response_code, binary_dicts, source_dicts, not_fd_components = \ - SelfDepend(query_parameter['db_list']).query_depend(query_parameter['packagename'], - int( - query_parameter['selfbuild']), - int( - query_parameter['withsubpack']), - query_parameter['packtype']) + SelfDepend(db_list).query_depend(packagename, + selfbuild, + withsubpack, + packtype) return { "binary_dicts": binary_dicts, "source_dicts": source_dicts, @@ -73,28 +85,19 @@ class SelfBuildDep: } def __call__(self): - if not self._validate(): - return ResponseCode.PARAM_ERROR - database_error = self.graph.database_priority() - if database_error: - return database_error - - @retry(wait_random_min=50, wait_random_max=500) def _query_depend(): query_result = self_build(self.query_parameter) - if query_result == 'LOADING': - raise Exception() - self.graph.package_datas = query_result['binary_dicts'] - - _query_depend() - if self.graph.package_datas: - self.graph.graph_data() - return ResponseCode.SUCCESS + return query_result['binary_dicts'] + return self.graph.get_depend_relation_data(self, _query_depend) class InstallDep: """ Installation dependent data query analysis + + Attributes: + graph:Diagram of an underlying operation instance + query_parameter:Parameters for a dependency query """ def __init__(self, graph): @@ -104,7 +107,8 @@ class InstallDep: 'db_list': self.graph.dbname } - def _validate(self): + def validate(self): + """Verify the validity of the data""" depend = InstallDependSchema().validate(self.query_parameter) if depend: return False @@ -113,39 +117,34 @@ class InstallDep: @staticmethod def query_depend_relation(query_parameter): """ - Query dependency data + Install dependent relational queries + + Args: + query_parameter:Parameters for a dependency query """ + db_list = query_parameter['db_list'] + binary_name = query_parameter['binaryName'] _response_code, install_dict, not_found_components = \ - InstallDepend(query_parameter['db_list'] - ).query_install_depend([query_parameter['binaryName']]) + InstallDepend(db_list).query_install_depend([binary_name]) return { "install_dict": install_dict, 'not_found_components': list(not_found_components) } def __call__(self): - if not self._validate(): - return ResponseCode.PARAM_ERROR - database_error = self.graph.database_priority() - if database_error: - return database_error - - @retry(wait_random_min=50, wait_random_max=500) def _query_depend(): query_result = install_depend(self.query_parameter) - if query_result == 'LOADING': - raise Exception() - self.graph.package_datas = query_result['install_dict'] - - _query_depend() - if self.graph.package_datas: - self.graph.graph_data() - return ResponseCode.SUCCESS + return query_result['install_dict'] + return self.graph.get_depend_relation_data(self, _query_depend) class BuildDep: """ Compile dependent data query analysis + + Attributes: + graph:Diagram of an underlying operation instance + query_parameter:Parameters for a dependency query """ def __init__(self, graph): @@ -155,7 +154,8 @@ class BuildDep: 'db_list': self.graph.dbname } - def _validate(self): + def validate(self): + """Verify the validity of the data""" depend = BuildDependSchema().validate(self.query_parameter) if depend: return False @@ -164,10 +164,14 @@ class BuildDep: @staticmethod def query_depend_relation(query_parameter): """ - Query dependency data + Compile dependent relational queries + + Args: + query_parameter:Parameters for a dependency query """ - build_ins = BuildDepend( - [query_parameter['sourceName']], query_parameter['db_list']) + source_name = query_parameter['sourceName'] + db_list = query_parameter['db_list'] + build_ins = BuildDepend([source_name], db_list) _res_code, builddep_dict, _, not_found_components = build_ins.build_depend_main() return { 'build_dict': builddep_dict, @@ -175,28 +179,19 @@ class BuildDep: } def __call__(self): - if not self._validate(): - return ResponseCode.PARAM_ERROR - database_error = self.graph.database_priority() - if database_error: - return database_error - - @retry(wait_random_min=50, wait_random_max=500) def _query_depend(): query_result = build_depend(self.query_parameter) - if query_result == 'LOADING': - raise Exception() - self.graph.package_datas = query_result['build_dict'] - - _query_depend() - if self.graph.package_datas: - self.graph.graph_data() - return ResponseCode.SUCCESS + return query_result['build_dict'] + return self.graph.get_depend_relation_data(self, _query_depend) class BeDependOn: """ - Dependent query + Dependent relational queries + + Attributes: + graph:Diagram of an underlying operation instance + query_parameter:Parameters for a dependency query """ def __init__(self, graph): @@ -210,7 +205,7 @@ class BeDependOn: 'withsubpack': self.graph.withsubpack } - def _validate(self): + def validate(self): """Verify the validity of the data""" bedependon = BeDependSchema().validate(self.query_parameter) if bedependon: @@ -220,33 +215,23 @@ class BeDependOn: @staticmethod def query_depend_relation(query_parameter): """ - Query dependency data + Dependent relational queries + + Args: + query_parameter:Parameters for a dependency query """ - bedepnd_ins = BeDepend( - query_parameter['packagename'], - query_parameter['dbname'], - query_parameter['withsubpack']) + packagename = query_parameter['packagename'] + db_name = query_parameter['dbname'] + withsubpack = query_parameter['withsubpack'] + bedepnd_ins = BeDepend(packagename, db_name, withsubpack) be_depend_dict = bedepnd_ins.main() return be_depend_dict def __call__(self): - if not self._validate(): - return ResponseCode.PARAM_ERROR - database_error = self.graph.database_priority() - if database_error: - return database_error - @retry(wait_random_min=50, wait_random_max=500) def _query_depend(): - query_result = bedepend(self.query_parameter) - if query_result == 'LOADING': - raise Exception() - self.graph.package_datas = query_result - - _query_depend() - if self.graph.package_datas: - self.graph.graph_data() - return ResponseCode.SUCCESS + return bedepend(self.query_parameter) + return self.graph.get_depend_relation_data(self, _query_depend) class BaseGraph: @@ -269,14 +254,18 @@ class BaseGraph: 'The query parameter type is wrong, and normal', ' dependent data analysis cannot be completed') self.graph = depend_graph(self) - self._color = ['#E02020', '#FA6400', '#F78500', '#6DD400', - '#44D7B6', '#32C5FF', '#0091FF', '#6236FF', '#B620E0', '#6D7278'] + self._color = ['#E02020', '#FA6400', '#F78500', '#6DD400', '#44D7B6', + '#32C5FF', '#0091FF', '#6236FF', '#B620E0', '#6D7278'] self.nodes = dict() self.edges = list() self.depend_package = dict() - self.package_datas = {} - self.up_depend_node = None - self.down_depend_nodes = None + self.package_datas = { + 'uplevel': dict(), + 'downlevel': dict() + } + self.up_depend_node = list() + self.down_depend_nodes = list() + self._quadrant = [1, -1] def __getattr__(self, value): if value not in self.__dict__: @@ -286,40 +275,34 @@ class BaseGraph: @property def color(self): """rgb random color value acquisition""" - return self._color[random.randint(0, len(self._color)-1)] + return self._color[random.randint(0, 9)] - @staticmethod - def dynamic_coordinate(level, upper_layer=True): + @property + def quadrant(self): + """Get the coordinate quadrant at random""" + return self._quadrant[random.randint(0, 1)] + + @property + def coordinate(self): """ Dynamically calculate the random coordinates of each package in the current level - Args: - level: The level of the package - upper_layer:Query the upper level of the package Returns: The coordinate value of the dynamically calculated dependent package example : (x,y) """ - min_value, max_value = (level - 1) * LEVEL_RADIUS, level * LEVEL_RADIUS - _x, _y = random.uniform(min_value, max_value), random.uniform( - min_value, max_value) - if not upper_layer: - _x, _y = _x * -1, _y * -1 + _x, _y = random.uniform(0, LEVEL_RADIUS) * self.quadrant, random.uniform( + 0, LEVEL_RADIUS) * self.quadrant return _x, _y - @staticmethod - def dynamin_node_size(level): - """ - Dynamically calculate the size of each node - """ - min_value, max_value = int( - NODE_SIZE / (level + 1)), int(NODE_SIZE / level) - node_size = random.randint( - min_value, max_value) * (1 - level / 6 * 1.0) + @property + def node_size(self): + """Dynamically calculate the size of each node """ + node_size = random.uniform(1, NODE_SIZE) return node_size - def database_priority(self): - """Priority list of databases""" + def _database_priority(self): + """Verify the validity of the query database""" databases = db_priority() if not databases: @@ -334,25 +317,27 @@ class BaseGraph: def create_dict(**kwargs): """ Create dictionary data + + Args: + kwargs: Create each key-Val key-value pair for the dictionary """ if isinstance(kwargs, dict): return kwargs return dict() - def combination_nodes(self, level_depend, package_name, upper_layer=True, root=True): + def _combination_nodes(self, package_name, root=True): """ Regroup node values Args: - level_depend:Level of dependency package_name:Dependent package name - upper_layer:The direction of dependency, upper-level - dependency or lower-level dependency + root:he coordinate value of the root node """ + _size = self.node_size if root: _x, _y = 0, 0 + _size = 30 else: - _x, _y = BaseGraph.dynamic_coordinate(level_depend, upper_layer) - _size = BaseGraph.dynamin_node_size(level_depend) + _x, _y = self.coordinate self.nodes[package_name] = BaseGraph.create_dict( color=self.color, label=package_name, @@ -361,12 +346,11 @@ class BaseGraph: id=package_name, size=_size) - def combination_edges(self, source_package_name, target_package_name): + def _combination_edges(self, source_package_name, target_package_name): """ Depend on the data combination of the edges node in the graph Args: source_package_name:Source node - level_depend:Level of dependency target_package_name:Target node """ self.edges.append(BaseGraph.create_dict( @@ -374,64 +358,54 @@ class BaseGraph: targetID=target_package_name, )) - def up_level_depend(self, level_depend): + def _up_level_depend(self): """ Data analysis of the previous layer - Args: - level_depend:Level of dependency """ _up_depend_nodes = [] for node_name in self.up_depend_node: - depend_data = self.package_datas[node_name][-1] - if depend_data: - for depend_item in depend_data: - _up_depend_nodes.append(depend_item[PACKAGE_NAME]) - self.combination_nodes( - level_depend, depend_item[PACKAGE_NAME], root=False) - self.combination_edges( - node_name, depend_item[PACKAGE_NAME]) + if node_name not in self.package_datas['uplevel'].keys(): + continue + depend_data = self.package_datas['uplevel'][node_name] + for depend_item in depend_data: + _up_depend_nodes.append(depend_item) + self._combination_nodes( + depend_item, root=False) + self._combination_edges( + node_name, depend_item) + self.up_depend_node = list(set(_up_depend_nodes)) - def down_level_depend(self, level_depend): + def _down_level_depend(self): """ Specify the next level of dependencies of dependent nodes - Args: - level_depend:Level of dependency """ _down_depend_nodes = [] - for package_name, package_depend in self.package_datas.items(): - for depend_item in package_depend[-1]: - if depend_item[PACKAGE_NAME] in self.down_depend_nodes: - _down_depend_nodes.append(package_name) - self.combination_nodes( - level_depend, package_name, False, root=False) - self.combination_edges( - depend_item[PACKAGE_NAME], package_name) + for node_name in self.down_depend_nodes: + if node_name not in self.package_datas['downlevel'].keys(): + continue + depend_data = self.package_datas['downlevel'][node_name] + for depend_item in depend_data: + _down_depend_nodes.append(depend_item) + self._combination_nodes( + depend_item, root=False) + self._combination_edges( + depend_item, node_name) + self.down_depend_nodes = list(set(_down_depend_nodes)) - def graph_data(self): + def _graph_data(self): """ Resolve the data in the dependency graph """ def depend_package(): - self.combination_nodes(1, self.node_name, root=False) - for level in range(1, 3): - self.up_level_depend(level) - self.down_level_depend(level) - - def source_depend_relation(): - for package_name, package_depend in self.package_datas.items(): - if package_depend[PACKAGE_NAME] == self.node_name: - self.up_depend_node.append(package_name) - self.down_depend_nodes.append(package_name) - - if self.packagetype == 'source': - self.up_depend_node, self.down_depend_nodes = list(), list() - source_depend_relation() - - if self.packagetype == "binary": - self.up_depend_node = [self.node_name] - self.down_depend_nodes = [self.node_name] + if self.packagetype == "binary": + self.up_depend_node.append(self.node_name) + self.down_depend_nodes.append(self.node_name) + self._combination_nodes(self.node_name) + for _level in range(1, 3): + self._up_level_depend() + self._down_level_depend() depend_package() self.depend_package = { @@ -439,10 +413,66 @@ class BaseGraph: 'edges': self.edges } + def _relation_recombine(self, package_datas): + """ + The data in the dependency query is recombined + into representations of the upper and lower dependencies + of the current node + + Args: + package_datas:Package dependency data + + """ + for package_name, package_depend in package_datas.items(): + if not package_depend or not isinstance(package_depend, list): + continue + if self.packagetype == 'source' and package_depend[PACKAGE_NAME] == self.node_name: + self.up_depend_node.append(package_name) + self.down_depend_nodes.append(package_name) + + for depend_item in package_depend[-1]: + if depend_item[PACKAGE_NAME] == 'root': + continue + if not self.package_datas['uplevel'].__contains__(package_name): + self.package_datas['uplevel'][package_name] = list() + if not self.package_datas['downlevel'].__contains__(depend_item[PACKAGE_NAME]): + self.package_datas['downlevel'][depend_item[PACKAGE_NAME]] = list( + ) + self.package_datas['uplevel'][package_name].append( + depend_item[PACKAGE_NAME]) + self.package_datas['downlevel'][depend_item[PACKAGE_NAME]].append( + package_name) + + def get_depend_relation_data(self, depend, func): + """ + Get data for different dependencies + + Args: + depend:Each of the dependent instance objects + SelfBuildDep()、BuildDep()、InstallDep()、BeDependOn() + func:Methods to query dependencies + """ + + if not depend.validate(): + return (ResponseCode.PARAM_ERROR, ResponseCode.CODE_MSG_MAP[ResponseCode.PARAM_ERROR]) + database_error = self._database_priority() + if database_error: + return database_error + _package_datas = func() + + if _package_datas: + self._relation_recombine(_package_datas) + try: + self._graph_data() + except KeyError as error: + LOGGER.logger.error(error) + return (ResponseCode.SERVICE_ERROR, ResponseCode.CODE_MSG_MAP[ResponseCode.SERVICE_ERROR]) + return (ResponseCode.SUCCESS, ResponseCode.CODE_MSG_MAP[ResponseCode.SUCCESS]) + def parse_depend_graph(self): """Analyze the data that the graph depends on""" - response_status = self.graph() + response_status, _msg = self.graph() if response_status != ResponseCode.SUCCESS: - return (response_status, None) + return (response_status, _msg, None) - return (response_status, self.depend_package) + return (response_status, _msg, self.depend_package) diff --git a/packageship/packageship/libs/conf/__init__.py b/packageship/packageship/libs/conf/__init__.py new file mode 100755 index 00000000..fbf34b13 --- /dev/null +++ b/packageship/packageship/libs/conf/__init__.py @@ -0,0 +1,124 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +System configuration file and default configuration file integration +""" +import os +import configparser + +from packageship.libs.exception import Error +from . import global_config + + +USER_SETTINGS_FILE_PATH = 'SETTINGS_FILE_PATH' + + +class PreloadingSettings(): + """ + The system default configuration file and the configuration + file changed by the user are lazily loaded. + """ + _setting_container = None + + def _preloading(self): + """ + Load the default configuration in the system and the related configuration + of the user, and overwrite the default configuration items of the system + with the user's configuration data + """ + settings_file = os.environ.get(USER_SETTINGS_FILE_PATH) + if not settings_file: + raise Error( + "The system does not specify the user configuration" + "that needs to be loaded:" % USER_SETTINGS_FILE_PATH) + + self._setting_container = Configs(settings_file) + + def __getattr__(self, name): + """ + Return the value of a setting and cache it in self.__dict__ + """ + if self._setting_container is None: + self._preloading() + value = getattr(self._setting_container, name, None) + self.__dict__[name] = value + return value + + def __setattr__(self, name, value): + """ + Set the configured value and re-copy the value cached in __dict__ + """ + if name is None: + raise KeyError("The set configuration key value cannot be empty") + if name == '_setting_container': + self.__dict__.clear() + self.__dict__["_setting_container"] = value + else: + self.__dict__.pop(name, None) + if self._setting_container is None: + self._preloading() + setattr(self._setting_container, name, value) + + def __delattr__(self, name): + """ + Delete a setting and clear it from cache if needed + """ + if name is None: + raise KeyError("The set configuration key value cannot be empty") + + if self._setting_container is None: + self._preloading() + delattr(self._setting_container, name) + self.__dict__.pop(name, None) + + @property + def config_ready(self): + """ + Return True if the settings have already been configured + """ + return self._setting_container is not None + + +class Configs(): + """ + The system's default configuration items and the user's + configuration items are integrated + """ + + def __init__(self, settings_file): + for config in dir(global_config): + if not config.startswith('_'): + setattr(self, config, getattr(global_config, config)) + + # Load user's configuration + self._conf_parser = configparser.ConfigParser() + self._conf_parser.read(settings_file) + + for section in self._conf_parser.sections(): + for option in self._conf_parser.items(section): + try: + _config_value = option[1] + _key = option[0] + except IndexError: + pass + else: + if not _config_value: + continue + if _config_value.isdigit(): + _config_value = int(_config_value) + elif _config_value.lower() in ('true', 'false'): + _config_value = bool(_config_value) + setattr(self, _key.upper(), _config_value) + + +configuration = PreloadingSettings() diff --git a/packageship/packageship/libs/conf/global_config.py b/packageship/packageship/libs/conf/global_config.py new file mode 100755 index 00000000..f1c1fa01 --- /dev/null +++ b/packageship/packageship/libs/conf/global_config.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +Global environment variable value when the system is running +""" + +import os + +# Configuration file path for data initialization +INIT_CONF_PATH = os.path.join('/', 'etc', 'pkgship', 'conf.yaml') + +# If the path of the imported database is not specified in the configuration file, the +# configuration in the system is used by default +DATABASE_FOLDER_PATH = os.path.join('/', 'var', 'run', 'pkgship_dbs') + +# The database engine supported in the system is sqlite database by default +DATABASE_ENGINE_TYPE = 'sqlite' + +# Port managed by the administrator, with write permission +WRITE_PORT = 8080 + +# Ordinary user query port, only the right to query data, no permission to write data +QUERY_PORT = 8090 + +# IP address path with write permission +WRITE_HOST = '127.0.0.1' + +# IP address path with permission to query data +QUERY_HOST = '127.0.0.1' + +# The address of the remote service, the command line can directly +# call the remote service to complete the data request +REMOTE_HOST = 'https://api.openeuler.org/pkgmanage' + +# If the directory of log storage is not configured, +# it will be stored in the following directory specified by the system by default +LOG_PATH = os.path.join('/', 'var', 'log', 'pkgship') + +# Logging level +# The log level option value can only be as follows +# INFO DEBUG WARNING ERROR CRITICAL +LOG_LEVEL = 'INFO' + +# logging name +LOG_NAME = 'log_info.log' + +# The number of dynamically created logs +# after the log file size reaches the upper limit. The default is 10 dynamically created logs +BACKUP_COUNT = 10 + +# The size of each log file, in bytes, the default size of a single log file is 300M +MAX_BYTES = 314572800 + +# Execution frequency and switch of timing tasks +# Whether to execute the switch for batch update of information such +# as the maintainer during initialization. When set to True, the maintainer +# and other information will be updated when the scheduled task starts +# to execute. When it is set to False, it will be updated when the scheduled +# task is executed. , Does not update information such as maintainers and maintenance levels + +OPEN = True + +# The time point at which the timing task is executed in a cycle. +# Every day is a cycle, and the time value can only be any integer period between 0 and 23 +HOUR = 3 + +# Recurring timing task starts to execute the task at the current time point. +# The value range of this configuration item is an integer between 0 and 59 +MINUTE = 0 + +# Configuration during the life cycle for tag information, issue and other information acquisition +# The yaml address of each package is stored in the remote address, which can be +# a remote warehouse address or the address of a static resource service +WAREHOUSE_REMOTE = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' + +# When performing timing tasks, you can open multi-threaded execution, and you can set +# the number of threads in the thread pool according to the configuration of the server + +POOL_WORKERS = 10 + +# The main body of the warehouse, the owner of the warehouse +# When this value is not set, the system will default to src-openeuler +WAREHOUSE = 'src-openeuler' + +# Maximum queue length +QUEUE_MAXSIZE = 1000 + + +# The address of the Redis cache server can be either a published +# domain or an IP address that can be accessed normally +# The link address defaults to 127.0.0.1 +# redis_host = 127.0.0.1 + +REDIS_HOST = '127.0.0.1' + +# Redis cache server link port number, default is 6379 +REDIS_PORT = 6379 + +# Maximum number of connections allowed by RedIS server at one time + +REDIS_MAX_CONNECTIONS = 10 diff --git a/packageship/packageship/package.ini b/packageship/packageship/package.ini index 93115caf..1767aa27 100644 --- a/packageship/packageship/package.ini +++ b/packageship/packageship/package.ini @@ -94,4 +94,20 @@ pool_workers=10 warehouse=src-openeuler ; Maximum queue length -queue_maxsize = 1000 \ No newline at end of file +queue_maxsize = 1000 + +[REDIS] + +# The address of the Redis cache server can be either a published +# domain or an IP address that can be accessed normally +# The link address defaults to 127.0.0.1 +# redis_host = 127.0.0.1 + +redis_host= 127.0.0.1 + +# Redis cache server link port number, default is 6379 +redis_port= 6379 + +# Maximum number of connections allowed by RedIS server at one time + +redis_max_connections= 10 diff --git a/packageship/setup.py b/packageship/setup.py index 0c7084ee..5df112c2 100644 --- a/packageship/setup.py +++ b/packageship/setup.py @@ -43,6 +43,7 @@ setup( 'packageship.libs.dbutils.sqlalchemy_helper', 'packageship.libs.exception.ext', 'packageship.libs.log.loghelper', + 'packageship.libs.conf.global_config', 'packageship.manage', 'packageship.pkgship', 'packageship.selfpkg', -- Gitee