diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 0045de0fbf66b737206da1311b2c72242daf8906..99930cbdc665bfba08f50b0a50a3251a69ae1011 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -13,8 +13,8 @@ from packageship.libs.exception import ContentNoneException from packageship.libs.exception import DatabaseRepeatException from packageship.libs.exception import Error from packageship.libs.exception import ConfigurationException -from packageship.libs.configutils.readconfig import ReadConfig -from packageship.libs.log import Log +from packageship.libs.log import LOGGER +from packageship.libs.conf import configuration from packageship.application.models.package import SrcPack from packageship.application.models.package import DatabaseInfo from packageship.application.models.package import BinPack @@ -23,9 +23,6 @@ from packageship.application.models.package import SrcRequires from packageship.application.models.package import BinProvides from packageship.application.models.package import BinFiles from packageship.application.models.package import Packages -from packageship import system_config - -LOGGER = Log(__name__) class InitDataBase(): @@ -47,12 +44,10 @@ class InitDataBase(): config_file_path: Configuration file path """ self.config_file_path = config_file_path - if self.config_file_path: # yaml configuration file content self.config_file_datas = self.__read_config_file() - self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) - self.db_type = 'sqlite' + self.db_type = configuration.DATABASE_ENGINE_TYPE self.sql = None self._database = None self._sqlite_db = None @@ -727,28 +722,20 @@ class SqliteDatabaseOperations(): kwargs: data related to configuration file nodes """ self.db_name = db_name - self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) - if getattr(kwargs, 'database_path', None) is None: - self._database_file_path() - else: + self.database_file_folder = configuration.DATABASE_FOLDER_PATH + if hasattr(kwargs, 'database_path'): self.database_file_folder = kwargs.get('database_path') + self._make_folder() self.tables = tables self.storage = storage - def _database_file_path(self): + def _make_folder(self): """ - Database file path - - Returns: + Create a folder to hold the database Raises: IOError: File or network operation io abnormal """ - self.database_file_folder = self._read_config.get_system( - 'data_base_path') - if not self.database_file_folder: - self.database_file_folder = system_config.DATABASE_FOLDER_PATH - if not os.path.exists(self.database_file_folder): try: os.makedirs(self.database_file_folder) diff --git a/packageship/packageship/libs/conf/global_config.py b/packageship/packageship/libs/conf/global_config.py index 0193d8114a214ef0da8e6cf0b54a4cce4e00de13..0b433a4f73b3b413c3f69f32157708b58fe87a09 100755 --- a/packageship/packageship/libs/conf/global_config.py +++ b/packageship/packageship/libs/conf/global_config.py @@ -33,10 +33,10 @@ WRITE_PORT = 8080 QUERY_PORT = 8090 # IP address path with write permission -WRITE_HOST = '127.0.0.1' +WRITE_IP_ADDR = '127.0.0.1' # IP address path with permission to query data -QUERY_HOST = '127.0.0.1' +QUERY_IP_ADDR = '127.0.0.1' # The address of the remote service, the command line can directly # call the remote service to complete the data request diff --git a/packageship/packageship/libs/dbutils/sqlalchemy_helper.py b/packageship/packageship/libs/dbutils/sqlalchemy_helper.py index 9e75e4c8d47555ea4eaa9c77df63e684cc158b35..84d39556d10f70171819f8297e4d5967ca70e1ea 100644 --- a/packageship/packageship/libs/dbutils/sqlalchemy_helper.py +++ b/packageship/packageship/libs/dbutils/sqlalchemy_helper.py @@ -15,8 +15,7 @@ from sqlalchemy.engine.url import URL from packageship.libs.exception.ext import Error from packageship.libs.exception.ext import DbnameNoneException from packageship.libs.exception.ext import ContentNoneException -from packageship.libs.configutils.readconfig import ReadConfig -from packageship import system_config +from packageship.libs.conf import configuration class BaseHelper(): @@ -25,7 +24,6 @@ class BaseHelper(): """ def __init__(self): - self.readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) self.engine = None @@ -44,12 +42,11 @@ class MysqlHelper(BaseHelper): def __init__(self, user_name=None, password=None, host=None, # pylint: disable=unused-argument port=None, database=None, **kwargs): super(MysqlHelper, self).__init__() - self.user_name = user_name or self.readconfig.get_database( - 'user_name') - self.password = password or self.readconfig.get_database('password') - self.host = host or self.readconfig.get_database('host') - self.port = port or self.readconfig.get_database('port') - self.database = database or self.readconfig.get_database('database') + self.user_name = user_name or configuration.USER_NAME + self.password = password or configuration.PASSWORD + self.host = host or configuration.HOST + self.port = port or configuration.PORT + self.database = database or configuration.DATABASE self.connection_type = 'mysql+pymysql' def create_database_engine(self): @@ -101,10 +98,7 @@ class SqliteHlper(BaseHelper): Raises: """ - _database_folder_path = self.readconfig.get_system( - 'data_base_path') - if not _database_folder_path: - _database_folder_path = system_config.DATABASE_FOLDER_PATH + _database_folder_path = configuration.DATABASE_FOLDER_PATH try: if not os.path.exists(_database_folder_path): os.makedirs(_database_folder_path) @@ -144,6 +138,7 @@ class DBHelper(BaseHelper): """ # The base class inherited by the data model BASE = declarative_base() + ENGINE_CONTAINER = dict() def __init__(self, user_name=None, password=None, host=None, # pylint: disable=R0913 port=None, db_name=None, connection_type=None, **kwargs): @@ -157,17 +152,37 @@ class DBHelper(BaseHelper): 'sqlite': SqliteHlper } if connection_type is None: - connection_type = self.readconfig.get_database( - 'dbtype') or 'sqlite' + connection_type = configuration.DATABASE_ENGINE_TYPE + self._engine_pool = connection_type + '_' + db_name _database_engine = self._database_engine.get(connection_type) if _database_engine is None: - raise DisconnectionError('') - _engine = _database_engine(user_name=user_name, password=password, - host=host, port=port, database=db_name, **kwargs) - _engine.create_database_engine() - self.engine = _engine.engine + raise DisconnectionError( + 'Database engine connection failed' + 'Not currently supported %s database' % connection_type) + _engine = self.ENGINE_CONTAINER.get(self._engine_pool) + if _engine: + self.engine = _engine + else: + _engine = _database_engine(user_name=user_name, password=password, + host=host, port=port, database=db_name, **kwargs) + _engine.create_database_engine() + self.engine = _engine.engine + self.ENGINE_CONTAINER[self._engine_pool] = self.engine self.session = None + def create_engine(self): + """ + Create related database engine connections + """ + session = sessionmaker() + try: + session.configure(bind=self.engine) + except DisconnectionError: + self.ENGINE_CONTAINER.pop(self._engine_pool) + else: + self.session = session() + return self + def __enter__(self): """ Description: functional description:Create a context manager for the database connection @@ -179,13 +194,8 @@ class DBHelper(BaseHelper): """ - session = sessionmaker() - if not hasattr(self, 'engine'): - raise DisconnectionError('Abnormal database connection') - session.configure(bind=self.engine) - - self.session = session() - return self + database_engine = self.create_engine() + return database_engine def __exit__(self, exc_type, exc_val, exc_tb): """ @@ -200,6 +210,8 @@ class DBHelper(BaseHelper): Raises: """ + if isinstance(exc_type, (AttributeError)): + raise SQLAlchemyError(exc_val) self.session.close() @classmethod @@ -247,10 +259,8 @@ class DBHelper(BaseHelper): if entity is None: raise ContentNoneException( 'The added entity content cannot be empty') - try: self.session.add(entity) - except SQLAlchemyError as sql_error: self.session.rollback() if isinstance(sql_error, OperationalError): diff --git a/packageship/packageship/libs/log/__init__.py b/packageship/packageship/libs/log/__init__.py index 3decd458bb39dd8edcf986def0957fc4fc6853c0..33cd95f042ffda33c5415a8a05c5e29689c2c629 100644 --- a/packageship/packageship/libs/log/__init__.py +++ b/packageship/packageship/libs/log/__init__.py @@ -3,6 +3,6 @@ Common methods for logging """ from packageship.libs.log.loghelper import setup_log -from packageship.libs.log.loghelper import Log +from packageship.libs.log.loghelper import Log, LOGGER -__all__ = ['setup_log', 'Log'] +__all__ = ['setup_log', 'Log', 'LOGGER'] diff --git a/packageship/packageship/libs/log/loghelper.py b/packageship/packageship/libs/log/loghelper.py index 362175c123369b7afecb0ee44da19c5414ddcb9c..1d38b02f1b6f31d6efd8201e498b7fb414d21c08 100644 --- a/packageship/packageship/libs/log/loghelper.py +++ b/packageship/packageship/libs/log/loghelper.py @@ -7,38 +7,25 @@ import threading import pathlib import logging from concurrent_log_handler import ConcurrentRotatingFileHandler -from packageship import system_config -from packageship.libs.configutils.readconfig import ReadConfig - - -READCONFIG = ReadConfig(system_config.SYS_CONFIG_PATH) +from packageship.libs.conf import configuration def setup_log(config=None): """ Log logging in the context of flask """ + _level = configuration.LOG_LEVEL if config: - logging.basicConfig(level=config.LOG_LEVEL) - else: - _level = READCONFIG.get_config('LOG', 'log_level') - if _level is None: - _level = 'INFO' - logging.basicConfig(level=_level) - path = READCONFIG.get_config('LOG', 'log_path') - log_name = READCONFIG.get_config('LOG', 'log_name') - backup_count = READCONFIG.get_config('LOG', 'backup_count') + _level = config.LOG_LEVEL + logging.basicConfig(level=_level) + backup_count = configuration.BACKUP_COUNT if not backup_count or not isinstance(backup_count, int): backup_count = 10 - max_bytes = READCONFIG.get_config('LOG', 'max_bytes') + max_bytes = configuration.MAX_BYTES if not max_bytes or not isinstance(max_bytes, int): max_bytes = 314572800 - if not log_name: - log_name = 'log_info.log' - if not path: - path = os.path.join(system_config.LOG_FOLDER_PATH, log_name) - else: - path = os.path.join(path, log_name) + + path = os.path.join(configuration.LOG_PATH, configuration.LOG_NAME) if not os.path.exists(path): try: os.makedirs(os.path.split(path)[0]) @@ -49,7 +36,9 @@ def setup_log(config=None): path, maxBytes=max_bytes, backupCount=backup_count) formatter = logging.Formatter( - '%(levelname)s %(filename)s:%(lineno)d %(message)s') + '%(asctime)s-%(name)s-%(filename)s-[line:%(lineno)d]' + '-%(levelname)s-[ log details ]: %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S') file_log_handler.setFormatter(formatter) @@ -67,37 +56,27 @@ class Log(): self.__file_handler = None - log_name = READCONFIG.get_config('LOG', 'log_name') - if not log_name: - log_name = 'log_info.log' + self.__path = os.path.join( + configuration.LOG_PATH, configuration.LOG_NAME) if path: - self.__path = os.path.join(system_config.LOG_FOLDER_PATH, path) - else: - self.__path = READCONFIG.get_config('LOG', 'log_path') - if not self.__path: - self.__path = os.path.join( - system_config.LOG_FOLDER_PATH, log_name) - else: - self.__path = os.path.join(self.__path, log_name) + self.__path = path if not os.path.exists(self.__path): try: os.makedirs(os.path.split(self.__path)[0]) except FileExistsError: pathlib.Path(self.__path).touch() - self.__level = READCONFIG.get_config('LOG', 'log_level') - if self.__level is None: - self.__level = 'INFO' + + self.__level = configuration.LOG_LEVEL self.__logger = logging.getLogger(self.__name) self.__logger.setLevel(self.__level) - self.backup_count = READCONFIG.get_config('LOG', 'backup_count') or 10 - self.max_bytes = READCONFIG.get_config('LOG', 'max_bytes') or 314572800 - try: - self.backup_count = int(self.backup_count) - self.max_bytes = int(self.max_bytes) - except ValueError: + self.backup_count = configuration.BACKUP_COUNT + if not self.backup_count or not isinstance(self.backup_count, int): self.backup_count = 10 + self.max_bytes = configuration.MAX_BYTES + if not self.max_bytes or not isinstance(self.max_bytes, int): self.max_bytes = 314572800 + self.__init_handler() self.__set_handler() self.__set_formatter() @@ -148,3 +127,6 @@ class Log(): Get logs """ return self.__logger + + +LOGGER = Log(__name__) diff --git a/packageship/packageship/pkgship.py b/packageship/packageship/pkgship.py index f9408c8f8a7f07dc0f414c9fc826cee3b8459d7c..a36499119862c4b63aea16a9ead8de00f2b8cfc4 100644 --- a/packageship/packageship/pkgship.py +++ b/packageship/packageship/pkgship.py @@ -18,12 +18,9 @@ try: from requests.exceptions import HTTPError import prettytable from prettytable import PrettyTable - from packageship import system_config - from packageship.libs.log import Log + from packageship.libs.conf import configuration + from packageship.libs.log import LOGGER from packageship.libs.exception import Error - from packageship.libs.configutils.readconfig import ReadConfig - - LOGGER = Log(__name__) except ImportError as import_error: print("Error importing related dependencies," "please check if related dependencies are installed") @@ -38,9 +35,6 @@ DB_NAME = 0 def main(): """ Description: Command line tool entry, register related commands - Args: - - Returns: Raises: Error: An error occurred while executing the command @@ -50,7 +44,7 @@ def main(): packship_cmd.parser_args() except Error as error: LOGGER.logger.error(error) - print('command error') + print('Command execution error please try again') class BaseCommand(): @@ -67,7 +61,6 @@ class BaseCommand(): Description: Class instance initialization """ - self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) self.write_host = None self.read_host = None self.__http = 'http://' @@ -86,9 +79,9 @@ class BaseCommand(): Raises: """ - wirte_port = self._read_config.get_system('write_port') + wirte_port = configuration.WRITE_PORT - write_ip = self._read_config.get_system('write_ip_addr') + write_ip = configuration.WRITE_IP_ADDR if not all([write_ip, wirte_port]): raise Error( "The system does not configure the relevant port and ip correctly") @@ -104,9 +97,9 @@ class BaseCommand(): Raises: """ - read_port = self._read_config.get_system('query_port') + read_port = configuration.QUERY_PORT - read_ip = self._read_config.get_system('query_ip_addr') + read_ip = configuration.QUERY_IP_ADDR if all([read_ip, read_port]): _read_host = self.__http + read_ip + ":" + read_port @@ -117,8 +110,7 @@ class BaseCommand(): Set read domain name """ if remote: - _host = self._read_config.get_system('remote_host') - self.read_host = _host + self.read_host = configuration.REMOTE_HOST if self.read_host is None: raise Error( "The system does not configure the relevant port and ip correctly")