From bcdd01337361ae7bd1defbd8b31beb3c7c9f9b61 Mon Sep 17 00:00:00 2001 From: gongzhengtang Date: Tue, 27 Oct 2020 13:58:38 +0800 Subject: [PATCH] =?UTF-8?q?pkgship=E5=88=9D=E5=A7=8B=E5=8C=96=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0filelist=E7=9A=84=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E6=97=B6=E6=94=AF=E6=8C=81=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E7=9A=84sqlite=E6=96=87=E4=BB=B6=E5=92=8C=E8=BF=9C=E7=A8=8B?= =?UTF-8?q?=E7=9A=84sqlite=E5=8E=8B=E7=BC=A9=E5=8C=85=E4=B8=A4=E7=A7=8D?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../application/initsystem/data_import.py | 276 +++++++++++++++++- .../packageship/application/models/package.py | 12 + packageship/packageship/libs/conf/__init__.py | 7 + .../packageship/libs/conf/global_config.py | 3 + packageship/packageship/package.ini | 3 + 5 files changed, 285 insertions(+), 16 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 99930cbd..7db82196 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -5,7 +5,17 @@ Description: Initialization of data import Class: InitDataBase,MysqlDatabaseOperations,SqliteDatabaseOperations """ import os +import re +import bz2 +import gzip +import tarfile +import zipfile +import shutil +import requests import yaml +from retrying import retry +from requests.exceptions import HTTPError +from requests.exceptions import RequestException from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError, InternalError from packageship.libs.dbutils.sqlalchemy_helper import DBHelper @@ -23,6 +33,7 @@ from packageship.application.models.package import SrcRequires from packageship.application.models.package import BinProvides from packageship.application.models.package import BinFiles from packageship.application.models.package import Packages +from packageship.application.models.package import FileList class InitDataBase(): @@ -57,7 +68,7 @@ class InitDataBase(): } self.database_name = None self._tables = ['src_pack', 'bin_pack', - 'bin_requires', 'src_requires', 'bin_provides', 'bin_files'] + 'bin_requires', 'src_requires', 'bin_provides', 'bin_files', 'filelist'] # Create life cycle related databases and tables if not self.create_database(db_name='lifecycle', tables=['packages_issue', @@ -106,9 +117,10 @@ class InitDataBase(): for config_item in init_database_config: if not isinstance(config_item, dict): raise ConfigurationException( - "The format of the initial database" - "configuration file is incorrect, and the value in a single node should" - "be presented in the form of key - val pairs:{}".format(self.config_file_path)) + "The format of the initial database " + "configuration file is incorrect, and the value " + "in a single node should be presented in the form " + "of key - val pairs:{}".format(self.config_file_path)) return init_database_config def init_data(self): @@ -173,6 +185,36 @@ class InitDataBase(): db_name=db_name, tables=tables, storage=storage).create_database(self) return _create_table_result + def _get_sqlite_file(self, database_config, db_name): + """ + Gets the path to a local or remote SQLite file + Args: + database_config:Initializes the contents of the configuration file + return: + A tuple that contains the binary, source package + and binary collection path addresses + """ + http_regex = r"^(ht|f)tp(s?)\:\/\/[0-9a-zA-Z]([-.\w]*[0-9a-zA-Z])"\ + r"*(:(0-9)*)*(\/?)([a-zA-Z0-9\-\.\?\,\'\/\\\+&%$#_]*)?" + src_db_file = database_config.get('src_db_file') + bin_db_file = database_config.get('bin_db_file') + if re.match(http_regex, src_db_file): + src_db_file = DownloadFile( + src_db_file, 'primary', db_name + '_src').get_remote_file() + if re.match(http_regex, bin_db_file): + bin_db_file = DownloadFile( + bin_db_file, 'primary', db_name + '_bin').get_remote_file() + + if src_db_file is None or bin_db_file is None: + raise ContentNoneException( + "The path to the sqlite file in the database initialization" + "configuration is incorrect ") + file_list = database_config.get('filelist_file') + if file_list and re.match(http_regex, file_list): + file_list = DownloadFile( + file_list, 'filelists', db_name + '_files').get_remote_file() + return (src_db_file, bin_db_file, file_list) + def _init_data(self, database_config): """ data initialization operation @@ -197,21 +239,16 @@ class InitDataBase(): raise SQLAlchemyError( 'Failed to create the specified database and table：%s' % database_config['dbname']) - # 2. get the data of binary packages and source packages - src_db_file = database_config.get('src_db_file') - bin_db_file = database_config.get('bin_db_file') - - if src_db_file is None or bin_db_file is None: - raise ContentNoneException( - "The path to the sqlite file in the database initialization" - "configuration is incorrect ") + # 2. get the data of binary packages and source packages + src_db_file, bin_db_file, file_list = self._get_sqlite_file( + database_config, _db_name) if not os.path.exists(src_db_file) or not os.path.exists(bin_db_file): raise FileNotFoundError( "sqlite file {src} or {bin} does not exist, please" "check and try again".format(src=src_db_file, bin=bin_db_file)) # 3. Obtain temporary source package files and binary package files if self.__save_data(database_config, - self.database_name): + self.database_name, src_db_file, bin_db_file, file_list): # Update the configuration file of the database database_content = { 'database_name': _db_name, @@ -224,6 +261,9 @@ class InitDataBase(): LOGGER.logger.error(error_msg) # Delete the specified database self.__del_database(_db_name) + finally: + # Delete the downloaded temporary files + DownloadFile.del_temporary_file() def __del_database(self, db_name): try: @@ -271,7 +311,7 @@ class InitDataBase(): LOGGER.logger.error(sql_error) return None - def __save_data(self, database_config, db_name): + def __save_data(self, database_config, db_name, src_db_file, bin_db_file, file_list): """ integration of multiple data files @@ -283,8 +323,6 @@ class InitDataBase(): Raises: """ - src_db_file = database_config.get('src_db_file') - bin_db_file = database_config.get('bin_db_file') table_name = database_config.get('dbname') lifecycle_status_val = database_config.get('lifecycle') try: @@ -304,6 +342,8 @@ class InitDataBase(): self._save_bin_requires(db_name) self._save_bin_provides(db_name) self._save_bin_files(db_name) + if file_list: + self._save_file_list(file_list, db_name) except (SQLAlchemyError, ContentNoneException) as sql_error: LOGGER.logger.error(sql_error) self.__del_database(db_name) @@ -311,6 +351,31 @@ class InitDataBase(): else: return True + def _save_file_list(self, file_list, db_name): + """ + Holds a collection of binary file paths + Args: + file_list:The file path + db_name:The name of the database saved + """ + if not os.path.exists(file_list): + raise FileNotFoundError( + "sqlite file {filelist} does not exist, please" + "check and try again".format(filelist=file_list)) + # Save filelist package + file_list_datas = None + with DBHelper(db_name=file_list, db_type='sqlite:///', complete_route_db=True)\ + as database: + self._database = database + self.sql = " select * from filelist " + file_list_datas = self.__get_data() + if not file_list_datas: + raise ContentNoneException( + "{db_name}:The binary path provided has no relevant data" + " in the SQLite database ".format(db_name=db_name)) + with DBHelper(db_name=db_name) as database: + database.batch_add(file_list_datas, FileList) + def _save_src_packages(self, db_name, table_name, lifecycle_status_val): """ Save the source package data @@ -798,3 +863,182 @@ class SqliteDatabaseOperations(): return False else: return True + + +class Unpack(): + """ + Decompression of a file is related to the operation + """ + + def __init__(self, file_path, save_file): + self.file_path = file_path + self.save_file = save_file + + @classmethod + def dispatch(cls, extend, *args, **kwargs): + """ + Specific decompression methods are adopted for different compression packages + Args: + extend:The format of the compressed package + """ + self = cls(*args, **kwargs) + meth = getattr(self, extend[1:].lower(), None) + if meth is None: + raise Error( + "Unzipping files in the current format is not supported：%s" % extend) + meth() + + def bz2(self): + """ + Unzip the bZ2 form of the compression package + """ + with open(self.save_file, 'wb') as file, bz2.BZ2File(self.file_path, 'rb') as bz_file: + for data in iter(lambda: bz_file.read(100 * 1024), b''): + file.write(data) + + def gz(self): + """ + Unzip the compressed package in GZIP format + """ + with open(self.save_file, 'wb') as file, gzip.GzipFile(self.file_path) as gzip_file: + for data in iter(lambda: gzip_file.read(100 * 1024), b''): + file.write(data) + + def tar(self): + """ + Unzip the tar package + """ + with open(self.save_file, 'wb') as file, tarfile.open(self.file_path) as tar_file: + file_names = tar_file.getnames() + if len(file_names) != 1: + raise IOError( + "Too many files in the zip file, do not" + " conform to the form of a single file：%s" % self.file_path) + _file = tar_file.extractfile(file_names[0]) + for data in iter(lambda: _file.read(100 * 1024), b''): + file.write(data) + + def zip(self): + """ + Unzip the zip package + """ + with open(self.save_file, 'wb') as file, zipfile.open(self.file_path) as zip_file: + file_names = zip_file.namelist() + if len(file_names) != 1: + raise IOError("Too many files in the zip file, do not" + " conform to the form of a single file：%s" % self.file_path) + file.write(zip_file.read()) + + +class DownloadFile(): + """ + Download the file for the remote address + """ + + def __init__(self, url, file_type, file_name): + if not url: + raise Error("The path to download the file is empty") + if not url.endswith('/'): + url += '/' + self.url = url + 'repodata/' + self._temporary_directory = configuration.TEMPORARY_DIRECTORY + self._file_regex = r'href=\"([^\"]*%s.sqlite.{3,4})\".?' % file_type + self._file_remote = None + self.file_name = file_name + + @retry(stop_max_attempt_number=3, stop_max_delay=1000) + def __download_file(self, url): + """ + Download the file or web page in the specified path + Args: + url:Remote online address + Return: + Successful download of binary stream + """ + try: + response = requests.get(url) + except RequestException as error: + raise RequestException(error) + if response.status_code != 200: + _msg = "There is an exception with the remote service [%s]，" \ + "Please try again later.The HTTP error code is：%s" % (url, str( + response.status_code)) + raise HTTPError(_msg) + return response + + @staticmethod + def del_temporary_file(): + """ + Delete temporary files or directories + """ + try: + del_list = os.listdir(configuration.TEMPORARY_DIRECTORY) + for file in del_list: + file_path = os.path.join( + configuration.TEMPORARY_DIRECTORY, file) + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except IOError as error: + LOGGER.error(error) + + def _matching_remote_file(self): + """ + Matches the filename that the remote needs to download + """ + try: + response = self.__download_file(self.url) + result = re.search( + self._file_regex, response.content.decode('utf-8')) + if result: + self._file_remote = self.url + result.group(1) + except (RequestException, HTTPError)as error: + LOGGER.error(error) + + def __unzip_file(self, file_path, file_name): + """ + Unzip a compressed package file in a specific format + Args: + file_path: The path of the compression package + """ + if not os.path.exists(file_path): + raise Error("The unzip file does not exist：%s" % file_path) + sqlite_file_path = os.path.join( + os.path.dirname(file_path), file_name + ".sqlite") + try: + Unpack.dispatch(os.path.splitext(file_path)[-1], file_path=file_path, + save_file=sqlite_file_path) + return sqlite_file_path + except IOError as error: + raise Error( + "An error occurred during file decompression：%s" % error) + finally: + os.remove(file_path) + + def get_remote_file(self): + """ + Gets the remote zip file + """ + self._matching_remote_file() + if not self._file_remote: + raise Error("There is no SQLite file in the specified remote" + " service %s that can be used for initialization" % self.url) + try: + response = self.__download_file(self._file_remote) + except (RequestException, HTTPError) as error: + LOGGER.error(error) + raise Error("There was an error downloading file: %s, " + "Please try again later." % self._file_remote) + try: + if not os.path.exists(self._temporary_directory): + os.makedirs(self._temporary_directory) + _file_path = os.path.join( + self._temporary_directory, self._file_remote.split('/')[-1]) + with open(_file_path, "wb") as file: + file.write(response.content) + except IOError as error: + LOGGER.error(error) + raise Error("There was an error downloading file: %s, " + "Please try again later." % self._file_remote) + return self.__unzip_file(_file_path, self.file_name) diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 2d9d2047..133a1fbf 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -212,3 +212,15 @@ class DatabaseInfo(DBHelper.BASE): id = Column(Integer, primary_key=True) name = Column(String(200), nullable=True) priority = Column(Integer, nullable=True) + + +class FileList(DBHelper.BASE): + """ + A collection of binary file paths + """ + __tablename__ = 'filelist' + id = Column(Integer, primary_key=True) + pkgKey = Column(Integer) + dirname = Column(String(500), nullable=True) + filenames = Column(String(500)) + filetypes = Column(String(200), nullable=True) diff --git a/packageship/packageship/libs/conf/__init__.py b/packageship/packageship/libs/conf/__init__.py index fbf34b13..b461968a 100755 --- a/packageship/packageship/libs/conf/__init__.py +++ b/packageship/packageship/libs/conf/__init__.py @@ -88,6 +88,13 @@ class PreloadingSettings(): """ return self._setting_container is not None + def reload(self): + """ + Add the reload mechanism + """ + self._setting_container = None + self._preloading() + class Configs(): """ diff --git a/packageship/packageship/libs/conf/global_config.py b/packageship/packageship/libs/conf/global_config.py index 0b433a4f..524abaed 100755 --- a/packageship/packageship/libs/conf/global_config.py +++ b/packageship/packageship/libs/conf/global_config.py @@ -108,3 +108,6 @@ REDIS_MAX_CONNECTIONS = 10 # Maximum queue length QUEUE_MAXSIZE = 1000 + +# A temporary directory for files downloaded from the network that are cleaned periodically +TEMPORARY_DIRECTORY = '/var/run/pkgship' diff --git a/packageship/packageship/package.ini b/packageship/packageship/package.ini index cc83e4e4..c859a2ce 100644 --- a/packageship/packageship/package.ini +++ b/packageship/packageship/package.ini @@ -26,6 +26,9 @@ query_ip_addr=127.0.0.1 ; call the remote service to complete the data request remote_host=https://api.openeuler.org/pkgmanage +; A temporary directory for files downloaded from the network that are cleaned periodically +temporary_directory=/var/run/pkgship + [LOG] ; Custom log storage path -- Gitee