From abf93cb6417e45a26eef732aab8ee006b10ad66a Mon Sep 17 00:00:00 2001 From: zt <1450026690@qq.com> Date: Mon, 14 Sep 2020 16:06:32 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=A4=E4=B8=AA=E7=BB=84?= =?UTF-8?q?=E4=BB=B6=E5=90=8C=E6=97=B6=E6=9D=A5=E8=87=AA=E4=BA=8E=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E4=BA=8C=E8=BF=9B=E5=88=B6=E5=8C=85=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E5=86=97=E4=BD=99=E6=9F=A5=E8=AF=A2=E9=97=AE=E9=A2=98,?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9=E5=BA=94=E8=BE=85=E5=8A=A9=E5=87=BD?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/package/function/searchdb.py | 580 ++++++++++++------ 1 file changed, 397 insertions(+), 183 deletions(-) diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index 400d422f..016f6d3b 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -15,10 +15,10 @@ from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log -from packageship.application.models.package import BinPack,SrcPack +from packageship.application.models.package import BinPack, SrcPack from packageship.libs.exception import ContentNoneException, Error from packageship.system_config import DATABASE_FILE_INFO -from .constants import ResponseCode +from packageship.application.apps.package.function.constants import ResponseCode LOGGER = Log(__name__) @@ -50,6 +50,7 @@ class SearchDB(): except DisconnectionError as connection_error: current_app.logger.error(connection_error) + # pylint: disable=R0914 def get_install_depend(self, binary_list): """ Description: get a package install depend from database: @@ -63,89 +64,158 @@ class SearchDB(): Raises: """ result_list = [] - get_list = [] provides_not_found = dict() + if not self.db_object_dict: - LOGGER.logger.warning("Unable to connect to the database, \ - check the database configuration") - return result_list + LOGGER.logger.warning("Unable to connect to the database," + "check the database configuration") + return result_list, set() + if None in binary_list: binary_list.remove(None) search_set = set(binary_list) + if not search_set: - LOGGER.logger.warning( - "The input is None, please check the input value.") - return result_list - return_tuple = namedtuple('return_tuple', - 'depend_name depend_version depend_src_name \ - search_name search_src_name search_version') + LOGGER.logger.warning("The input is None, please check the input value.") + return result_list, set() + + return_tuple = namedtuple('return_tuple', [ + 'depend_name', + 'depend_version', + 'depend_src_name', + 'search_name', + 'search_src_name', + 'search_version' + ]) + for db_name, data_base in self.db_object_dict.items(): try: - name_in = literal_column('name').in_(search_set) - sql_com = text(""" - SELECT DISTINCT - bin_pack.NAME AS depend_name, - bin_pack.version AS depend_version, - s2.name AS depend_src_name, - bin_requires.NAME AS req_name, - bin.NAME AS search_name, - s1.name AS search_src_name, - bin.version AS search_version - FROM - ( SELECT pkgKey, NAME, version, rpm_sourcerpm FROM bin_pack WHERE {} ) bin - LEFT JOIN src_pack s1 ON bin.rpm_sourcerpm = s1.src_name - LEFT JOIN bin_requires ON bin.pkgKey = bin_requires.pkgKey - LEFT JOIN bin_provides ON bin_provides.name = bin_requires.name - LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey - LEFT JOIN src_pack s2 ON bin_pack.rpm_sourcerpm = s2.src_name; - """.format(name_in)) - install_set = data_base.session. \ - execute(sql_com, {'name_{}'.format(i): v - for i, v in enumerate(search_set, 1)}).fetchall() - if install_set: - # find search_name in db_name - # depend_name's db_name will be found in next loop - for result in install_set: - get_list.append(result.search_name) - if not result.depend_name and result.req_name: - if result.req_name in provides_not_found: - provides_not_found[result.req_name].append( - [result.search_name, result.search_src_name, result.search_version, db_name]) - else: - provides_not_found[result.req_name] = [ - [result.search_name, result.search_src_name, result.search_version, db_name]] - else: - obj = return_tuple( - result.depend_name, - result.depend_src_name, - result.depend_version, - result.search_name, - result.search_src_name, - result.search_version, - ) - result_list.append((obj, db_name)) - get_set = set(get_list) - get_list.clear() - search_set.symmetric_difference_update(get_set) - if not search_set: - install_result = self._get_install_pro_in_other_database( - provides_not_found) - result_list.extend(install_result) - return result_list, set(provides_not_found.keys()) - else: + req_set = self._get_install_search_info(search_set, data_base) + + if not req_set: continue + + depend_set, req_pk_dict, not_fd_com = self._filter_provide_same_comp_pkg(req_set, + data_base) + + res_list, get_list = self._comb_install_list(depend_set, + req_pk_dict, + not_fd_com, + return_tuple, + db_name, + provides_not_found, + req_set) + + result_list += res_list + + search_set.symmetric_difference_update(set(get_list)) + + if not search_set: + result_list.extend( + self._get_install_pro_in_other_database(provides_not_found, + db_name) + ) + return result_list, set(provides_not_found.keys()) + except AttributeError as error_msg: LOGGER.logger.error(error_msg) except SQLAlchemyError as error_msg: LOGGER.logger.error(error_msg) - install_result = self._get_install_pro_in_other_database( - provides_not_found) - result_list.extend(install_result) + + result_list.extend( + self._get_install_pro_in_other_database(provides_not_found) + ) + for binary_name in search_set: result_list.append((return_tuple(None, None, None, binary_name, None, None), 'NOT FOUND')) return result_list, set(provides_not_found.keys()) + @staticmethod + def _get_install_search_info(search_set, data_base): + req_set = [] + try: + sql_com = text(""" + SELECT DISTINCT + bin_requires.NAME AS req_name, + bin.NAME AS search_name, + s1.name as search_src_name, + bin.version AS search_version + FROM + ( SELECT pkgKey, NAME, version, rpm_sourcerpm FROM bin_pack WHERE {} ) bin + LEFT JOIN src_pack s1 ON bin.rpm_sourcerpm = s1.src_name + LEFT JOIN bin_requires ON bin.pkgKey = bin_requires.pkgKey; + """.format(literal_column('name').in_(search_set))) + req_set = data_base.session. \ + execute(sql_com, {'name_{}'.format(i): v + for i, v in enumerate(search_set, 1)}).fetchall() + except AttributeError as error_msg: + LOGGER.logger.error(error_msg) + except SQLAlchemyError as error_msg: + LOGGER.logger.error(error_msg) + return req_set + + # pylint: disable=R0913 + @staticmethod + def _comb_install_list(depend_set, + req_pk_dict, + not_fd_com, + return_tuple, + db_name, + provides_not_found, + req_set): + get_list = [] + ret_list = [] + depend_info_tuple = namedtuple('depend_info', [ + 'depend_name', + 'depend_version', + 'depend_src_name' + ]) + depend_info_dict = { + info.pk: depend_info_tuple(info.depend_name, + info.depend_version, + info.depend_src_name) + for info in depend_set + } + + for req_name, search_name, search_src_name, search_version in req_set: + get_list.append(search_name) + + if not req_name: + obj = return_tuple( + None, + None, + None, + search_name, + search_src_name, + search_version, + ) + ret_list.append((obj, db_name)) + + elif req_name in req_pk_dict: + bin_pk = req_pk_dict[req_name] + depend_info_t = depend_info_dict[bin_pk] + obj = return_tuple( + depend_info_t.depend_name, + depend_info_t.depend_version, + depend_info_t.depend_src_name, + search_name, + search_src_name, + search_version, + ) + ret_list.append((obj, db_name)) + + else: + if req_name in not_fd_com: + if req_name not in provides_not_found: + provides_not_found[req_name] = [[search_name, search_src_name, + search_version, db_name]] + else: + provides_not_found[req_name].append([search_name, search_src_name, + search_version, db_name]) + + return ret_list, get_list + def get_src_name(self, binary_name): """ Description: get a package source name from database: @@ -173,7 +243,9 @@ class SearchDB(): AND bin_pack.name = :binary_name; """ try: - bin_obj = data_base.session.execute(text(sql_str), {"binary_name": binary_name}).fetchone() + bin_obj = data_base.session.execute(text(sql_str), + {"binary_name": binary_name} + ).fetchone() source_name = bin_obj.source_name source_version = bin_obj.source_version if source_name is not None: @@ -201,8 +273,7 @@ class SearchDB(): """ if not self.db_object_dict: return ResponseCode.DIS_CONNECTION_DB, None - search_set = set([ - source_name for source_name in source_name_list if source_name]) + search_set = {source_name for source_name in source_name_list if source_name} result_list = [] get_list = [] if not search_set: @@ -211,14 +282,15 @@ class SearchDB(): try: name_in = literal_column('name').in_(search_set) sql_com = text(''' - SELECT - bin_pack.name AS subpack_name, - bin_pack.version AS sub_pack_version, - src.name AS search_name, - src.version AS search_version - FROM - (SELECT name,version,src_name FROM src_pack WHERE {}) src - LEFT JOIN bin_pack on src.src_name = bin_pack.rpm_sourcerpm'''.format(name_in)) + SELECT + bin_pack.name AS subpack_name, + bin_pack.version AS sub_pack_version, + src.name AS search_name, + src.version AS search_version + FROM + (SELECT name,version,src_name FROM src_pack WHERE {}) src + LEFT JOIN bin_pack on src.src_name = bin_pack.rpm_sourcerpm + '''.format(name_in)) subpack_tuple = data_base.session. \ execute(sql_com, {'name_{}'.format(i): v for i, v in enumerate(search_set, 1)}).fetchall() @@ -280,40 +352,28 @@ class SearchDB(): search_list.clear() try: sql_string = text(""" - SELECT DISTINCT - s1.name AS source_name, - t1.NAME AS bin_name, - t1.version, - t2.NAME AS req_name - FROM - src_pack s1, - bin_pack t1, - bin_provides t2 - WHERE - t2.{} - AND t1.pkgKey = t2.pkgKey - AND t1.rpm_sourcerpm = s1.src_name; + SELECT DISTINCT + s1.name AS source_name, + t1.NAME AS bin_name, + t1.version, + t2.NAME AS req_name + FROM + src_pack s1, + bin_pack t1, + bin_provides t2 + WHERE + t2.{} + AND t1.pkgKey = t2.pkgKey + AND t1.rpm_sourcerpm = s1.src_name; """.format(literal_column('name').in_(search_set))) bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v for i, v in enumerate(search_set, 1)}).fetchall() if bin_set: - for result in bin_set: - if result.req_name not in not_found_binary: - LOGGER.logger.warning( - result.req_name + " contains in two rpm packages!!!") - else: - for source_info in not_found_binary[result.req_name]: - obj = return_tuple( - source_info[0], - result.source_name, - result.bin_name, - result.version, - db_name, - source_info[1] - ) - result_list.append(obj) - del not_found_binary[result.req_name] + result_list += self._comb_build_info(bin_set, + not_found_binary, + return_tuple, + db_name) if not not_found_binary: return result_list except AttributeError as attr_err: @@ -335,56 +395,87 @@ class SearchDB(): result_list.append(obj) return result_list - def _get_install_pro_in_other_database(self, not_found_binary): + @staticmethod + def _comb_build_info(bin_set, + not_found_binary, + return_tuple, + db_name): + ret_list = [] + for result in bin_set: + if result.req_name not in not_found_binary: + LOGGER.logger.warning( + "%s contains in two rpm packages!!!", result.req_name + ) + else: + for source_info in not_found_binary[result.req_name]: + obj = return_tuple( + source_info[0], + result.source_name, + result.bin_name, + result.version, + db_name, + source_info[1] + ) + ret_list.append(obj) + del not_found_binary[result.req_name] + return ret_list + + def _get_install_pro_in_other_database(self, not_found_binary, _db_name=None): if not not_found_binary: return [] - return_tuple = namedtuple('return_tuple', - 'depend_name depend_version depend_src_name \ - search_name search_src_name search_version') + + return_tuple = namedtuple('return_tuple', [ + 'depend_name', + 'depend_version', + 'depend_src_name', + 'search_name', + 'search_src_name', + 'search_version' + ]) search_list = [] result_list = [] + for db_name, data_base in self.db_object_dict.items(): + if db_name == _db_name: + continue + for key, values in not_found_binary.items(): search_list.append(key) search_set = set(search_list) search_list.clear() - sql_string = text(""" - SELECT DISTINCT - s1.name AS source_name, - t1.NAME AS bin_name, - t1.version, - t2.NAME AS req_name - FROM - src_pack s1, - bin_pack t1, - bin_provides t2 - WHERE - t2.{} - AND t1.pkgKey = t2.pkgKey - AND t1.rpm_sourcerpm = s1.src_name; - """.format(literal_column('name').in_(search_set))) - bin_set = data_base.session. \ - execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(search_set, 1)}).fetchall() - if bin_set: - for result in bin_set: - if result.req_name not in not_found_binary: - LOGGER.logger.warning( - result.req_name + " contains in two rpm packages!!!") - else: - for binary_info in not_found_binary[result.req_name]: - obj = return_tuple( - result.bin_name, - result.version, - result.source_name, - binary_info[0], - binary_info[1], - binary_info[2] - ) - result_list.append((obj, binary_info[3])) - del not_found_binary[result.req_name] - if not not_found_binary: - return result_list + + in_tuple = namedtuple("in_tuple", 'req_name') + in_tuple_list = [] + for k, _ in not_found_binary.items(): + in_tuple_list.append( + in_tuple(k) + ) + + depend_set, req_pk_dict, _ = self._filter_provide_same_comp_pkg( + in_tuple_list, + data_base + ) + + depend_info_tuple = namedtuple('depend_info', [ + 'depend_name', + 'depend_version', + 'depend_src_name' + ]) + depend_info_dict = { + info.pk: depend_info_tuple(info.depend_name, + info.depend_version, + info.depend_src_name) + for info in depend_set + } + result_list += self._comb_install_info(search_set, + req_pk_dict, + depend_info_dict, + not_found_binary, + return_tuple, + db_name) + if not not_found_binary: + return result_list + if not_found_binary: for key, values in not_found_binary.items(): for info in values: @@ -399,6 +490,31 @@ class SearchDB(): result_list.append((obj, info[3])) return result_list + @staticmethod + def _comb_install_info(search_set, + req_pk_dict, + depend_info_dict, + not_found_binary, + return_tuple, + db_name): + ret_list = [] + for req_name in search_set: + if req_name in req_pk_dict: + pk_ = req_pk_dict[req_name] + if pk_ in depend_info_dict: + for binary_info in not_found_binary[req_name]: + obj = return_tuple( + depend_info_dict[pk_].depend_name, + depend_info_dict[pk_].depend_version, + depend_info_dict[pk_].depend_src_name, + binary_info[0], + binary_info[1], + binary_info[2] + ) + ret_list.append((obj, db_name)) + del not_found_binary[req_name] + return ret_list + def get_build_depend(self, source_name_li): """ Description: get a package build depend from database @@ -425,7 +541,7 @@ class SearchDB(): s_name_set = set(source_name_li) if not s_name_set: - return ResponseCode.PARAM_ERROR, set() + return ResponseCode.PARAM_ERROR, list(), set() provides_not_found = dict() build_list = [] @@ -467,48 +583,55 @@ class SearchDB(): if not build_set: continue - # When processing source package without compilation dependency - get_list = [] - for result in build_set: - get_list.append(result.search_name) - if not result.bin_name and result.req_name: - if result.req_name in provides_not_found: - provides_not_found[result.req_name].append( - [result.search_name, result.search_version, db_name] - ) - else: - provides_not_found[result.req_name] = [ - [result.search_name, result.search_version, db_name] - ] - else: - obj = return_tuple( - result.search_name, - result.source_name, - result.bin_name, - result.version, - db_name, - result.search_version - ) - build_list.append(obj) - - get_set = set(get_list) - get_list.clear() - s_name_set.symmetric_difference_update(get_set) + ret_list, get_list = self._comb_build_list(build_set, + provides_not_found, + db_name, + return_tuple) + build_list += ret_list + s_name_set.symmetric_difference_update(set(get_list)) if not s_name_set: - build_result = self._get_binary_in_other_database( - provides_not_found) - build_list.extend(build_result) + build_list.extend(self._get_binary_in_other_database( + provides_not_found)) return ResponseCode.SUCCESS, build_list, set(provides_not_found.keys()) if s_name_set: - build_result = self._get_binary_in_other_database( - provides_not_found) - build_list.extend(build_result) + build_list.extend(self._get_binary_in_other_database( + provides_not_found)) for source in s_name_set: LOGGER.logger.warning( - "CANNOT FOUND THE source " + source + " in all database") + "CANNOT FOUND THE source %s in all database", source) return ResponseCode.SUCCESS, build_list, set(provides_not_found.keys()) + @staticmethod + def _comb_build_list(build_set, + provides_not_found, + db_name, + return_tuple): + ret_list = [] + get_list = [] + for result in build_set: + get_list.append(result.search_name) + if not result.bin_name and result.req_name: + if result.req_name in provides_not_found: + provides_not_found[result.req_name].append( + [result.search_name, result.search_version, db_name] + ) + else: + provides_not_found[result.req_name] = [ + [result.search_name, result.search_version, db_name] + ] + else: + obj = return_tuple( + result.search_name, + result.source_name, + result.bin_name, + result.version, + db_name, + result.search_version + ) + ret_list.append(obj) + return ret_list, get_list + def binary_search_database_for_first_time(self, binary_name): """ Args: @@ -553,6 +676,97 @@ class SearchDB(): return None, None + def _filter_provide_same_comp_pkg(self, comps_obj, data_base): + """ + Description: Filter redundant queries + when the same binary package is provided to multiple components + Args: + comps_obj: List of sqlalchemy objects with component names. + data_base: The database currently being queried + Returns: + depend_set: List of related dependent sqlalchemy objects + req_pk_dict: Mapping dictionary of component name and pkgKey + not_fd_req: Components not found + Raises: + AttributeError: The object does not have this property + SQLAlchemyError: sqlalchemy error + """ + depend_set = [] + req_pk_dict = {} + not_fd_req = set() + try: + req_names = {req_info.req_name + for req_info in comps_obj + if req_info.req_name is not None} + req_name_in = literal_column('name').in_(req_names) + sql_com_pro = text(""" + SELECT DISTINCT + NAME as req_name, + pkgKey + FROM + ( SELECT name, pkgKey FROM bin_provides UNION ALL SELECT name, pkgKey FROM bin_files ) + WHERE + {}; + """.format(req_name_in)) + + pkg_key_set = data_base.session.execute( + sql_com_pro, { + 'name_{}'.format(i): v + for i, v in enumerate(req_names, 1) + } + ).fetchall() + + req_pk_dict = {} + pk_val = [pk for _, pk in pkg_key_set] + for req_name, pk_ in pkg_key_set: + if not req_name: + continue + if req_name not in req_pk_dict: + req_pk_dict[req_name] = pk_ + else: + if pk_val.count(pk_) > 1: + req_pk_dict[req_name] = pk_ + not_fd_req = req_names - set(req_pk_dict.keys()) + depend_set = self._get_depend_info(req_pk_dict, data_base) + + except SQLAlchemyError as sql_err: + LOGGER.logger.error(sql_err) + except AttributeError as error_msg: + LOGGER.logger.error(error_msg) + + return depend_set, req_pk_dict, not_fd_req + + @staticmethod + def _get_depend_info(req_pk_dict, data_base): + depend_set = [] + try: + bin_src_pkg_key = req_pk_dict.values() + pk_in = literal_column('pkgKey').in_(bin_src_pkg_key) + sql_bin_src = text(""" + SELECT DISTINCT + bin.pkgKey as pk, + bin.name AS depend_name, + bin.version AS depend_version, + src_pack.name AS depend_src_name + FROM + ( SELECT name, pkgKey,version, rpm_sourcerpm FROM bin_pack WHERE {} ) bin + LEFT JOIN src_pack ON src_pack.src_name = bin.rpm_sourcerpm; + """.format(pk_in)) + + depend_set = data_base.session.execute( + sql_bin_src, { + 'pkgKey_{}'.format(i): v + for i, v in enumerate(bin_src_pkg_key, 1) + } + ).fetchall() + + except SQLAlchemyError as sql_err: + LOGGER.logger.error(sql_err) + except AttributeError as error_msg: + LOGGER.logger.error(error_msg) + + return depend_set + def db_priority(): """ -- Gitee