diff --git a/packageship/packageship/application/apps/package/function/build_depend.py b/packageship/packageship/application/apps/package/function/build_depend.py index 672cbe60ea27a9e98a8936cac6d58f04d78dd66b..b65464b954fbf7c4a75f81cc199ea930e6115f2c 100644 --- a/packageship/packageship/application/apps/package/function/build_depend.py +++ b/packageship/packageship/application/apps/package/function/build_depend.py @@ -80,9 +80,9 @@ class BuildDepend(): res_status, build_list = self.search_db.get_build_depend(pkg_list) if not build_list: - return res_status if res_status == \ - ResponseCode.DIS_CONNECTION_DB else \ + return res_status if res_status == ResponseCode.DIS_CONNECTION_DB else \ ResponseCode.PACK_NAME_NOT_FOUND + # create root node and get next search list search_list = self._create_node_and_get_search_list(build_list, pkg_list) @@ -153,7 +153,7 @@ class BuildDepend(): self.result_dict[obj.bin_name] = [ obj.source_name, obj.version, - obj.db_name, + self.search_db.binary_search_database_for_first_time(obj.bin_name), [ [obj.search_name, 'build'] ] @@ -191,19 +191,21 @@ class BuildDepend(): return # generate data content + search_name_set = set() for obj in bin_info_lis: + search_name_set.add(obj.search_name) + if obj.search_name not in self.source_dict: + self.source_dict[obj.search_name] = [obj.db_name, obj.search_version] + if not obj.bin_name: continue - # for first loop, init the source_dict - if not self.source_dict: - for src_name in pkg_name_li: - self.source_dict[src_name] = [obj.db_name, obj.search_version] + if obj.bin_name not in self.result_dict: self.result_dict[obj.bin_name] = [ obj.source_name if obj.source_name else None, obj.version if obj.version else None, - obj.db_name if obj.db_name else "NOT_FOUND", + self.search_db.binary_search_database_for_first_time(obj.bin_name), [ [obj.search_name, "build"] ] @@ -216,11 +218,14 @@ class BuildDepend(): if obj.source_name and \ obj.source_name not in self.source_dict and \ - obj.source_name not in self.history_dicts: - self.source_dict[obj.source_name] = [obj.db_name, - obj.version] + obj.source_name not in self.history_dicts: next_src_set.add(obj.source_name) + not_found_pkg = set(pkg_name_li) - search_name_set + for pkg_name in not_found_pkg: + if pkg_name not in self.source_dict: + self.source_dict[pkg_name] = ['NOT FOUND', 'NOT FOUND'] + not_found_pkg.clear() self.self_build(next_src_set) return diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index bba2994b5dea437de38a69d2bf9d51c24c649527..62c4a86b01691a6a8f8625de185a0c489fa2880c 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -11,6 +11,7 @@ from flask import current_app from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError, DisconnectionError from sqlalchemy.sql import literal_column +from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log @@ -30,6 +31,7 @@ class SearchDB(): db_object_dict:A dictionary for storing database connection objects changeLog: """ + def __new__(cls, *args, **kwargs): # pylint: disable=w0613 if not hasattr(cls, "_instance"): @@ -209,7 +211,7 @@ class SearchDB(): (return_tuple(None, None, search_name), 'NOT_FOUND')) return ResponseCode.SUCCESS, result_list - def get_binary_in_other_database(self, not_found_binary, db_): + def _get_binary_in_other_database(self, not_found_binary): """ Description: Binary package name data not found in the current database, go to other databases to try @@ -235,57 +237,59 @@ class SearchDB(): "version", "db_name", "search_version", - "req_name" ]) - src_req_map = {req_: src for src, req_ in not_found_binary} - - local_search_set = {req_ for _, req_ in not_found_binary} - - local_dict = {k: v for k, v in self.db_object_dict.items() if k != db_} - res = [] + search_list = [] + result_list = [] + for db_name, data_base in self.db_object_dict.items(): + for key, _ in not_found_binary.items(): + search_list.append(key) - for db_name, data_base in local_dict.items(): + search_set = set(search_list) + search_list.clear() try: sql_string = text(""" - SELECT - t3.NAME AS source_name, - t1.NAME AS bin_name, - t1.version, - t3.version as search_version, - t2.NAME AS req_name - FROM - bin_pack t1, - pack_provides t2, - src_pack t3 - WHERE - t2.{} - AND t1.id = t2.binIDkey - AND t1.srcIDkey = t3.id; - """.format(literal_column('name').in_(local_search_set))) - build_set_2 = data_base.session. \ + SELECT DISTINCT + t1.src_name AS source_name, + t1.NAME AS bin_name, + t1.version, + t2.NAME AS req_name + FROM + bin_pack t1, + bin_provides t2 + WHERE + t2.{} + AND t1.pkgKey = t2.pkgKey; + """.format(literal_column('name').in_(search_set))) + bin_set = data_base.session. \ execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(local_search_set, 1)}).fetchall() - if not build_set_2: - continue - - res.extend([return_tuple( - src_req_map.get(bin_pack.req_name), - bin_pack.source_name, - bin_pack.bin_name, - bin_pack.version, - db_name, - bin_pack.search_version, - bin_pack.req_name - ) for bin_pack in build_set_2 if bin_pack.bin_name]) - - for obj in res: - local_search_set.remove(obj.req_name) - - except AttributeError as attr_error: - current_app.logger.error(attr_error) - except SQLAlchemyError as sql_error: - current_app.logger.error(sql_error) - return res + for i, v in enumerate(search_set, 1)}).fetchall() + if bin_set: + for result in bin_set: + if result.req_name not in not_found_binary: + LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + else: + for source_info in not_found_binary[result.req_name]: + obj = return_tuple( + source_info[0], + result.source_name, + result.bin_name, + result.version, + db_name, + source_info[1] + ) + result_list.append(obj) + del not_found_binary[result.req_name] + if not not_found_binary: + return result_list + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) + + if not_found_binary: + for key, values in not_found_binary.items(): + LOGGER.logger.warning("CANNOT FOUND THE component" + key + " in all database") + return result_list def get_build_depend(self, source_name_li): """ @@ -315,89 +319,108 @@ class SearchDB(): if not s_name_set: return ResponseCode.PARAM_ERROR, None - not_found_binary = set() + provides_not_found = dict() build_list = [] for db_name, data_base in self.db_object_dict.items(): - try: - sql_com = text("""SELECT DISTINCT - src.NAME AS search_name, - src.version AS search_version, - s2.NAME AS source_name, - pack_provides.binIDkey AS bin_id, - pack_requires.NAME AS req_name, - bin_pack.version AS version, - bin_pack.NAME AS bin_name - FROM - ( SELECT id, NAME,version FROM src_pack WHERE {} ) src - LEFT JOIN pack_requires ON src.id = pack_requires.srcIDkey - LEFT JOIN pack_provides ON pack_provides.id = pack_requires.depProIDkey - LEFT JOIN bin_pack ON bin_pack.id = pack_provides.binIDkey - LEFT JOIN src_pack s1 ON s1.id = pack_requires.srcIDkey - LEFT JOIN src_pack s2 ON bin_pack.srcIDkey = s2.id; - """.format(literal_column("name").in_(s_name_set))) - - build_set = data_base.session. \ - execute(sql_com, {'name_{}'.format(i): v - for i, v in enumerate(s_name_set, 1)}).fetchall() - - if not build_set: - continue - # When processing source package without compilation dependency - to_remove_obj_index = [] - for index, b_pack in enumerate(build_set): - if not b_pack.source_name and not b_pack.req_name: - obj = return_tuple( - b_pack.search_name, - b_pack.source_name, - b_pack.bin_name, - b_pack.version, - db_name, - b_pack.search_version + build_set = [] + try: + temp_list = list(s_name_set) + for input_name_li in [temp_list[i:i + 900] for i in range(0, len(temp_list), 900)]: + sql_com = text(""" + SELECT DISTINCT + src.NAME AS search_name, + src.version AS search_version, + bin_pack.src_name AS source_name, + bin_provides.pkgKey AS bin_id, + src_requires.NAME AS req_name, + bin_pack.version AS version, + bin_pack.NAME AS bin_name + FROM + ( SELECT pkgKey, NAME, version FROM src_pack WHERE {} ) src + LEFT JOIN src_requires ON src.pkgKey = src_requires.pkgKey + LEFT JOIN bin_provides ON bin_provides.NAME = src_requires.NAME + LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; + """.format(literal_column("name").in_(input_name_li))) + res = data_base.session.execute( + sql_com, + {'name_{}'.format(i): v + for i, v in enumerate(input_name_li, 1)} + ).fetchall() + + build_set.extend(res) + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) + + if not build_set: + continue + + # When processing source package without compilation dependency + get_list = [] + for result in build_set: + get_list.append(result.search_name) + if not result.bin_name and result.req_name: + if result.req_name in provides_not_found: + provides_not_found[result.req_name].append( + [result.search_name, result.search_version, db_name] ) - - build_list.append(obj) - to_remove_obj_index.append(index) - - for i in reversed(to_remove_obj_index): - build_set.pop(i) - - if not build_set: - continue - - build_list.extend([ - return_tuple( - bin_pack.search_name, - bin_pack.source_name, - bin_pack.bin_name, - bin_pack.version, + else: + provides_not_found[result.req_name] = [ + [result.search_name, result.search_version, db_name] + ] + else: + obj = return_tuple( + result.search_name, + result.source_name, + result.bin_name, + result.version, db_name, - bin_pack.search_version - ) for bin_pack in build_set if bin_pack.bin_id and bin_pack.bin_name - ]) - # Component name can't find its binary package name - not_found_binary.update([(bin_pack.search_name, bin_pack.req_name) - for bin_pack in build_set if not bin_pack.bin_id]) - - s_name_set -= {bin_pack.search_name for bin_pack in build_set - if bin_pack.bin_id} - - if not not_found_binary and not s_name_set: - return ResponseCode.SUCCESS, build_list - - for obj in self.get_binary_in_other_database(not_found_binary, db_name): + result.search_version + ) build_list.append(obj) - not_found_binary.clear() - - except AttributeError as attr_error: - current_app.logger.error(attr_error) - except SQLAlchemyError as sql_error: - current_app.logger.error(sql_error) - return ResponseCode.DIS_CONNECTION_DB, None + get_set = set(get_list) + get_list.clear() + s_name_set.symmetric_difference_update(get_set) + if not s_name_set: + build_result = self._get_binary_in_other_database(provides_not_found) + build_list.extend(build_result) + return ResponseCode.SUCCESS, build_list + + if s_name_set: + build_result = self._get_binary_in_other_database(provides_not_found) + build_list.extend(build_result) + for source in s_name_set: + LOGGER.logger.warning("CANNOT FOUND THE source " + source + " in all database") return ResponseCode.SUCCESS, build_list + def binary_search_database_for_first_time(self, binary_name): + """ + Args: + binary_name: a binary package name + + Returns: + The name of the first database + in which the binary package appears according to priority + If it does not exist or exception occurred , return 'NOT FOUND' + + """ + try: + for db_name, data_base in self.db_object_dict.items(): + if data_base.session.query( + exists().where(bin_pack.name == binary_name) + ).scalar(): + return db_name + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) + + return 'NOT FOUND' + def db_priority(): """