From 48ca572374015fb2b720b6b3d6aa9ac5eac3dd46 Mon Sep 17 00:00:00 2001 From: MementoMori <1003350679@qq.com> Date: Tue, 22 Sep 2020 14:20:19 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=9C=80=E5=A4=A7=E9=98=9F=E5=88=97=E9=95=BF?= =?UTF-8?q?=E5=BA=A6=E9=85=8D=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packageship/packageship/application/apps/lifecycle/view.py | 2 +- packageship/test/common_files/package.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packageship/packageship/application/apps/lifecycle/view.py b/packageship/packageship/application/apps/lifecycle/view.py index d8ef03c0..327d4fdd 100644 --- a/packageship/packageship/application/apps/lifecycle/view.py +++ b/packageship/packageship/application/apps/lifecycle/view.py @@ -458,7 +458,7 @@ class IssueCatch(Resource): pool_workers = 10 pool = ThreadPoolExecutor(max_workers=pool_workers) with DBHelper(db_name="lifecycle") as database: - for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer', 'database_info'], + for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'], database.engine.table_names()): cls_model = Packages.package_meta(table_name) for package_item in database.session.query(cls_model).filter( diff --git a/packageship/test/common_files/package.ini b/packageship/test/common_files/package.ini index 11eabd6d..1bfb3279 100644 --- a/packageship/test/common_files/package.ini +++ b/packageship/test/common_files/package.ini @@ -27,4 +27,5 @@ minute = 0 warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ pool_workers = 10 warehouse = src-openeuler +queue_maxsize = 1000 -- Gitee From b684466d8aba3d45aea264c84f07c5f9fbfe26c4 Mon Sep 17 00:00:00 2001 From: MementoMori <1003350679@qq.com> Date: Fri, 25 Sep 2020 10:46:17 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E9=94=99=E8=AF=AF=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/lifecycle/function/gitee.py | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/packageship/packageship/application/apps/lifecycle/function/gitee.py b/packageship/packageship/application/apps/lifecycle/function/gitee.py index 4ac077f1..c0dbd325 100644 --- a/packageship/packageship/application/apps/lifecycle/function/gitee.py +++ b/packageship/packageship/application/apps/lifecycle/function/gitee.py @@ -42,6 +42,8 @@ class Gitee(): "patch_files_path") self.table_name = table_name self.producer_consumer = ProducerConsumer() + self._issue_url = None + self.total_page = 0 def query_issues_info(self, issue_id=""): """ @@ -53,55 +55,58 @@ class Gitee(): Raises: """ - issue_url = self.api_url + \ - "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) + self._issue_url = self.api_url + \ + "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) try: - response = requests.get( - issue_url, params={"state": "all", "per_page": 100}) - except Error as error: + response = self._request_issue(0) + except HTTPError as error: LOGGER.logger.error(error) return None - if response.status_code != 200: - return None - total_page = 1 if issue_id else int(response.headers['total_page']) + + self.total_page = 1 if issue_id else int( + response.headers['total_page']) total_count = int(response.headers['total_count']) + if total_count > 0: - issue_list = self._query_per_page_issue_info(total_page, issue_url) + issue_list = self._query_per_page_issue_info() if not issue_list: LOGGER.logger.error( "An error occurred while querying {}".format(self.repo)) return None self._save_issues(issue_list) - def _query_per_page_issue_info(self, total_page, issue_url): + @retry(stop_max_attempt_number=3, stop_max_delay=1000) + def _request_issue(self, page): + try: + response = requests.get(self._issue_url, + params={"state": "all", "per_page": 100, "page": page}) + except HTTPError as error: + raise HTTPError(error) + if response.status_code != 200: + err_msg = "There is an exception with the remote service [%s], " \ + "Please try again later,the HTTP error code is:%s" % ( + self._issue_url, str(response.status_code)) + raise HTTPError(err_msg) + return response + + def _query_per_page_issue_info(self): """ Description: View the issue details Args: total_page: total page - issue_url: issue url Returns: """ issue_content_list = [] - for i in range(1, total_page + 1): - - @retry(stop_max_attempt_number=3, stop_max_delay=1000) - def request_issue(page, issue_url): - try: - response = requests.get(issue_url, - params={"state": "all", "per_page": 100, "page": page}) - except HTTPError: - raise HTTPError('Network request error') - return response - + for i in range(1, self.total_page + 1): try: - response = request_issue(i, issue_url) - if response.status_code != 200: - LOGGER.logger.warning(response.content.decode("utf-8")) - continue + response = self._request_issue(i) issue_content_list.extend( self.parse_issues_content(response.json())) + except HTTPError as error: + LOGGER.logger.error(error) + continue except (JSONDecodeError, Error) as error: LOGGER.logger.error(error) return issue_content_list @@ -114,12 +119,9 @@ class Gitee(): try: def _save(issue_module): with DBHelper(db_name='lifecycle') as database: - exist_issues = database.session.query(PackagesIssue).filter( PackagesIssue.issue_id == issue_module['issue_id']).first() if exist_issues: - - # Save the issue for key, val in issue_module.items(): setattr(exist_issues, key, val) else: @@ -130,11 +132,11 @@ class Gitee(): with DBHelper(db_name='lifecycle') as database: database.add(package_module) + # Save the issue for issue_item in issue_list: - self.producer_consumer.put( - (copy.deepcopy(issue_item), _save)) + self.producer_consumer.put((copy.deepcopy(issue_item), _save)) - # The number of various issues in the update package + # The number of various issues in the update package self.pkg_info.defect = self.defect self.pkg_info.feature = self.feature self.pkg_info.cve = self.cve -- Gitee