From d3b2f9f40043c8a87b247defec1f1ad0d3ea821c Mon Sep 17 00:00:00 2001 From: Zhangyifan Date: Fri, 20 Nov 2020 11:33:52 +0800 Subject: [PATCH] =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=8C=87=E5=AE=9A=E8=B7=9F?= =?UTF-8?q?=E8=B8=AA=E7=9A=84=E8=B5=B7=E5=A7=8Bcommit=5Fid=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、git类型，判断commit是否存在哪些分支;2、github类型，获取最新前100条commit，如果指定跟踪的起始commit不在前100条中，日志记录最近100条commit中没有找到指定commit --- README.md | 37 +- patch_tracking/cli/patch_tracking_cli.py | 7 +- patch_tracking/tests/tracking_test.py | 458 +++++++++++++---------- patch_tracking/util/upstream/git.py | 88 ++++- patch_tracking/util/upstream/github.py | 27 +- 5 files changed, 378 insertions(+), 239 deletions(-) diff --git a/README.md b/README.md index bf1a37e..898e68a 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ patch-tracking采用 C/S 架构。 3. 创建临时分支，将获取到的补丁文件提交到临时分支。 4. 自动提交issue到对应项目，并生成关联 issue 的 PR。 -![PatchTracking](images/PatchTracking.jpg) +![PatchTracking](./images/PatchTracking.jpg) * Maintainer对提交的补丁处理流程 @@ -37,7 +37,7 @@ patch-tracking采用 C/S 架构。 1. Maintainer分析临时分支中的补丁文件，判断是否合入。 2. 执行构建，构建成功后判断是否合入PR。 -![Maintainer](images/Maintainer.jpg) +![Maintainer](./images/Maintainer.jpg) ## 数据结构 @@ -75,7 +75,7 @@ rpm 包获取地址：https://build.openeuler.org/package/show/openEuler:20.09/p #### 方法1：从repo源安装 -1. 使用 dnf 挂载 repo源（需要 20.09 或更新的 repo 源，具体方法参考[应用开发指南](https://openeuler.org/zh/docs/20.03_LTS/docs/ApplicationDev/%E5%BC%80%E5%8F%91%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87.html)），然后执行如下指令下载以及安装pkgship及其依赖。 +1. 使用 dnf 挂载 repo源（需要 20.09 或更新的 repo 源，具体方法参考[应用开发指南](https://openeuler.org/zh/docs/20.03_LTS/docs/ApplicationDev/%E5%BC%80%E5%8F%91%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87.html)），然后执行如下指令下载以及安装 patch-tracking 及其依赖。 2. 执行以下命令安装`patch-tracking`。 @@ -88,7 +88,7 @@ rpm 包获取地址：https://build.openeuler.org/package/show/openEuler:20.09/p 1. 首先安装相关依赖。 ```shell script - dnf install python3-uWSGI python3-flask python3-Flask-SQLAlchemy python3-Flask-APScheduler python3-Flask-HTTPAuth python3-requests python3-pandas + dnf install python3-uWSGI python3-flask python3-Flask-SQLAlchemy python3-Flask-APScheduler python3-Flask-HTTPAuth python3-requests python3-pandas git ``` 2. 以`patch-tracking-1.0.0-1.oe1.noarch.rpm`为例，执行如下命令安装。 @@ -147,7 +147,7 @@ openssl req -x509 -days 3650 -subj "/CN=self-signed" \ > `USER`默认值为`admin`。 -​ 执行如下指令，获取口令的哈希值，其中Test@123为设置的口令。 +执行如下指令，获取口令的哈希值，其中Test@123为设置的口令。 ``` [root]# generate_password Test@123 @@ -189,16 +189,17 @@ pbkdf2:sha256:150000$w38eLeRm$ebb5069ba3b4dda39a698bd1d9d7f5f848af3bd93b11e0cde2 >--user ：POST接口需要进行认证的用户名，同settings.conf中的USER参数 \ --password ：POST接口需要进行认证的口令，为settings.conf中的PASSWORD哈希值对应的实际的口令字符串 \ --server ：启动Patch Tracking服务的URL，例如：127.0.0.1:5001 \ ---version_control ：上游仓库版本的控制工具，只支持github \ ---repo： 需要进行跟踪的仓库名称，格式：组织/仓库 \ +--version_control ：上游仓库版本的控制工具，支持github, git类型 \ +--repo： 需要进行跟踪的仓库URL，不支持需要配置SSH key公私钥对才能克隆的URL \ --branch ：需要进行跟踪的仓库的分支名称 \ ---scm_repo ：被跟踪的上游仓库的仓库名称，github格式：组织/仓库 \ +--scm_repo ：被跟踪的上游仓库的仓库名称，--version_control为github的格式：组织/仓库；--version_control为git的格式：仓库URL，不支持需要配置SSH key公私钥对才能克隆的URL \ --scm_branch： 被跟踪的上游仓库的仓库的分支 \ +--scm_commit：指定跟踪的起始commit，选填，默认从当前最新commit开始跟踪 \ --enabled ：是否自动跟踪该仓库 例如： ```shell script -patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --enabled true +patch-tracking-cli add --server 127.0.0.1:5001 --user admin --password Test@123 --version_control github --repo https://gitee.com/testPatchTrack/testPatch1 --branch master --scm_repo BJMX/testPatch01 --scm_branch test --scm_commit --enabled true ``` ### 指定文件添加 @@ -220,20 +221,22 @@ yaml文件内容格式如下，冒号左边的内容不可修改，右边内容 ```shell script version_control: github -scm_repo: xxx/xxx +scm_repo: scm_branch: master -repo: xxx/xxx +repo: branch: master enabled: true ``` >version_control ：上游仓库版本的控制工具，只支持github \ -scm_repo ：被跟踪的上游仓库的仓库名称，github格式：组织/仓库 \ +scm_repo ：被跟踪的上游仓库的仓库名称，github格式：组织/仓库；git格式：仓库URL，不支持需要配置SSH key公私钥对才能克隆的URL \ scm_branch ：被跟踪的上游仓库的仓库的分支 \ -repo ：需要进行跟踪的仓库名称，格式：组织/仓库 \ +repo ：需要进行跟踪的仓库URL，不支持需要配置SSH key公私钥对才能克隆的URL \ branch ：需要进行跟踪的仓库的分支名称 \ enabled ：是否自动跟踪该仓库 +如果指定起始的commit，增加一行`scm_commit: ` + ### 指定目录添加 在指定的目录，例如`test_yaml`下放入多个`xxx.yaml`文件，执行如下命令，记录指定目录下所有yaml文件的跟踪项。 @@ -280,7 +283,7 @@ patch-tracking-cli delete --server SERVER --user USER --password PWD --repo REPO ``` 例如： ```shell script -patch-tracking-cli delete --server 127.0.0.1:5001 --user admin --password Test@123 --repo testPatchTrack/testPatch1 --branch master +patch-tracking-cli delete --server 127.0.0.1:5001 --user admin --password Test@123 --repo https://gitee.com/testPatchTrack/testPatch1 --branch master ``` > 可以删除指定repo和branch的单条数据；也可直接删除指定repo下所有branch的数据。 @@ -291,4 +294,10 @@ patch-tracking-cli delete --server 127.0.0.1:5001 --user admin --password Test@1 登录Gitee上进行跟踪的软件项目，在该项目的Issues和Pull Requests页签下，可以查看到名为`[patch tracking] TIME`，例如` [patch tracking] 20200713101548`的条目，该条目即是刚生成的补丁文件的issue和对应PR。 +# FAQ + +* 访问 `api.github.com` Connection refused 异常 + +patch-tracking 运行过程中，可能会出现如下报错 `9月 21 22:00:10 localhost.localdomain patch-tracking[36358]: 2020-09-21 22:00:10,812 - patch_tracking.util.github_api - WARNING - HTTPSConnectionPool(host='api.github.com', port=443): Max retries exceeded with url: /user (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))`，原因是 patch-tracking 与 GitHub API 服务之间网络访问不稳定导致。 +请确保在与 GitHub API 服务之间网络稳定的环境中（如华为云香港区域）运行 patch-tracking。 diff --git a/patch_tracking/cli/patch_tracking_cli.py b/patch_tracking/cli/patch_tracking_cli.py index 2c54058..61d6739 100755 --- a/patch_tracking/cli/patch_tracking_cli.py +++ b/patch_tracking/cli/patch_tracking_cli.py @@ -10,7 +10,6 @@ import pandas import requests from requests.auth import HTTPBasicAuth from requests.packages.urllib3.exceptions import InsecureRequestWarning -from requests.packages.urllib3.exceptions import HTTPError requests.packages.urllib3.disable_warnings(InsecureRequestWarning) pandas.set_option('display.max_rows', None) @@ -156,6 +155,7 @@ def params_input_track(params, file_path=None): branch = params['branch'] scm_repo = params['scm_repo'] scm_branch = params['scm_branch'] + scm_commit = params['scm_commit'] version_control = params['version_control'].lower() enabled = params['enabled'].lower() server = params['server'] @@ -185,6 +185,7 @@ def params_input_track(params, file_path=None): 'version_control': version_control, 'scm_repo': scm_repo, 'scm_branch': scm_branch, + 'scm_commit': scm_commit, 'repo': repo, 'branch': branch, 'enabled': enabled @@ -248,6 +249,7 @@ def add(args): 'branch': args.branch, 'scm_repo': args.scm_repo, 'scm_branch': args.scm_branch, + 'scm_commit': args.scm_commit, 'version_control': args.version_control, 'enabled': args.enabled, 'server': args.server, @@ -388,7 +390,7 @@ authentication_parser.add_argument('--password', required=True, help='authentica ADD_USAGE = """ patch_tracking_cli add --server SERVER --user USER --password PASSWORD --version_control github --scm_repo SCM_REPO --scm_branch SCM_BRANCH - --repo REPO --branch BRANCH --enabled True + --repo REPO --branch BRANCH --enabled True [--scm_commit SHA] patch_tracking_cli add --server SERVER --user USER --password PASSWORD --file FILE patch_tracking_cli add --server SERVER --user USER --password PASSWORD --dir DIR""" parser_add = subparsers.add_parser( @@ -398,6 +400,7 @@ parser_add.set_defaults(func=add) parser_add.add_argument("--version_control", choices=["github", "git"], help="upstream version control system") parser_add.add_argument("--scm_repo", help="upstream scm repository") parser_add.add_argument("--scm_branch", help="upstream scm branch") +parser_add.add_argument("--scm_commit", help="upstream scm commit sha") parser_add.add_argument("--repo", help="source package repository") parser_add.add_argument("--branch", help="source package branch") parser_add.add_argument("--enabled", choices=["True", "true", "False", "false"], help="whether tracing is enabled") diff --git a/patch_tracking/tests/tracking_test.py b/patch_tracking/tests/tracking_test.py index 79c1962..f6f797b 100644 --- a/patch_tracking/tests/tracking_test.py +++ b/patch_tracking/tests/tracking_test.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- -''' +""" Automated testing of the Tracking interface, including POST requests and GET requests -''' +""" import unittest import json from base64 import b64encode @@ -13,14 +13,14 @@ from patch_tracking.api.constant import ResponseCode class TestTracking(unittest.TestCase): - ''' + """ Automated testing of the Tracking interface, including POST requests and GET requests - ''' - def setUp(self) -> None: - ''' + """ + def setUp(self): + """ Prepare the environment :return: - ''' + """ self.client = app.test_client() reset_db.reset() app.config["USER"] = "hello" @@ -30,10 +30,10 @@ class TestTracking(unittest.TestCase): self.auth = {"Authorization": f"Basic {credentials}"} def test_none_data(self): - ''' + """ In the absence of data, the GET interface queries all the data :return: - ''' + """ with app.app_context(): resp = self.client.get("/tracking") @@ -55,10 +55,10 @@ class TestTracking(unittest.TestCase): self.assertEqual(resp_dict.get("data"), [], msg="Error in data information return") def test_find_nonexistent_data(self): - ''' + """ The GET interface queries data that does not exist :return: - ''' + """ with app.app_context(): resp = self.client.get("/tracking?repo=aa&branch=aa") @@ -80,11 +80,11 @@ class TestTracking(unittest.TestCase): self.assertEqual(resp_dict.get("data"), [], msg="Error in data information return") def test_insert_data(self): - ''' + """ The POST interface inserts data :return: - ''' - data = { + """ + data_github = { "version_control": "github", "scm_repo": "A", "scm_branch": "A", @@ -94,41 +94,18 @@ class TestTracking(unittest.TestCase): "enabled": 0 } - resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") - - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS), - resp_dict.get("msg"), - msg="Error in status code return" - ) - - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertIsNotNone(resp_dict.get("data"), msg="Error in data information return") - - def test_query_inserted_data(self): - ''' - The GET interface queries existing data - :return: - ''' - with app.app_context(): - data_insert = { - "version_control": "github", - "scm_repo": "B", - "scm_branch": "B", - "scm_commit": "B", - "repo": "B", - "branch": "B", - "enabled": False - } - - create_tracking(data_insert) - - resp = self.client.get("/tracking?repo=B&branch=B") + data_git = { + "version_control": "git", + "scm_repo": "A", + "scm_branch": "A", + "scm_commit": "A", + "repo": "A", + "branch": "A", + "enabled": 0 + } + for data in [data_git, data_github]: + resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) resp_dict = json.loads(resp.data) self.assertIn("code", resp_dict, msg="Error in data format return") self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") @@ -142,37 +119,72 @@ class TestTracking(unittest.TestCase): self.assertIn("data", resp_dict, msg="Error in data format return") self.assertIsNotNone(resp_dict.get("data"), msg="Error in data information return") - self.assertIn(data_insert, resp_dict.get("data"), msg="Error in data information return") - def test_only_input_branch(self): - ''' - Get interface queries enter only BRANCH, not REPO + def test_query_inserted_data(self): + """ + The GET interface queries existing data :return: - ''' + """ with app.app_context(): - data_insert = { + data_github = { "version_control": "github", - "scm_repo": "C", - "scm_branch": "C", - "scm_commit": "C", - "repo": "C", - "branch": "C", - "enabled": 0 + "scm_repo": "B", + "scm_branch": "B", + "scm_commit": "B", + "repo": "B1", + "branch": "B1", + "enabled": False + } + + data_git = { + "version_control": "git", + "scm_repo": "B", + "scm_branch": "B", + "scm_commit": "B", + "repo": "B2", + "branch": "B2", + "enabled": False } - create_tracking(data_insert) + for data_insert in [data_github, data_git]: + create_tracking(data_insert) - resp = self.client.get("/tracking?branch=B") + resp = self.client.get("/tracking?repo={}&branch={}".format(data_insert["repo"], data_insert["branch"])) + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") + + self.assertIn("msg", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS), + resp_dict.get("msg"), + msg="Error in status code return" + ) + + self.assertIn("data", resp_dict, msg="Error in data format return") + self.assertIsNotNone(resp_dict.get("data"), msg="Error in data information return") + self.assertIn(data_insert, resp_dict.get("data"), msg="Error in data information return") + + def test_fewer_parameters(self): + """ + When the POST interface passes in parameters, fewer parameters must be passed + :return: + """ + data_github = {"version_control": "github", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": 1} + data_git = {"version_control": "git", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": 1} + + for data in [data_git, data_github]: + resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) resp_dict = json.loads(resp.data) self.assertIn("code", resp_dict, msg="Error in data format return") self.assertEqual( - ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return" + ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return" ) self.assertIn("msg", resp_dict, msg="Error in data format return") self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS), + ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), resp_dict.get("msg"), msg="Error in status code return" ) @@ -180,92 +192,42 @@ class TestTracking(unittest.TestCase): self.assertIn("data", resp_dict, msg="Error in data format return") self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") - def test_fewer_parameters(self): - ''' - When the POST interface passes in parameters, fewer parameters must be passed - :return: - ''' - data = {"version_control": "github", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": 1} - - resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return") - - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), - resp_dict.get("msg"), - msg="Error in status code return" - ) - - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") - def test_error_parameters_value(self): - ''' + """ The post interface passes in the wrong parameter :return: - ''' - data = {"version_control": "github", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": "AA"} + """ + data_github = {"version_control": "github", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": "AA"} + data_git = {"version_control": "git", "scm_commit": "AA", "repo": "AA", "branch": "AA", "enabled": "AA"} - resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return") + for data in [data_git, data_github]: + resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return" + ) - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), - resp_dict.get("msg"), - msg="Error in status code return" - ) + self.assertIn("msg", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), + resp_dict.get("msg"), + msg="Error in status code return" + ) - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") + self.assertIn("data", resp_dict, msg="Error in data format return") + self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") def test_post_error_parameters(self): - ''' + """ The post interface passes in the wrong parameter :return: - ''' - data = {"version_control": "github", "scm_commit": "AA", "oper": "AA", "hcnarb": "AA", "enabled": "AA"} - - resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return") - - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), - resp_dict.get("msg"), - msg="Error in status code return" - ) - - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") - - def test_get_error_parameters(self): - ''' - The get interface passes in the wrong parameter - :return: - ''' - with app.app_context(): - data_insert = { - "version_control": "github", - "scm_repo": "BB", - "scm_branch": "BB", - "scm_commit": "BB", - "repo": "BB", - "branch": "BB", - "enabled": True - } - - create_tracking(data_insert) - - resp = self.client.get("/tracking?oper=B&chcnsrb=B") + """ + data_github = {"version_control": "github", "scm_commit": "AA", "oper": "AA", "hcnarb": "AA", "enabled": "AA"} + data_git = {"version_control": "git", "scm_commit": "AA", "oper": "AA", "hcnarb": "AA", "enabled": "AA"} + for data in [data_git, data_github]: + resp = self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) resp_dict = json.loads(resp.data) self.assertIn("code", resp_dict, msg="Error in data format return") self.assertEqual( @@ -282,94 +244,171 @@ class TestTracking(unittest.TestCase): self.assertIn("data", resp_dict, msg="Error in data format return") self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") - def test_update_data(self): - ''' - update data + def test_get_error_parameters(self): + """ + The get interface passes in the wrong parameter :return: - ''' + """ with app.app_context(): - data_old = { + data_github = { "version_control": "github", - "scm_repo": "str", - "scm_branch": "str", - "scm_commit": "str", - "repo": "string", - "branch": "string", - "enabled": False + "scm_repo": "BB", + "scm_branch": "BB", + "scm_commit": "BB", + "repo": "BB1", + "branch": "BB1", + "enabled": True } - self.client.post("/tracking", json=data_old, content_type="application/json", headers=self.auth) - - data_new = { - "branch": "string", - "enabled": True, - "repo": "string", - "scm_branch": "string", - "scm_commit": "string", - "scm_repo": "string", + data_git = { "version_control": "github", + "scm_repo": "BB", + "scm_branch": "BB", + "scm_commit": "BB", + "repo": "BB2", + "branch": "BB2", + "enabled": True } - self.client.post("/tracking", json=data_new, content_type="application/json") + for data_insert in [data_github, data_git]: + create_tracking(data_insert) - resp = self.client.get("/tracking?repo=string&branch=string") + resp = self.client.get("/tracking?oper=B&chcnsrb=B") - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return" + ) - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS), - resp_dict.get("msg"), - msg="Error in status code return" - ) + self.assertIn("msg", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), + resp_dict.get("msg"), + msg="Error in status code return" + ) - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertIsNotNone(resp_dict.get("data"), msg="Error in data information return") - #self.assertIn(data_new, resp_dict.get("data"), msg="Error in data information return") + self.assertIn("data", resp_dict, msg="Error in data format return") + self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") + + def test_update_data(self): + """ + update data + :return: + """ + with app.app_context(): + data_old_list = [ + { + "version_control": "github", + "scm_repo": "str", + "scm_branch": "str", + "scm_commit": "str", + "repo": "string", + "branch": "string", + "enabled": False + }, { + "version_control": "git", + "scm_repo": "str", + "scm_branch": "str", + "scm_commit": "str", + "repo": "string", + "branch": "string", + "enabled": False + } + ] + data_new_list = [ + { + "version_control": "github", + "scm_repo": "string", + "scm_branch": "string", + "scm_commit": "string", + "repo": "string", + "branch": "string", + "enabled": False + }, { + "branch": "string", + "enabled": True, + "repo": "string", + "scm_branch": "string", + "scm_commit": "string", + "scm_repo": "string", + "version_control": "git", + } + ] + + for data_old, data_new in zip(data_old_list, data_new_list): + self.client.post("/tracking", json=data_old, content_type="application/json", headers=self.auth) + self.client.post("/tracking", json=data_new, content_type="application/json") + + resp = self.client.get("/tracking?repo=string&branch=string") + + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") + + self.assertIn("msg", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.CODE_MSG_MAP.get(ResponseCode.SUCCESS), + resp_dict.get("msg"), + msg="Error in status code return" + ) + + self.assertIn("data", resp_dict, msg="Error in data format return") + self.assertIsNotNone(resp_dict.get("data"), msg="Error in data information return") + #self.assertIn(data_new, resp_dict.get("data"), msg="Error in data information return") def test_get_interface_uppercase(self): - ''' + """ The get interface uppercase :return: - ''' + """ with app.app_context(): - data_insert = { + data_github = { "version_control": "github", "scm_repo": "BBB", "scm_branch": "BBB", "scm_commit": "BBB", - "repo": "BBB", - "branch": "BBB", + "repo": "BBB1", + "branch": "BBB1", "enabled": False } - create_tracking(data_insert) + data_git = { + "version_control": "github", + "scm_repo": "BBB", + "scm_branch": "BBB", + "scm_commit": "BBB", + "repo": "BBB2", + "branch": "BBB2", + "enabled": False + } - resp = self.client.get("/tracking?rep=BBB&BRAnch=BBB") + for data_insert in [data_github, data_git]: + create_tracking(data_insert) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return" - ) + resp = self.client.get("/tracking?rep=BBB&BRAnch=BBB") - self.assertIn("msg", resp_dict, msg="Error in data format return") - self.assertEqual( - ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), - resp_dict.get("msg"), - msg="Error in status code return" - ) + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.INPUT_PARAMETERS_ERROR, resp_dict.get("code"), msg="Error in status code return" + ) - self.assertIn("data", resp_dict, msg="Error in data format return") - self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") + self.assertIn("msg", resp_dict, msg="Error in data format return") + self.assertEqual( + ResponseCode.CODE_MSG_MAP.get(ResponseCode.INPUT_PARAMETERS_ERROR), + resp_dict.get("msg"), + msg="Error in status code return" + ) + + self.assertIn("data", resp_dict, msg="Error in data format return") + self.assertEqual(resp_dict.get("data"), None, msg="Error in data information return") def test_version_control_error(self): - ''' + """ The POST version control error :return: - ''' + """ data = { "version_control": "gitgitgit", "scm_repo": "A", @@ -400,7 +439,7 @@ class TestTracking(unittest.TestCase): The POST interface inserts data :return: """ - data = { + data_github = { "version_control": "github", "scm_repo": "test_delete", "scm_branch": "test_delete", @@ -410,19 +449,34 @@ class TestTracking(unittest.TestCase): "enabled": 0 } - self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) + data_git = { + "version_control": "git", + "scm_repo": "test_delete", + "scm_branch": "test_delete", + "scm_commit": "test_delete", + "repo": "test_delete1", + "branch": "test_delete1", + "enabled": 0 + } - resp = self.client.delete("/tracking?repo=test_delete1&branch=test_delete1", content_type="application/json", headers=self.auth) - resp_dict = json.loads(resp.data) - self.assertIn("code", resp_dict, msg="Error in data format return") - self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") + for data in [data_git, data_github]: + self.client.post("/tracking", json=data, content_type="application/json", headers=self.auth) + + resp = self.client.delete( + "/tracking?repo=test_delete1&branch=test_delete1", content_type="application/json", headers=self.auth + ) + resp_dict = json.loads(resp.data) + self.assertIn("code", resp_dict, msg="Error in data format return") + self.assertEqual(ResponseCode.SUCCESS, resp_dict.get("code"), msg="Error in status code return") def test_delete_not_found(self): """ The POST interface inserts data :return: """ - resp = self.client.delete("/tracking?repo=not_found1&branch=not_found1", content_type="application/json", headers=self.auth) + resp = self.client.delete( + "/tracking?repo=not_found1&branch=not_found1", content_type="application/json", headers=self.auth + ) resp_dict = json.loads(resp.data) self.assertIn("code", resp_dict, msg="Error in data format return") self.assertEqual(ResponseCode.DELETE_DB_NOT_FOUND, resp_dict.get("code"), msg="Error in status code return") diff --git a/patch_tracking/util/upstream/git.py b/patch_tracking/util/upstream/git.py index d5604ea..a7e1485 100644 --- a/patch_tracking/util/upstream/git.py +++ b/patch_tracking/util/upstream/git.py @@ -6,6 +6,7 @@ import git import git.exc from flask import current_app from sqlalchemy.exc import SQLAlchemyError +import time from patch_tracking.api.business import update_tracking import patch_tracking.util.upstream.upstream as upstream @@ -60,28 +61,41 @@ class Git(upstream.Upstream): def git_fetch(self, repo): """git fetch""" logging.info("Fetching repo %s.", repo) - try: - if os.path.exists(repo): - repo = git.Repo(repo) - repo.remote().fetch() - logging.info("Fetch repo %s finish.", repo) - else: + if os.path.exists(repo): + repo = git.Repo(repo) + count = 10 + while count > 0: + try: + repo.remote().fetch() + logging.info("Fetch repo %s finish.", repo) + break + except git.exc.GitError as err: + logging.warning("Fetching repo %s failed. Error: %s", repo, err) + count -= 1 + time.sleep(1) + if count == 0: + logging.error("Fetching repo %s failed.", repo) + return False + else: + try: self.git_clone() logging.info("Cloned repo done %s.", repo) - return True - except git.exc.GitError as err: - logging.error("Fetching repo %s failed. Error: %s", repo, err) - return False + except git.exc.GitError as err: + logging.error("Fetching repo %s failed. Error: %s", repo, err) + return False + return True def git_latest_sha(self): """ get latest commit id """ repo_path = os.path.join(self.base_path, self.repo_dir_name) - logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.scm_branch) try: repo = git.Repo(repo_path) sha = repo.commit(self.track.scm_branch).hexsha + logging.info( + "Getting latest commit id of repo: %s branch: %s sha: %s .", repo_path, self.track.scm_branch, sha + ) return sha except git.exc.GitError as err: logging.error( @@ -89,17 +103,53 @@ class Git(upstream.Upstream): ) return False - def get_commit_list(self, repo, start_commit): + def get_all_commit_list(self): + """get all commits of a repo/branch""" + repo_path = os.path.join(self.base_path, self.repo_dir_name) + logging.info("Getting all commit id of repo: %s.", repo_path) + try: + repo = git.Repo(repo_path) + repo.git.symbolic_ref("HEAD", "refs/heads/" + self.track.scm_branch) + all_commit_list = [str(item) for item in repo.iter_commits()] + logging.info("Get all commit id of repo: %s branch: %s.", repo_path, self.track.scm_branch) + return all_commit_list + except git.exc.GitError as err: + logging.error("Get all commit id of repo: %s failed. Error: %s", repo_path, err) + return False + + def check_commit_exist(self, commit): + """check commit exist""" + repo_path = os.path.join(self.base_path, self.repo_dir_name) + branch = self.track.scm_branch + repo = git.Repo(repo_path) + try: + ret = repo.git.branch("--contains", commit) + if ret: + ret = [r.split(" ")[-1] for r in ret.split("\n")] + if branch in ret: + return True + return False + except git.exc.GitCommandError as err: + logging.error("Error: %s.", err) + return False + + def get_commit_list(self, start_commit, latest_commit): """get commit list""" commit_list = list() + if start_commit == latest_commit: + return commit_list fetch_ret = self.git_fetch(os.path.join(self.base_path, self.repo_dir_name)) if fetch_ret: - repo = git.Repo(repo) - all_commit_list = list(repo.iter_commits()) - commit_list = list() - for item in all_commit_list: - if str(item) != start_commit: - commit_list.append(str(item)) + if not self.check_commit_exist(start_commit): + logging.error( + "Commit sha: %s not exist in repo: %s branch: %s.", start_commit, self.track.scm_repo, + self.track.scm_branch + ) + return commit_list + all_commit_list = self.get_all_commit_list() + for commit in all_commit_list: + if commit != start_commit: + commit_list.append(commit) else: break commit_list.append(start_commit) @@ -162,7 +212,7 @@ class Git(upstream.Upstream): return None - commit_list = self.get_commit_list(repo, self.track.scm_commit) + commit_list = self.get_commit_list(self.track.scm_commit, latest_commit) if not commit_list: return None diff --git a/patch_tracking/util/upstream/github.py b/patch_tracking/util/upstream/github.py index 28bf8fb..fc971ee 100644 --- a/patch_tracking/util/upstream/github.py +++ b/patch_tracking/util/upstream/github.py @@ -62,7 +62,7 @@ class GitHub(upstream.Upstream): 'Accept': 'application/json' } - def api_request(self, url): + def api_request(self, url, params=None): """ request GitHub API """ @@ -70,7 +70,7 @@ class GitHub(upstream.Upstream): count = 30 while count > 0: try: - response = requests.get(url, headers=self.headers) + response = requests.get(url, headers=self.headers, params=params) return response except exceptions.ConnectionError as err: logger.warning(err) @@ -111,6 +111,20 @@ class GitHub(upstream.Upstream): ret_dict['api_ret'] = 'fail to connect github by api.' return ret_dict + def get_all_commit_list(self): + """ + get latest 100 commit + """ + url = '/'.join(['https://api.github.com/repos', self.track.scm_repo, 'commits']) + params = {"sha": self.track.scm_branch, "page": 0, "per_page": 100} + all_commits = list() + ret = self.api_request(url, params=params) + for item in ret.json(): + all_commits.append(item) + + logger.debug('[Patch Tracking] Successful get all commits.') + return all_commits + def get_latest_commit_id(self): """ get latest commit_ID, commit_message, commit_date @@ -161,6 +175,15 @@ class GitHub(upstream.Upstream): """ get patch list """ + all_commits_info = self.get_all_commit_list() + all_commits = [item["sha"] for item in all_commits_info] + + if self.track.scm_commit not in all_commits: + logger.error( + '[Patch Tracking] Scm repo commit : %s not found in latest 100 commits of scm_repo: %s scm_branch: %s.', + self.track.scm_commit, self.track.scm_repo, self.track.scm_branch + ) + return None commit_list = list() status, result = self.get_latest_commit_id() if status != 'success': -- Gitee