diff --git a/patch-tracking/.gitignore b/patch-tracking/.gitignore index 283bf0a0cf7a80027a9fff4ffb782238d0105c4c..1e12b5459cfea4cfdcd6de76ce2a6733d28f354b 100644 --- a/patch-tracking/.gitignore +++ b/patch-tracking/.gitignore @@ -68,3 +68,7 @@ ENV/ # log file *.log + + +# git base +git_base diff --git a/patch-tracking/Pipfile b/patch-tracking/Pipfile index 65c8b43e3b622defe15a2288384bea5e12ace0a0..7487b78cea79e685aa34b89a960e05133b087517 100644 --- a/patch-tracking/Pipfile +++ b/patch-tracking/Pipfile @@ -17,6 +17,8 @@ werkzeug = "*" flask-httpauth = "*" sqlalchemy = "*" pandas = "*" +gitpython = "*" +pydriller = "*" [requires] python_version = "3.7" diff --git a/patch-tracking/patch_tracking/api/tracking.py b/patch-tracking/patch_tracking/api/tracking.py index 4a8a24cf78e244656c4a4b587eb2b00b9ebf4268..c34edf83bf5503c477da27272854706bee105775 100644 --- a/patch-tracking/patch_tracking/api/tracking.py +++ b/patch-tracking/patch_tracking/api/tracking.py @@ -22,10 +22,14 @@ def delete(): input_params = request.args keys = list(input_params.keys()) + param_error = False if not keys or "repo" not in keys: - return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) + param_error = True if len(set(keys) - {"repo", "branch"}) != 0: + param_error = True + + if param_error: return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) try: @@ -35,19 +39,16 @@ def delete(): delete_tracking(input_params['repo'], input_params['branch']) logger.info('Delete tracking repo: %s, branch: %s', input_params['repo'], input_params['branch']) return ResponseCode.ret_message(code=ResponseCode.SUCCESS) - else: - logger.info( - 'Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch'] - ) - return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) - else: - if Tracking.query.filter(Tracking.repo == input_params['repo']).first(): - delete_tracking(input_params['repo']) - logger.info('Delete tracking repo: %s', input_params['repo']) - return ResponseCode.ret_message(code=ResponseCode.SUCCESS) - else: - logger.info('Delete tracking repo: %s not found.', input_params['repo']) - return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) + + logger.info('Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch']) + return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) + if Tracking.query.filter(Tracking.repo == input_params['repo']).first(): + delete_tracking(input_params['repo']) + logger.info('Delete tracking repo: %s', input_params['repo']) + return ResponseCode.ret_message(code=ResponseCode.SUCCESS) + + logger.info('Delete tracking repo: %s not found.', input_params['repo']) + return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND) except SQLAlchemyError as err: return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_ERROR, data=err) @@ -98,7 +99,7 @@ def post(): if len(required_params) > 1 or (len(required_params) == 1 and required_params[0] != 'scm_commit'): return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) - if data['version_control'] != 'github': + if data['version_control'] not in ["github", "git"]: return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR) track = Tracking.query.filter_by(repo=data['repo'], branch=data['branch']).first() diff --git a/patch-tracking/patch_tracking/cli/patch_tracking_cli.py b/patch-tracking/patch_tracking/cli/patch_tracking_cli.py index a223d1f2c8f9c5c8d37e0665bce5280cc63018f0..10aaa29d567c644f68873a99bb8f16ff6d1edef0 100755 --- a/patch-tracking/patch_tracking/cli/patch_tracking_cli.py +++ b/patch-tracking/patch_tracking/cli/patch_tracking_cli.py @@ -21,27 +21,20 @@ def query_table(args): if args.table == "tracking": url = '/'.join(['https:/', server, 'tracking']) - params = {'repo': args.repo, 'branch': args.branch} - try: - ret = requests.get(url, params=params, verify=False) - if ret.status_code == 200 and ret.json()['code'] == '2001': - return 'success', ret - - return 'error', ret - except Exception as exception: - return 'error', 'Connect server error: ' + str(exception) elif args.table == "issue": url = '/'.join(['https:/', server, 'issue']) - params = {'repo': args.repo, 'branch': args.branch} - try: - ret = requests.get(url, params=params, verify=False) - if ret.status_code == 200 and ret.json()['code'] == '2001': - return 'success', ret + else: + return 'error', 'table ' + args.table + ' not found' - return 'error', ret - except Exception as exception: - return 'error', 'Connect server error: ' + str(exception) - return 'error', 'table ' + args.table + ' not found' + params = {'repo': args.repo, 'branch': args.branch} + try: + ret = requests.get(url, params=params, verify=False) + if ret.status_code == 200 and ret.json()['code'] == '2001': + return 'success', ret + + return 'error', ret + except Exception as exception: + return 'error', 'Connect server error: ' + str(exception) def add_param_check_url(params, file_path=None): @@ -55,16 +48,19 @@ def add_param_check_url(params, file_path=None): if server_ret[0] != 'success': return 'error' - scm_ret = repo_branch_check(scm_url) - if scm_ret[0] != 'success': - if file_path: - print( - f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. \n" - f"Error in {file_path}. {scm_ret[1]}" - ) - else: - print(f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. {scm_ret[1]}") - return 'error' + if params["version_control"] == "github": + scm_ret = repo_branch_check(scm_url) + if scm_ret[0] != 'success': + if file_path: + print( + f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. \n" + f"Error in {file_path}. {scm_ret[1]}" + ) + else: + print( + f"scm_repo: {params['scm_repo']} and scm_branch: {params['scm_branch']} check failed. {scm_ret[1]}" + ) + return 'error' ret = repo_branch_check(url) if ret[0] != 'success': if file_path: @@ -147,7 +143,7 @@ def params_input_track(params, file_path=None): enabled ) - if version_control not in ["github"]: + if version_control not in ["github", "git"]: print(add_usage) return "error", "error: version_control: invalid value: '{}' (choose from 'github')".format(version_control) @@ -183,6 +179,9 @@ def params_input_track(params, file_path=None): def check_add_param(params): + """ + check add param + """ success = True required_params = ["repo", "branch", "scm_repo", "scm_branch", "version_control", "enabled"] miss_params = list() @@ -192,7 +191,7 @@ def check_add_param(params): success = False if not success: print( - "patch_tracking_cli add: error: the following arguments are required: --{}".format( + "patch_tracking_cli add: error: the following arguments are required: --{}".format( ", --".join(miss_params) ) ) @@ -271,7 +270,7 @@ def delete(args): if ret.status_code == 200 and ret.json()['code'] == '2001': print('Tracking delete successfully.') return - elif ret.status_code == 200 and ret.json()['code'] == '6005': + if ret.status_code == 200 and ret.json()['code'] == '6005': print('Delete Nothing. Tracking not exist.') return @@ -286,9 +285,9 @@ def query(args): """ status, ret = query_table(args) if status == "success": - df = pandas.DataFrame.from_dict(ret.json()["data"], orient="columns") - df.index = range(1, len(df) + 1) - print(df) + data_frame = pandas.DataFrame.from_dict(ret.json()["data"], orient="columns") + data_frame.index = range(1, len(data_frame) + 1) + print(data_frame) else: print(ret) @@ -374,7 +373,7 @@ parser_add = subparsers.add_parser( 'add', parents=[common_parser, authentication_parser], help="add tracking", usage=add_usage, allow_abbrev=False ) parser_add.set_defaults(func=add) -parser_add.add_argument("--version_control", choices=['github'], help="upstream version control system") +parser_add.add_argument("--version_control", choices=["github", "git"], help="upstream version control system") parser_add.add_argument("--scm_repo", help="upstream scm repository") parser_add.add_argument("--scm_branch", help="upstream scm branch") parser_add.add_argument("--repo", help="source package repository") @@ -400,6 +399,9 @@ parser_query.add_argument("--branch", help="source package branch") def main(): + """ + main + """ args_ = parser.parse_args() if args_.subparser_name: if args_.func(args_) != "success": diff --git a/patch-tracking/patch_tracking/settings.conf b/patch-tracking/patch_tracking/settings.conf index 779a498a23953725cb218e86a5aaf8f42b3244d0..d71fcacd52c851ea58730d1089ed4c5e79132201 100644 --- a/patch-tracking/patch_tracking/settings.conf +++ b/patch-tracking/patch_tracking/settings.conf @@ -15,3 +15,6 @@ USER = "admin" # password PASSWORD = "" + +# git clone base path +GIT_BASE_PATH = "" diff --git a/patch-tracking/patch_tracking/task/task.py b/patch-tracking/patch_tracking/task/task.py index 27408b437a578a6e49436888e86f882b8ae31ebd..580bd4e79155af6f46be52abd0bac4f61650b8ac 100644 --- a/patch-tracking/patch_tracking/task/task.py +++ b/patch-tracking/patch_tracking/task/task.py @@ -3,10 +3,13 @@ load job/task of tracking """ import datetime import logging +import os +from pydriller import RepositoryMining from patch_tracking.task import scheduler from patch_tracking.database.models import Tracking from patch_tracking.util.github_api import GitHubApi from patch_tracking.api.business import update_tracking +from patch_tracking.util.git import get_repo_name logger = logging.getLogger(__name__) @@ -59,9 +62,33 @@ def check_empty_commit_id(flask_app): for item in new_track: if item.scm_commit: continue - status, result = github_api.get_latest_commit(item.scm_repo, item.scm_branch) - if status == 'success': - commit_id = result['latest_commit'] + if item.version_control == "github": + status, result = github_api.get_latest_commit(item.scm_repo, item.scm_branch) + if status == 'success': + commit_id = result['latest_commit'] + data = { + 'version_control': item.version_control, + 'repo': item.repo, + 'branch': item.branch, + 'enabled': item.enabled, + 'scm_commit': commit_id, + 'scm_branch': item.scm_branch, + 'scm_repo': item.scm_repo + } + update_tracking(data) + else: + logger.error( + 'Check empty CommitID: Fail to get latest commit id of scm_repo: %s scm_branch: %s. \ + Return val: %s', item.scm_repo, item.scm_branch, result + ) + elif item.version_control == "git": + #get commit lastest id + #update trackign + repo_name = get_repo_name(item.scm_repo) + base_path = flask_app.config['GIT_BASE_PATH'] + repo = os.path.join(base_path, "git_base", repo_name) + commits = RepositoryMining(repo, only_in_branch=item.scm_branch).traverse_commits() + commit_id = list(commits)[-1].hash data = { 'version_control': item.version_control, 'repo': item.repo, diff --git a/patch-tracking/patch_tracking/task/task_apscheduler.py b/patch-tracking/patch_tracking/task/task_apscheduler.py index 41d871877eb78ce068afb06bc8da15d5e90ef119..5f7d39e05428ca7cc7506ed7d902941a0a3e728f 100644 --- a/patch-tracking/patch_tracking/task/task_apscheduler.py +++ b/patch-tracking/patch_tracking/task/task_apscheduler.py @@ -4,6 +4,8 @@ tracking job import logging import base64 import time +import os +from flask import current_app from patch_tracking.util.gitee_api import create_branch, upload_patch, create_gitee_issue from patch_tracking.util.gitee_api import create_pull_request, get_path_content, upload_spec, create_spec from patch_tracking.util.github_api import GitHubApi @@ -11,6 +13,7 @@ from patch_tracking.database.models import Tracking from patch_tracking.api.business import update_tracking, create_issue from patch_tracking.task import scheduler from patch_tracking.util.spec import Spec +from patch_tracking.util.upstream.git_upstream import GitFactory logger = logging.getLogger(__name__) @@ -76,7 +79,6 @@ def get_scm_patch(track): Different warehouse has different acquisition methods :return: """ - github_api = GitHubApi() scm_dict = dict( scm_repo=track.scm_repo, scm_branch=track.scm_branch, @@ -86,42 +88,36 @@ def get_scm_patch(track): branch=track.branch, version_control=track.version_control ) - status, result = github_api.get_latest_commit(scm_dict['scm_repo'], scm_dict['scm_branch']) - logger.debug( - 'repo: %s branch: %s. get_latest_commit: %s %s', scm_dict['scm_repo'], scm_dict['scm_branch'], status, result - ) - if status == 'success': - commit_id = result['latest_commit'] - if not scm_dict['scm_commit']: - data = { - 'version_control': scm_dict['version_control'], - 'repo': scm_dict['repo'], - 'branch': scm_dict['branch'], - 'enabled': scm_dict['enabled'], - 'scm_commit': commit_id, - 'scm_branch': scm_dict['scm_branch'], - 'scm_repo': scm_dict['scm_repo'] - } - update_tracking(data) - logger.info( - '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: None.', - scm_dict['scm_repo'], scm_dict['scm_branch'], result['latest_commit'] - ) - else: - if commit_id != scm_dict['scm_commit']: - commit_list = get_all_commit_info(scm_dict['scm_repo'], scm_dict['scm_commit'], commit_id) - scm_dict['commit_list'] = commit_list - return scm_dict - logger.info( - '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: %s. Nothing need to do.', - scm_dict['scm_repo'], scm_dict['scm_branch'], commit_id, scm_dict['scm_commit'] - ) - else: - logger.error( - '[Patch Tracking] Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s', - scm_dict['scm_repo'], scm_dict['scm_branch'], result + git_factory = GitFactory() + if track.version_control == "github": + github_token = current_app.config['GITHUB_ACCESS_TOKEN'] + git_api = git_factory.get_git( + track.version_control, track.scm_repo, track.scm_branch, scm_dict['scm_commit'], github_token + ) + commit_list = git_api.get_patch_list() + if commit_list: + scm_dict['commit_list'] = commit_list + return scm_dict + + logger.info('repo: %s branch: %s. get_latest_commit is None.', scm_dict['scm_repo'], scm_dict['scm_branch']) + + elif track.version_control == "git": + base_path = current_app.config['GIT_BASE_PATH'] + if not base_path: + base_path = os.path.join(os.path.abspath('.'), "git_base") + git_api = git_factory.get_git( + track.version_control, track.scm_repo, track.scm_branch, scm_dict['scm_commit'], base_path ) + git_api.git_clone(track.scm_repo) + commit_list = git_api.get_patch_list() + + if commit_list: + scm_dict['commit_list'] = commit_list + return scm_dict + + logger.info('repo: %s branch: %s. get_latest_commit is None.', scm_dict['scm_repo'], scm_dict['scm_branch']) + return None diff --git a/patch-tracking/patch_tracking/tests/settings.conf b/patch-tracking/patch_tracking/tests/settings.conf index 960c2d2e5061ff6628a53d5273faaca384dbd034..006edb0dcd4508889833715c9b9a9bbdd03f166f 100644 --- a/patch-tracking/patch_tracking/tests/settings.conf +++ b/patch-tracking/patch_tracking/tests/settings.conf @@ -15,3 +15,6 @@ USER = "" # password PASSWORD = "" + +# git clone base path +GIT_BASE_PATH = "" diff --git a/patch-tracking/patch_tracking/util/git.py b/patch-tracking/patch_tracking/util/git.py new file mode 100644 index 0000000000000000000000000000000000000000..dec82e1f93b25de7552f58509a7445a19a815f8f --- /dev/null +++ b/patch-tracking/patch_tracking/util/git.py @@ -0,0 +1,18 @@ +""" +git common func +""" + + +def get_repo_name(repo): + """ + get scm repo name + """ + repo_strip_right = repo.rstrip(".git") + left_strip = ["@", "//"] + repo_strip = repo_strip_right + for item in left_strip: + if item in repo_strip_right: + repo_strip = repo_strip_right.split(item)[1] + repo_name = repo_strip.replace("/", "_").replace(":", "_") + + return repo_name diff --git a/patch-tracking/patch_tracking/util/upstream/__init__.py b/patch-tracking/patch_tracking/util/upstream/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/patch-tracking/patch_tracking/util/upstream/git_upstream.py b/patch-tracking/patch_tracking/util/upstream/git_upstream.py new file mode 100644 index 0000000000000000000000000000000000000000..021cf7bc866aea6dda058c58d9b274992f36fbd3 --- /dev/null +++ b/patch-tracking/patch_tracking/util/upstream/git_upstream.py @@ -0,0 +1,212 @@ +""" +upstream git repo +""" +import errno +import logging +import os +import shutil +import stat +import git +from pydriller import RepositoryMining +from patch_tracking.util.git import get_repo_name +from patch_tracking.util.github_api import GitHubApi +logger = logging.getLogger(__name__) + + +class GitUpstream: + """ + Git Upstream + """ + def __init__(self, repo, branch, last_commit): + self.repo = repo + self.branch = branch + self.last_commit = last_commit + + def get_patch_list(self): + """ + Abstract method + """ + pass + + +class GitHub(GitUpstream): + """ + GitHub type + """ + def __init__(self, repo, branch, last_commit, token): + super().__init__(repo, branch, last_commit) + self.token = token + self.github_api = GitHubApi() + + def get_latest_commit_id(self): + """ + get latest commit id + """ + status, result = self.github_api.get_latest_commit(self.repo, self.branch) + logger.debug('repo: %s branch: %s. get_latest_commit: %s %s', self.repo, self.branch, status, result) + + return status, result + + def get_commit_info(self, commit_id): + """ + get commit info + """ + status, result = self.github_api.get_commit_info(self.repo, commit_id) + logger.debug('get_commit_info: %s %s', status, result) + + return status, result + + def get_patch_list(self): + """ + get patch list + """ + commit_list = list() + status, result = self.get_latest_commit_id() + if status != 'success': + logger.error( + '[Patch Tracking] Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s', + self.repo, self.branch, result + ) + return None + latest_commit = result['latest_commit'] + while self.last_commit != latest_commit: + status, result = self.get_commit_info(latest_commit) + logger.debug('get_commit_info: %s %s', status, result) + if status == 'success': + if 'parent' in result: + ret = self.github_api.get_patch(self.repo, latest_commit, latest_commit) + logger.debug('get patch api ret: %s', ret) + if ret['status'] == 'success': + result['patch_content'] = ret['api_ret'] + # inverted insert commit_list + commit_list.insert(0, result) + else: + logger.error( + 'Get scm: %s commit: %s patch failed. Result: %s', self.repo, latest_commit, result + ) + latest_commit = result['parent'] + else: + logger.info( + '[Patch Tracking] Successful get scm commit from %s to %s ID/message/time/patch.', + self.last_commit, latest_commit + ) + break + else: + logger.error( + '[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', self.repo, + latest_commit, result + ) + + return commit_list + + +class Git(GitUpstream): + """ + Git type + """ + def __init__(self, repo, branch, last_commit, base_path): + super().__init__(repo, branch, last_commit) + self.base_path = base_path + + # 创建错误处理handle, 解决read only文件 + @staticmethod + def handle_remove_read_only(func, path, exc): + """ + handle remove read only + """ + excvalue = exc[1] + if func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES: + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 + func(path) + else: + logger.debug('os chmod failed.') + + def git_clone(self, repo_name): + """ + git clone + """ + repo_name = get_repo_name(repo_name) + repo_path_exists = os.path.exists(os.path.join(self.base_path, repo_name)) + if repo_path_exists: + if os.path.exists(os.path.join(self.base_path, repo_name, ".git")): + repo = git.Repo(os.path.join(self.base_path, repo_name)) + remote = repo.remote() + remote.pull() + else: + shutil.rmtree(os.path.join(self.base_path, repo_name), onerror=self.handle_remove_read_only) + os.makedirs(os.path.join(self.base_path, repo_name)) + git.Repo.clone_from(url=self.repo, to_path=os.path.join(self.base_path, repo_name), branch=self.branch) + else: + os.makedirs(os.path.join(self.base_path, repo_name)) + git.Repo.clone_from(url=self.repo, to_path=os.path.join(self.base_path, repo_name), branch=self.branch) + + def get_latest_commit_id(self, repo_name): + """ + get latest commit id + """ + repo_name = get_repo_name(repo_name) + repo = os.path.join(self.base_path, repo_name) + commits = RepositoryMining(repo, only_in_branch=self.branch).traverse_commits() + return list(commits)[-1].hash + + def get_commit_id_list(self, repo_name): + """ + get commit id list + """ + repo_name = get_repo_name(repo_name) + repo = os.path.join(self.base_path, repo_name) + latest_commit_id = self.get_latest_commit_id(repo_name) + commit_list = list() + if self.last_commit and self.last_commit != latest_commit_id: + commits = RepositoryMining( + repo, only_in_branch=self.branch, from_commit=self.last_commit, to_commit=latest_commit_id + ).traverse_commits() + for commit in commits: + commit_list.append(commit.hash) + commit_list.remove(self.last_commit) + return commit_list + + def get_commit_info(self, commit_id, repo_name): + """ + get commit info + """ + repo_name = get_repo_name(repo_name) + repo = os.path.join(self.base_path, repo_name) + + parent_commit = git.Repo(repo).commit(commit_id).parents[0] + message = git.Repo(repo).commit(commit_id).message + patch_content = git.Repo(repo).git.diff(parent_commit, commit_id) + + return commit_id, message, patch_content + + def get_patch_list(self): + """ + get patch list + """ + repo_name = get_repo_name(self.repo) + patch_list = list() + commit_list = self.get_commit_id_list(repo_name) + for commit in commit_list: + patch_dict = dict() + patch_dict['commit_id'] = commit + patch_dict['time'], patch_dict['message'], \ + patch_dict['patch_content'] = self.get_commit_info(commit, repo_name) + patch_list.append(patch_dict) + + return patch_list + + +class GitFactory: + """ + Git Factory + """ + @staticmethod + def get_git(version_control, repo, branch, last_commit, param): + """ + git type + """ + if version_control == 'github': + return GitHub(repo, branch, last_commit, param) + if version_control == 'git': + return Git(repo, branch, last_commit, param) + return None