diff --git a/app.properties b/app.properties deleted file mode 100644 index ac82217cd542fb3157bbe8829d14f30d92fb35d7..0000000000000000000000000000000000000000 --- a/app.properties +++ /dev/null @@ -1,42 +0,0 @@ -# sanic -host = 0.0.0.0 -port = 8005 -debug = True -access_log = True -workers = 3 -sanic_key = test-library -auto_reload = False -jwt_secret_key = test-lib-jwt-secret-key -main_domain = http://test-lib:8005/ - -# log -log_mode = False -log_bytes = 20*1024*1024 -log_backup = 5 -formatter = %(asctime)s [%(process)d] [%(levelname)s] %(message)s -formatter_access = %(asctime)s [%(process)d] [%(levelname)s] [%(host)s]: %(request)s %(message)s %(status)d %(byte)d - -# daily -db_url = mysql+aiomysql://root:tonedbadmin@121.196.236.71:3306/test-lib -drop_all = False -create_all = False -pool_size = 10 -over_size = 10 -recycle = 3600 - -# redis -redis_url = redis://:toneredisadmin@121.196.236.71:6379/10 - -# tone prod -tone_host = http://tone:7001/ -tone_token = xxxx -tone_user_name = xxxxx - -# font -oss_url=http://0.0.0.0:8005 - -# tone-storage -storage_host = 127.0.0.1 -storage_sftp_port = 22 -storage_user = tonestorage -storage_password = 123456 diff --git a/app/__init__.py b/app/__init__.py index dbc9c41a5d6abeac2aa0d85cd92d66aaf1ebad07..795a21fa74b6bc2ad6ec031ab9b7b08145824224 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -3,7 +3,6 @@ import os.path from app.conf import conf from app.db import db from app.log import log -from app.oss import alg_oss from app.redis import redis from app.setting import STATIC_ROOT, STATICFILES_DIRS, OUTLINE_PATH diff --git a/app/conf.py b/app/conf.py index db8644ffa93fdef1ce3f93a61dbfd68a9eb8e7b9..b7d3b73f5a58cca6f51ab62b5510a20dd2a9eb8d 100644 --- a/app/conf.py +++ b/app/conf.py @@ -11,7 +11,10 @@ class SanicConf(object): def init_app(self, app, env=None, group=None): self.config = None - data = self.get_conf('app.properties') + conf_filename = 'app.properties' + data = dict() + if os.path.exists(conf_filename): + data = self.get_conf(conf_filename) self.update_environ(data, os.environ) app.config.update(data) self.config = app.config diff --git a/app/oss.py b/app/oss.py deleted file mode 100644 index f8052efd6a6824472932b42051a696f17747dbb7..0000000000000000000000000000000000000000 --- a/app/oss.py +++ /dev/null @@ -1,77 +0,0 @@ -import oss2 -from oss2.exceptions import NoSuchKey - - -class OSSClient(object): - LINK_VALID_TIME = 7200 - READABLE_FILE_LIST = ['log', 'json', 'txt', 'text', 'sh', 'yaml', 'time', 'output', 'task', 'fs', 'cvs'] - - def __init__(self): - pass - - def init_app(self, app): - self.access_id = None - self.access_key = None - self.endpoint = None - self.bucket_name = None - - @app.listener('before_server_start') - async def redis_conn(app, loop): - self.access_id = app.config['ALG_OSS_ACCESS_KEY'] - self.access_key = app.config['ALG_OSS_ACCESS_SECRET'] - self.bucket_name = app.config['ALG_OSS_BUCKET'] - self.endpoint = app.config['ALG_OSS_ENDPOINT'] - auth = oss2.Auth(self.access_id, self.access_key) - self.bucket = oss2.Bucket(auth, self.endpoint, self.bucket_name) - - def file_exists(self, file_name): - return self.bucket.object_exists(file_name) - - def put_file(self, file_name): - with open(file_name, 'rb') as file_handle: - return self.bucket.put_object(file_name, file_handle) - - def put_object_from_file(self, object_name, local_file): - return self.bucket.put_object_from_file(object_name, local_file) - - def put_object_from_bytes(self, object_name, file_bytes): - return self.bucket.put_object(object_name, file_bytes) - - def get_object(self, object_name, params=None, headers=None): - try: - return self.bucket.get_object(object_name, params=params, headers=headers) - except NoSuchKey: - return None - - def get_object_to_file(self, object_name, local_file): - try: - return self.bucket.get_object_to_file(object_name, local_file) - except NoSuchKey: - return None - - def get_sign_url(self, object_name, expires=7200): - object_name = object_name.strip('/') - if self.bucket.object_exists(object_name): - sign_url = self.bucket.sign_url('GET', object_name, expires) - if isinstance(sign_url, str): - sign_url = sign_url.replace('http://', 'https://') - sign_url = sign_url.replace('%2F', '/') - return sign_url - else: - raise RuntimeError('object:%s not exist in oss bucket:%s' % (object_name, self.bucket)) - - def update_file_content_type(self, object_name, content_type='text/plain'): - object_name = object_name.strip('/') - result = self.bucket.update_object_meta(object_name, {'Content-Type': content_type}) - return result.status == 200 - - def get_file_content_type(self, file): - file_headers = self.bucket.head_object(file) - return file_headers.headers.get('Content-Type') - - def remove_object(self, objects, params=None, headers=None): - result = self.bucket.delete_object(objects, params, headers) - return result.status == 204 - - -alg_oss = OSSClient() diff --git a/common/tone/tone_request.py b/common/tone/tone_request.py index e17d1f9a6e5ddea090379ee16e28de5f83ad441f..7bf394f87706fb4343e540d6c2b9be48c938b13e 100644 --- a/common/tone/tone_request.py +++ b/common/tone/tone_request.py @@ -9,7 +9,7 @@ from common.http import http_request async def get_res_from_tone(req_type, req_api, req_params): tone_url = conf.config['TONE_HOST'] + req_api - token = conf.config['TONE_USER_NAME'] + '|' + conf.config['TONE_TOKEN'] + '|' + str(time.time()) + token = conf.config['TONE_USER_NAME'] + '|' + str(conf.config['TONE_TOKEN']) + '|' + str(time.time()) signature = base64.b64encode(token.encode('utf-8')).decode('utf-8') req_data = { 'username': conf.config['TONE_USER_NAME'], diff --git a/common/tools.py b/common/tools.py index a861a770b9e3d99adf8a4248308316365df501d9..58109081e19644c4a267b543773223b8987fd753 100644 --- a/common/tools.py +++ b/common/tools.py @@ -30,6 +30,10 @@ def string_toDatetime(st): return datetime.datetime.strptime(st, "%Y-%m-%d %H:%M:%S") +def string_toShortDatetime(st): + return datetime.datetime.strptime(st, "%Y-%m-%d") + + def string_toTimestamp(st): return time.mktime(time.strptime(st, "%Y-%m-%d %H:%M:%S")) diff --git a/common/utils/sftp_client.py b/common/utils/sftp_client.py new file mode 100644 index 0000000000000000000000000000000000000000..50ad67e3347187bb0714afbfd6104676957ec5b1 --- /dev/null +++ b/common/utils/sftp_client.py @@ -0,0 +1,48 @@ +import paramiko +import os +import uuid +from app.conf import conf +from app.log import log + + +async def upload_static_file(uploaded_file): + # SSH连接信息 + host = conf.config['TONE_STORAGE_HOST'] + domain = conf.config['TONE_STORAGE_DOMAIN'] + port = conf.config['TONE_STORAGE_SFTP_PORT'] + proxy_port = conf.config['TONE_STORAGE_PROXY_PORT'] + username = conf.config['TONE_STORAGE_USER'] + password = conf.config['TONE_STORAGE_PASSWORD'] + try: + # 生成一个随机且唯一的文件名 + file_uuid = str(uuid.uuid4()) + file_extension = os.path.splitext(uploaded_file.name)[1] # 获取文件扩展名 + random_filename = f"{file_uuid}{file_extension}" + temp_file_path = '/tmp/' + random_filename # 指定临时保存路径（包含随机文件名） + remote_path = '/testlib/' + random_filename + # 保存上传的文件到本地临时位置 + with open(temp_file_path, 'wb') as f: + f.write(uploaded_file.body) + # 创建SSH客户端 + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # 连接SFTP服务器 + ssh.connect(host, port, username=username, password=password) + # 创建SFTP对象 + sftp = ssh.open_sftp() + # 检查目录是否存在 + try: + sftp.stat('/testlib') + except FileNotFoundError: + sftp.mkdir('/testlib', mode=0o755) # 创建目录 + # 将临时文件上传到SFTP服务器 + sftp.put(temp_file_path, remote_path) + # 删除本地临时文件（可选） + os.remove(temp_file_path) + # 关闭SFTP和SSH连接 + sftp.close() + ssh.close() + return 'http://' + domain + ':' + proxy_port + remote_path + except Exception as e: + log.logger.error(f'sftp upload file failed! error:{e}') + return '' diff --git a/dist/index.html b/dist/index.html index 9e8888eeb971909f745ebb2ee6545f6412e2280f..0553e18cd30df119778aab877ee29905b7397602 100644 --- a/dist/index.html +++ b/dist/index.html @@ -8,10 +8,10 @@ - +
- + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ed6a9a60e64c88ed967fd6b0ce8800121bd84366..7f27a71e64a62da36ebea0e85044619c01b2754f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ openpyxl~=3.0.9 oss2~=2.15.0 PyJWT~=2.3.0 pyyaml -httplib2 \ No newline at end of file +httplib2 +paramiko \ No newline at end of file diff --git a/services/case_service.py b/services/case_service.py index 94e09d4d19982b6c036171fb9422f23508215a68..84f5505fa0d99e3382439b162e01530502df55eb 100644 --- a/services/case_service.py +++ b/services/case_service.py @@ -7,7 +7,7 @@ from sanic.log import logger from sqlalchemy import or_ from common.enums import User_Role -from common.tools import string_toDatetime +from common.tools import string_toShortDatetime from common.utils.excel import write_excel, read_excel from models.case_model import * from services.const import ERROR_UN_EXISTED_CASE, ERROR_UN_EXISTED_TS, ERROR_DUPLICATED_NAME, ERROR_LACK_PERMISSION @@ -355,13 +355,16 @@ async def __search_child_nodes_by_path(parent_path): return [result.id for result in results] -async def import_yaml(file_body, file_name): - yaml_info = yaml.load(file_body, Loader=yaml.FullLoader) - result, ok = await yaml_save_to_case(yaml_info, file_name, 0) +async def import_yaml(file_body, file_name, parent): + try: + yaml_info = yaml.load(file_body, Loader=yaml.FullLoader) + except Exception as ex: + return ERROR_INVALID_YAML, False + result, ok = await yaml_save_to_case(yaml_info, file_name, parent) return result, ok -async def import_tar(file_body, file_name): +async def import_tar(file_body, file_name, parent): file_name = 'common/dist/' + file_name open(file_name, 'wb').write(file_body) tar_file = tarfile.open(file_name, 'r') @@ -370,7 +373,6 @@ async def import_tar(file_body, file_name): if file_info.name.count('/') > 0: folder = file_info.name.split('/') level = file_info.name.count('/') - parent = 0 path = '/' for dir_name in folder: if dir_name.count('.yaml') > 0: @@ -394,7 +396,7 @@ async def import_tar(file_body, file_name): level -= 1 else: yaml_info = yaml.load(tar_file.extractfile(file_info.name).read(), Loader=yaml.FullLoader) - await yaml_save_to_case(yaml_info, file_info.name, 0) + await yaml_save_to_case(yaml_info, file_info.name, parent) return None, True @@ -421,7 +423,7 @@ async def yaml_save_to_case(yaml_info, file_name, parent): case['suite_name'] = yaml_info.get('测试套', 'default') case['labels'] = yaml_info['通用标签'] case['desc'] = yaml_info['用例描述'] - case['pre_condition'] = yaml_info['前置条件'] + case['pre_condition'] = str(yaml_info['前置条件']) steps = list() step_index = 1 for step in yaml_info['测试步骤']: @@ -505,12 +507,12 @@ async def export_excel(case_ids): content['执行模式'].append(case.run_model.value) content['是否可用'].append(case.is_available) content['关联的tone用例'].append(case.tone_case) - content['设备类型'].append(case.device_type.value) - content['设备架构'].append(case.device_arch.value) + content['设备类型'].append(case.device_type) + content['设备架构'].append(case.device_arch) content['所属模块'].append(case_nodes_dict[case.parent]) content['创建时间'].append(case.gmt_created) content['改动时间'].append(case.gmt_modified) - content['用例描述'].append(case.base_fields) + content['用例描述'].append(case.desc) content['扩展'].append(case.custom_fields) return await write_excel(content) @@ -529,17 +531,19 @@ async def search_case(data): node_list = await __search_child_nodes_by_path(node.path) conditions.append(Case.parent.in_(node_list)) if 'labels' in data: - case_ids = await get_cases_by_label_names(data['labels']) + case_ids = await get_cases_by_label_names(data.get('labels').split(',')) conditions.append(Case.id.in_(case_ids)) if 'is_available' in data: result = True if data.get('is_available') == 'true' else False conditions.append(Case.is_available == result) if 'priority' in data: pr_list = list() - for pr in data['priority']: + for pr in data.get('priority').split(','): if __get_priority(pr): pr_list.append(__get_priority(pr)) conditions.append(Case.priority.in_(pr_list)) + if 'type' in data: + conditions.append(Case.type == data.get('type')) if 'test_type' in data: conditions.append(Case.type == data.get('test_type')) if 'run_method' in data: @@ -547,27 +551,27 @@ async def search_case(data): if 'run_model' in data: conditions.append(Case.run_model == data.get('run_model')) if 'suite_name' in data: - conditions.append(Case.suite_name.in_(data.get('suite_name'))) + conditions.append(Case.suite_name.in_(data.get('suite_name').split(','))) if 'creator' in data: - conditions.append(Case.creator.in_(data.get('creator'))) + conditions.append(Case.creator.in_(data.get('creator').split(','))) start_time = data.get('start_time', None) end_time = data.get('end_time', None) if start_time: - start_time = string_toDatetime(start_time) + start_time = string_toShortDatetime(start_time) conditions.append(Case.gmt_created >= start_time) if end_time: - end_time = string_toDatetime(end_time) + end_time = string_toShortDatetime(end_time) conditions.append(Case.gmt_modified <= end_time) - if 'device_type' in data and Device_Type_EN.UNLIMIT.value not in data['device_type']: + if 'device_type' in data and Device_Type_EN.UNLIMIT.value not in data.get('device_type').split(','): device_type_list = list() device_type_list.append(Case.device_type == Device_Type_EN.UNLIMIT.value) - for item in data['device_type']: + for item in data.get('device_type').split(','): device_type_list.append(Case.device_type.contains(item)) multi_condition.append(or_(*device_type_list)) - if 'device_arch' in data and Device_Arch.NOARCH.value not in data['device_arch']: + if 'device_arch' in data and Device_Arch.NOARCH.value not in data.get('device_arch').split(','): device_type_list = list() device_type_list.append(Case.device_type == Device_Type_EN.UNLIMIT.value) - for item in data['device_type']: + for item in data.get('device_arch').split(','): device_type_list.append(Case.device_type.contains(item)) multi_condition.append(or_(*device_type_list)) if 'key' in data: @@ -578,7 +582,7 @@ async def search_case(data): if conditions: result = await Case.query_obj_all_by_fields(fields, None, *conditions) else: - result = await Case.query_obj_all_by_fields(fields) + result = await Case.query_obj_all_by_fields(fields, desc=Case.id) return {'path': path, 'level': level, 'data': result} @@ -593,7 +597,7 @@ def __get_priority(pr): async def excel_template(): - filepath = 'common/dist/temp.xlsx' + filepath = 'common/static/temp.xlsx' with open(filepath, "rb") as f: content = f.read() return content @@ -605,7 +609,7 @@ async def get_test_suites(is_paginate=True, name=None, page_num=1, page_size=20) match = None if name: match = {'name': f'{name}%'} - return await TestSuite.query_page(page_num, page_size, match=match) + return await TestSuite.query_page(page_num, page_size, match=match, desc=TestSuite.id) async def create_test_suite(data, name): diff --git a/services/common_service.py b/services/common_service.py index 6e7707a42d3cb5cebdf90e433f843c75aac1b927..2978ba1fb60d3d425e4a7cdf21642aa2098c0646 100644 --- a/services/common_service.py +++ b/services/common_service.py @@ -1,9 +1,7 @@ -import re from uuid import uuid4 from app import redis from app.conf import conf -from app.oss import alg_oss from common.token import common_sign @@ -36,10 +34,3 @@ def get_content_type_key(content_type): if content_type == 'image': return 'image-{}'.format(uuid4()) return 'others-{}'.format(uuid4()) - - -def upload_static_file_to_oss(filename, file_bytes): - alg_oss.put_object_from_bytes(filename, file_bytes) - oss_link = alg_oss.get_sign_url(filename) - oss_link = re.sub(r'&Expires=.*&', '&', oss_link) - return oss_link diff --git a/services/requirement_service.py b/services/requirement_service.py index 771749060f8d09dd1b5a0a18e1baee118222ee34..9729f4a9b281dab09027a297363333c7f30a4d94 100644 --- a/services/requirement_service.py +++ b/services/requirement_service.py @@ -11,7 +11,7 @@ REQUIREMENT_KEY = 'requirement_dict' async def get_requirements(page_num=1, page_size=10): - return await Requirement.query_page(page_num=page_num, page_size=page_size) + return await Requirement.query_page(page_num=page_num, page_size=page_size, desc=Requirement.id) async def query_requirements(prefix): diff --git a/services/task_service.py b/services/task_service.py index 700750b39a777b14d0bde1df39e29a12c60ec147..931da56a069adc69d8bcaf01c49cbdf8f168d783 100644 --- a/services/task_service.py +++ b/services/task_service.py @@ -141,7 +141,7 @@ async def modify_task(data, task_id, user): task.config = data['config'] if 'cases' in data: if data['cases']: - case_infos = await __gen_run_result_case_infos(data['cases'].split(',')) + case_infos = await __gen_run_result_case_infos(data['cases']) task.run_result['case_infos'] = case_infos else: task.run_result['case_infos'] = [] diff --git a/views/case_view.py b/views/case_view.py index e8047457c73489272c86b5000c152335ee55e0c1..30b39795ea2c840aff23eef2a4bda293521f7635 100644 --- a/views/case_view.py +++ b/views/case_view.py @@ -104,14 +104,15 @@ async def batch_move(request, user_infos): async def import_cases(request, user_infos): file = request.files.get('file') file_type = os.path.splitext(file.name) + parent = request.form.get('parent', None) result = '' ok = True if file_type[1] in ['.xls', '.xlsx']: await import_excel(file.body) elif file_type[1] in ['.yaml', '.yml']: - result, ok = await import_yaml(file.body, file.name) + result, ok = await import_yaml(file.body, file.name, parent) elif file_type[1] in ['.tar']: - result, ok = await import_tar(file.body, file.name) + result, ok = await import_tar(file.body, file.name, parent) else: return rsp(code=400, msg=ERROR_OUTLINE_FORMAT) if not ok: diff --git a/views/common_view.py b/views/common_view.py index 8937ecc91741b657faf1aec79dfc9d3ad902d630..40822121dabd630e13e86250d6e686c3b8f1cc23 100644 --- a/views/common_view.py +++ b/views/common_view.py @@ -4,7 +4,7 @@ from sanic.exceptions import NotFound from app.conf import conf from common.http import rsp, http_request from services.auth_service import login_auth -from services.common_service import upload_static_file_to_oss +from common.utils.sftp_client import upload_static_file bp = Blueprint('common', url_prefix='') @@ -13,7 +13,7 @@ bp = Blueprint('common', url_prefix='') @login_auth async def upload_images(request, user_infos): file = request.files.get('image') - result = upload_static_file_to_oss(file.name, file.body) + result = await upload_static_file(file) return rsp(data=result) diff --git a/views/requirement_view.py b/views/requirement_view.py index dfccf8554f93502c676b660d40c800f14ce8261e..3f3e3ccc9cb0055735d70f918d6744e637f2cd2c 100644 --- a/views/requirement_view.py +++ b/views/requirement_view.py @@ -13,7 +13,7 @@ bp = Blueprint('requirement', url_prefix='api/requirement') @login_auth async def get_requirement_list(request, user_infos): page_num = int(request.args.get('page_num')) if request.args.get('page_num') else 1 - page_size = int(request.args.get('page_size')) if request.args.get('page_size') else 10 + page_size = int(request.args.get('page_size')) if request.args.get('page_size') else 100 result = await get_requirements(page_num, page_size) return rsp(paginate=result)