From be0a81eeb28b1f3874c10d1d37c6f24e2fe705b4 Mon Sep 17 00:00:00 2001 From: wb-msm261421 Date: Thu, 29 Feb 2024 15:31:14 +0800 Subject: [PATCH 1/2] fix(sysom-hotfix): Release hotfix kernel version bulk import --- .../sysom_hotfix/apps/hotfix/serializer.py | 163 +++++++++++++++--- 1 file changed, 140 insertions(+), 23 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py index 72bbbc4d..5f36fb95 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py @@ -1,12 +1,16 @@ import os +import re import tempfile import pandas as pd +from typing import List from datetime import datetime from pandas.core.frame import DataFrame +from pandas._libs.tslibs.timestamps import Timestamp from clogger import logger from django.utils import timezone from django.core.files.uploadedfile import InMemoryUploadedFile from rest_framework import serializers +from rest_framework.fields import empty from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel, ReleasedHotfixListModule class HotfixSerializer(serializers.ModelSerializer): @@ -139,7 +143,7 @@ class BulkImportHotfixReleasedSerializer(serializers.Serializer): self._save_file_path = None self._default_save_path = '/tmp/' self._suffixs = ['xls', 'csv', 'xlsx'] - self._file_header_fields = ["hotfix_id", "modified_time", "kernel_version", "download_link"] + self._file_header_fields = ["hotfix_id", "released_time", "released_kernel_version"] self._action = { 'csv': pd.read_csv, 'xls': pd.read_excel, @@ -173,7 +177,7 @@ class BulkImportHotfixReleasedSerializer(serializers.Serializer): for chunk in self._file.chunks(chunk_size=1024): f.write(chunk) - def _file_parse(self) -> [DataFrame]: + def _file_parse(self) -> DataFrame: """parse the file""" if self._suffix is None: raise serializers.ValidationError("file suffix is not none!") @@ -186,35 +190,147 @@ class BulkImportHotfixReleasedSerializer(serializers.Serializer): ) return context + def _kernel_version_or_download_link_map( + self, + kernel_versions: List[str], + urls: List[str] + ) -> List[dict]: + ''' + @Description: Find the corresponding download link in the + download links by using the key characters in the kernel- + version + @Params `kernel_version`: Type(List) Store the kernel ve- + rsion list + @Params `urls`: Type(List) Store the donwload_link list + @return {*} + ''' + _version_download_link_list = list() + reg_compile = re.compile( + "(\d+)\.(\d+)\.(\d+)-(\d+)\.?(\d+){0,1}(?!_)" + ) + + def _filter(marks: List[str], url: str): + ''' + @Description: + @param {List} marks + @param {str} url + @return {*} + ''' + flag = True + for mark in marks: + if mark not in url: + flag = False + break + return flag + + def _filter_subset(filter_sub : List, kernel_version_flag): + """ + filter out the exact downloadlink + @ filter_flag: ['http:', '', 'yum.tbsite.net', 'alios', '7u2', 'os', 'aarch64', ※'kernel-hotfix-CVE-2023-0461-5.10.134-13' ※, + 'kernel-hotfix-CVE-2023-0461-5.10.134-13-1.0-20230316152330.al8.aarch64.rpm'] + """ + for kernel_link in filter_sub: + filter_flag = kernel_link.split("/")[7] + filter_flag_fin = re.search(reg_compile, filter_flag).group(0) + if kernel_version_flag == filter_flag_fin: + return kernel_link + + for kernel_version in kernel_versions: + v = re.search(reg_compile, kernel_version).groups() + version_s = re.sub(reg_compile, "", kernel_version) + marks = version_s.split(".")[1:] + marks.append(v) + marks = re.findall(r"([\d\-\.]+)\.((\w+)\.)+(\w+)", kernel_version)[0] # tuple + # [5.10.23-5 , al8 , x86_64] + filter_sub = list(filter(lambda x: _filter(marks, x), urls)) + if len(filter_sub) >= 1: + subset_result = filter_sub[0] if len(filter_sub) == 1 else _filter_subset(filter_sub, marks[0]) #!!! + _version_download_link_list.append((kernel_version, subset_result)) + return _version_download_link_list + def create(self, validated_data): """Save the data to the database""" - - def _structure_released_hotfix(data_rows) -> list[ReleasedHotfixListModule]: + def _released_time(released_time): + """ + @Description: released time handler + @param {*} released_time: `released_time` type is string or DateTime or empty string + @return {*} released_time: type(datetime) + """ + if isinstance(released_time, str) and released_time != "": + released_time = datetime.strptime( + released_time.split(".")[0], "%Y-%m-%d %H:%M:%S" + ) + released_time = timezone.get_current_timezone().localize(released_time) + elif isinstance(released_time, Timestamp): + released_time = timezone.get_current_timezone().localize(released_time) + else: + released_time = timezone.now() + return released_time + + def _structure_released_hotfix(**data_rows: dict) -> List[ReleasedHotfixListModule]: """ initizaer released hotfix model """ - released_hotfix_list = [] - for kernel_version in data_rows['kernel_version'].split(","): - kwargs = dict() - kwargs["hotfix_id"] = data_rows.get("hotfix_id") - kwargs["download_link"] = data_rows.get("download_link", "") - kwargs["released_kernel_version"] = kernel_version - - released_time = data_rows['modified_time'] - if isinstance(released_time, str): - released_time = datetime.strptime( - released_time.split(".")[0], "%Y-%m-%d %H:%M:%S" - ) - kwargs['released_time'] = timezone.get_current_timezone().localize(released_time) - + def _release_kernel_version_model(hotfix_id, released_kernel_version, released_time, **kwargs): try: - ReleasedHotfixListModule.objects.get(**kwargs) + ReleasedHotfixListModule.objects.get( + hotfix_id=hotfix_id, released_kernel_version=released_kernel_version) + return None except ReleasedHotfixListModule.DoesNotExist: - released_hotfix_list.append(ReleasedHotfixListModule(**kwargs)) + return ReleasedHotfixListModule( + hotfix_id=hotfix_id, released_kernel_version=released_kernel_version, + released_time=released_time, **kwargs + ) + + released_hotfix_list = list() + hotfix_id = data_rows.pop("hotfix_id") + released_time = _released_time(data_rows.pop("released_time")) + released_kernel_version: str = data_rows.pop("released_kernel_version") + + model_fields_list = [ + field.name + for field in ReleasedHotfixListModule._meta.get_fields() + ] + + _other_kwargs = { + k: v + for k, v in data_rows.items() + if k in model_fields_list\ + and ( + v is not None + and + v != "" + ) + } + + kernel_versions = released_kernel_version.split(",") + if len(kernel_versions) == 1: + _model = _release_kernel_version_model( + hotfix_id, released_kernel_version, released_time, **_other_kwargs + ) + released_hotfix_list.append(_model) if _model is not None else ... + else: + download_link: str = _other_kwargs.get("download_link") + download_links = download_link.split(" ") + if download_link is None or len(download_links) == 1: + for kernel_version in kernel_versions: + _model = _release_kernel_version_model( + hotfix_id, kernel_version, released_time, **_other_kwargs + ) + released_hotfix_list.append(_model) if _model is not None else ... + else: + for item in self._kernel_version_or_download_link_map( + kernel_versions, download_links + ): + kernel_version, download_link = item + _other_kwargs.update({"download_link": download_link}) + _model = _release_kernel_version_model( + hotfix_id, kernel_version, released_time, **_other_kwargs + ) + released_hotfix_list.append(_model) if _model is not None else ... return released_hotfix_list excel_file_content: DataFrame = validated_data.get("file") - for param in [ field for field in self._file_header_fields\ if field not in excel_file_content.columns.values @@ -224,8 +340,9 @@ class BulkImportHotfixReleasedSerializer(serializers.Serializer): ) for data in excel_file_content.to_dict(orient='records'): - ReleasedHotfixListModule.objects.bulk_create(_structure_released_hotfix(data)) - + ReleasedHotfixListModule.objects.bulk_create( + _structure_released_hotfix(**data) + ) return [] def close_file(self): -- Gitee From 7789b452c641137fe9329f8bb644a8d1000df4bf Mon Sep 17 00:00:00 2001 From: wb-msm261421 Date: Thu, 29 Feb 2024 15:34:06 +0800 Subject: [PATCH 2/2] fix(sysom-hotfix): Fix xlrd version lower --- script/server/sysom_hotfix/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/server/sysom_hotfix/requirements.txt b/script/server/sysom_hotfix/requirements.txt index 433c06ce..aa9175c2 100644 --- a/script/server/sysom_hotfix/requirements.txt +++ b/script/server/sysom_hotfix/requirements.txt @@ -18,6 +18,6 @@ pandas>=1.1.5 requests==2.27.1 gunicorn==20.1.0 xlwt==1.3.0 -xlrd==2.0.1 +xlrd==1.2.0 beautifulsoup4==4.12.2 openpyxl==3.1.2 \ No newline at end of file -- Gitee