diff --git a/baseline_tools/README.md b/baseline_tools/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3bcb3fd56006564095547b438cba41004e2f36e0 --- /dev/null +++ b/baseline_tools/README.md @@ -0,0 +1,48 @@ +## 当前功能描述 +根据manifest.yaml文件提供的基线，生成目标分支软件包的差异表格。 + +## 原理 +根据yocto-meta-openeuler/.oebuild/manifest.yaml文件提供的基线，下载基线仓库到src_tmp（和openeuler fetch同样原理，仅下载基线代码，depth为1），生成基线和目标分支当前最新提交的关键差异（spec文件差异），差异存放于{local name}.oe.specdiff，进而汇总specdiff到输出表格output.xlsx + +## 使用方法 +1. 安装脚本工具依赖 + ``` + pip install yaml pandas openpyxl + ``` +3. 检查black.repo黑名单，在黑名单中的local name对应仓库，不会进行差异分析（当前文件包含kernel和src-kernel） + ``` + # cat black.repo + allwinner-kernel + kernel-5.10 + kernel-5.10-tag3093 + rockchip-kernel + src-allwinner-kernel + src-kernel-5.10 + src-kernel-5.10-tag3093 + src-rockchip-kernel + ``` +5. 将要分析的基线manifest.yaml文件（yocto-meta-openeuler/.oebuild/manifest.yaml），**拷贝到本工具目录**（oe_makediff.sh文件所在目录） + ``` + cp xxx/yocto-meta-openeuler/.oebuild/manifest.yaml ./ + ``` +6. 确认目标分支名，注意区分src-openeuler通用分支名和ROS软件包的特殊分支名，修改oe_makediff.sh文件中的target_branch变量和target_branch_ros变量。例： + ``` + target_branch="openEuler-22.03-LTS-SP4" + target_branch_ros="Multi-Version_ros-humble_openEuler-22.03-LTS-SP2" + ``` + 注意，还有一个变量force_makediff，对应值配置功能如下： + ``` + force_makediff="true" #用于已经执行过一遍脚本（下载好全部仓库），可加速增量更新。 + force_makediff="false" #用于断点继续下载分析仓库，建议作为默认配置。 + ``` +7. **在本工具目录**，执行oe_makediff.sh + ``` + ./oe_makediff.sh + ``` + **脚本执行了什么？**——该脚本将会下载基线仓库到src_tmp并输出目标差异，分析目标分支时，将先尝试分析target_branch，如果目标分支不存在，对比target_branch_ros，如果存在target_branch_ros，认为是ros包，否则标记为目标分支。 + **脚本支持断点继续执行**——原理：下载和分析中的仓库，会获取/创建filelock.{baselinecommit}文件锁，仓库完成分析会释放/删除该文件锁，重新执行时，对于已经有目录但没有锁的仓库，会跳过，有文件锁的仓库说明未分析完，将删除该仓库重新分析。 + +8. **在本工具目录**，执行python3脚本make_xls.py，汇总diff数据生成output.xlsx表格 + ``` + python3 make_xls.py + ``` diff --git a/baseline_tools/black.repo b/baseline_tools/black.repo new file mode 100644 index 0000000000000000000000000000000000000000..ee82b4d49531d4ecddf27e8dbc652c100e22a54d --- /dev/null +++ b/baseline_tools/black.repo @@ -0,0 +1,8 @@ +allwinner-kernel +kernel-5.10 +kernel-5.10-tag3093 +rockchip-kernel +src-allwinner-kernel +src-kernel-5.10 +src-kernel-5.10-tag3093 +src-rockchip-kernel diff --git a/baseline_tools/format_info.py b/baseline_tools/format_info.py new file mode 100644 index 0000000000000000000000000000000000000000..178747d96d67c614dc295cac95b82f40b69f7b36 --- /dev/null +++ b/baseline_tools/format_info.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# coding=utf-8 +import yaml + +# 加载 YAML 文件 +with open('manifest.yaml', 'r') as file: + manifest_data = yaml.safe_load(file) + +for reponame, repo_info in manifest_data['manifest_list'].items(): + remote_url = repo_info['remote_url'] + version = repo_info['version'] + print(f"{reponame}#{remote_url}#{version}") diff --git a/baseline_tools/make_xls.py b/baseline_tools/make_xls.py new file mode 100644 index 0000000000000000000000000000000000000000..ba2f403e6e61ccbb5ec77561a8656a202b515b8e --- /dev/null +++ b/baseline_tools/make_xls.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# coding=utf-8 +import os +import pandas as pd + +# 设置源目录和目标Excel文件名 +src_dir = 'src_tmp' +output_excel = 'output.xlsx' + +# 创建一个空的DataFrame来存储数据 +data = [] + +# 遍历src_tmp目录下的所有子目录 +for subdir, dirs, files in os.walk(src_dir): + for file in files: + # 检查文件名是否符合[文件夹名].oe.specdiff的模式 + if file.endswith('.oe.specdiff') and file.startswith(os.path.basename(subdir) + '.'): + # 读取文件内容 + with open(os.path.join(subdir, file), 'r', encoding='utf-8') as f: + lines = f.readlines() + + # 检查文件是否有足够的行 + if len(lines) < 3: + print(f"{subdir} 已跳过，specdiff文件格式异常") + continue # 跳过行数不足的文件 + + # 提取所需信息 + folder_name = os.path.basename(subdir) + target_branch_name = lines[0].strip() # 第一行：目标分支名 + target_branch_commit = lines[1].strip() # 第二行：目标分支commit号 + diff_analysis = ''.join(lines[2:]).strip() # 第三行及其之后：差异分析 + + # 将数据添加到列表中 + data.append((folder_name, target_branch_name, target_branch_commit, diff_analysis)) + +# 将数据转换为DataFrame +df = pd.DataFrame(data, columns=['LOCALNAME', '目标分支名', '目标分支commit号', '差异分析']) + +# 保存到Excel文件 +df.to_excel(output_excel, index=False) + +print(f"Excel文件已保存到：{output_excel}") diff --git a/baseline_tools/oe_makediff.sh b/baseline_tools/oe_makediff.sh new file mode 100755 index 0000000000000000000000000000000000000000..e5c28acc31459702b31e9059a3320c371c1338f6 --- /dev/null +++ b/baseline_tools/oe_makediff.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +target_branch="openEuler-22.03-LTS-SP4" +target_branch_ros="Multi-Version_ros-humble_openEuler-22.03-LTS-SP2" +force_makediff="true" + +black_list=`cat black.repo` +infos=`python format_info.py` +for info in $infos; +do + reponame=$(echo $info | awk -F '#' '{print $1}') + url=$(echo $info | awk -F '#' '{print $2}') + basecommit=$(echo $info | awk -F '#' '{print $3}') + + # cross black list. no need to analyse + is_black=0 + for item in $black_list; + do + if [ "$item" == "$reponame" ]; then + is_black=1 + continue + fi + done + if [ "$is_black" == "1" ]; then + continue + fi + + + if [ -e src_tmp/$reponame/filelock.$basecommit ];then + # not clean, remove it + rm -rf src_tmp/$reponame + fi + if [ "$force_makediff" == "false" ];then + if [ -d src_tmp/$reponame ];then + # already done, go cross it + continue + fi + fi + + mkdir -p src_tmp/$reponame; cd src_tmp/$reponame + touch filelock.$basecommit + git remote -vv | grep upstream | grep "$url" + if [ ! "$?" == "0" ];then + echo "---------init baseline of $url------------" + git init . ; git remote add upstream $url + git fetch upstream $basecommit --depth=1 + git checkout $basecommit + fi + git fetch upstream $target_branch --depth=1 + if [ "$?" == "0" ];then + echo "$target_branch" > ${reponame}.oe.specdiff + lastcommit=$(git rev-parse upstream/$target_branch) + echo $lastcommit >> ${reponame}.oe.specdiff + git diff $basecommit upstream/$target_branch *spec >> ${reponame}.oe.specdiff + if [ $(wc -l < ${reponame}.oe.specdiff) -eq 2 ]; then + echo "无差异" >> ${reponame}.oe.specdiff + fi + else + git fetch upstream $target_branch_ros --depth=1 + if [ "$?" == "0" ];then + echo "$target_branch_ros" > ${reponame}.oe.specdiff + lastcommit=$(git rev-parse upstream/$target_branch_ros) + echo $lastcommit >> ${reponame}.oe.specdiff + git diff $basecommit upstream/$target_branch_ros *spec >> ${reponame}.oe.specdiff + if [ $(wc -l < ${reponame}.oe.specdiff) -eq 2 ]; then + echo "无差异" >> ${reponame}.oe.specdiff + fi + else + echo "目标分支不存在" > ${reponame}.oe.specdiff + echo "目标commit未知" >> ${reponame}.oe.specdiff + echo "差异未知" >> ${reponame}.oe.specdiff + fi + fi + rm filelock.$basecommit + cd - + +done +