diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1318247143e794f95c0503b3e464ee7d36bd0276 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +test +project +__pycache__ +oebuild.egg-info +.vscode +build +dist \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..e4f1bc15085e38f616b53e64151180f7897717f8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include src/oebuild/app/conf/* +include src/oebuild/app/plugins/*/*.py \ No newline at end of file diff --git a/README.md b/README.md index cf75749844fc2380b912361c37c2d8469cb1736a..2b4342ddcec59f1980674ad4287e788c26c3a7d5 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,515 @@ -# oebuild - -#### 介绍 -oebuild is a meta tool for openEuler Embedded - -#### 软件架构 -软件架构说明 - - -#### 安装教程 - -1. xxxx -2. xxxx -3. xxxx - -#### 使用说明 - -1. xxxx -2. xxxx -3. xxxx - -#### 参与贡献 - -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request - - -#### 特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +#### 总体介绍 + +oebuild是openEuler Embedded孵化的一个开源项目，是为了辅助开发openEuler Embedded项目而衍生的辅助开发工具，是openEuler Embedded项目健康运行的一个催化剂，目前oebuild主要实现了主体框架，业务只涵盖了构建，将来会涉及到CICD，本地测试，云构建等等，oebuild的使用将不仅仅限于命令行窗口，还会搭载上层IDE来使用。yocto实现了构建的灵活性，但是做应用的定制与实现需要有较高的学习成本与环境成本，oebuild将完全摒弃这些开发羁绊，解放你的双手，只需要几个指令，即可获得你要的应用镜像，你不需要去下载代码，不需要去准备编译环境，如果你要的应用不是特有定制的，甚至不要求你去学习如何修改bb文件，一切都可以交给oebuild来做，而oebuild需要的，仅仅是一个网络而已。对，就这么简单！！！ + +#### 如何编译一个标准镜像 + +这里将介绍如何使用oebuild来进行基于openEuler Embedded项目编译标准的aarch64的qemu应用 + +##### 安装python3和pip + +oebuild由python语言编写而成，通过pip命令来完成安装，python3与pip的安装会根据运行系统的不同而不同，如果时ubuntu类系统，则通过如下命令安装： + +``` +apt-get install python3 python3-pip +``` + +如果是centos，则通过如下命令安装： + +``` +yum install python3 python3-pip +``` + +如果时suse，则通过如下命令安装： + +``` +zepper install python3 python3-pip +``` + +注1：由于版本或名称可能会有所不同，因此在各系统安装python3或pip时，请根据实际环境来安装，例如：有些系统会默认python即python3，有些系统则需要使用python-is-python3包来安装。 + +注2：如果该系统没有pip包，则可以通过离线方式来安装，预先下载pip包，然后再通过python安装，通过如下命令来完成pip的安装： + +``` +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py # 下载安装脚本 +python get-pip.py # python is python3 +``` + +##### 下载oebuild.wheel + +安装完pip后则可以通过以下命令来完成oebuild的安装 + +``` +pip install oebuild +``` + +或者点击[该地址](https://gitee.com/alichinese/oebuild-bin)进入oebuild二进制仓，在该仓下有最新的二进制包，运行以下命令来完成oebuild的安装 + +``` +pip install +``` + +oebuild会依赖一些第三方库辅助运行，因此相关的第三方库也会直接被安装，如果在安装过程中显示错误，或报第三方库安装请求失败等，请再次重新运行安装oebuild的命令即可。 + +注： 如果安装在安装oebuild时是以root用户执行，则在安装后可以直接运行，如果是以普通用户安装的，则需要注意将普通用户执行路径添加到环境变量中，可以参考以下方式来完成添加： + +1，以普通用户身份打开.bashrc + +``` +cd ~ && vim .bashrc +``` + +2, 在文件末尾查看是否将命令执行路径添加到环境变量中，该路径一般为`/home//.local/bin`,如果没有，则将以下语句添加到文件末尾 + +``` +export PATH=/home//.local/bin:$PATH +``` + +在以上命令行中，``表示是你此时的用户名，如果不确定，则可以通过执行`whoami`来确定 + +3，在添加完成后保存退出，vim编辑板的保存推出按键为：先按`esc`，再按`shift + :`, 最后输入`wq`然后按回车键即可 + +4，退出后还需要刷新以下环境变量，执行以下命令完成环境变量的刷新 + +``` +source ~/.bashrc +``` + +这样你就可以使用oebuild这个工具了，试着执行一下`oebuild -h`，看看能否显示oebuild的帮助信息 + +##### 初始化oebuild目录 + +运行如下命令完成oebuild的初始化工作： + +``` +oebuild init +``` + +该操作会初始化oebuild的目录，``表示要初始化目录的名称 + +注：由于oebuild的运行整体依赖docker环境的运行，因此，如果你本地没有安装docker应用，则请按照oebuild给出的提示进行操作，或者按以下给出的方式完成docker的安装 + +1，确定你本机的系统类型，这里以ubuntu为例讲解，执行以下命令完成docker的安装 + +``` +sudo apt install docker docker.io -y +``` + +2, 添加docker用户组 + +``` +sudo groupadd docker +``` + +3, 将本用户添加到docker组内 + +``` +sudo usermod -a -G docker $(whoami) +``` + +4, 重新启动docker + +``` +sudo systemctl-reload && systemctl restart docker +``` + +5, 修改docker.sock读写权限 + +``` +sudo chmod o+rw /var/run/docker.sock +``` + +其他操作系统请参考ubuntu方式进行 + +##### 更新oebuild运行环境 + +运行如下命令来完成初期环境的准备工作： + +``` +oebuild update +``` + +更新工作主要有两点：一，pull相关的运行容器镜像；二，从gitee上下载yocto-meta-openeuler仓代码，如果本地没有openeuler相关容器，则在这一步执行会比较漫长，请耐心等待。 + +##### 创建编译配置文件 + +运行如下命令来产生编译配置文件： + +``` +oebuild generate +``` + +默认配置文件对应的镜像是aarch64标准镜像 + +##### 执行构建操作 + +执行如下命令会进入镜像构建程序： + +``` +oebuild bitbake openeuler-image +``` + +请耐心等待20分钟，你就可以得到一个标准的openEuler Embedded aarch64架构的镜像 + +#### 命令介绍 + +##### oebuild init + +目录初始化指令，主要用于初始化oebuild项目目录，运行该指令在后面需要跟要初始化的目录名，如下： + +``` +oebuild init [directory] [-u yocto_remote_url] [-b branch] +``` + +directory: 表示要初始化的目录名称（注意：我们无法在已经初始化的目录内再次执行初始化操作） + +yocto_remote_url：yocto-meta-openeuler的remote远程链接 + +branch：yocto-meta-openeuler的分支 + +（注意：oebuild在执行构建任务时是依赖已经适配oebuild的yocto-meta-openeuler的仓的） + +例如初始化demo目录只需要执行如下命令： + +``` +oebuild init demo +``` + +init命令执行后主要执行两个任务：一是创建src源码目录，创建.oebuild目录，拷贝config配置文件到.oebuild，二是如果设置了-u或-b参数，则对config文件进行相应的修改 + +初始化目录后demo的目录结构如下： + +``` +.oebuild + config +src +``` + +src：该目录用于存放跟编译相关的源码 + +.oebuild：目录用于存放全局性配置文件，在oebuild执行初始化后，会看到有一个config配置文件，该配置文件将在搭建编译基础环境时应用到。 + +##### oebuild update + +基础环境更新指令，在执行初始化目录指令后，在执行构建环节之前必须要先执行该命令。 + +``` +oebuild update [-t docker_tag] [-l list] [-i ignore] [-e enable] +``` + +-t: 指更新哪个tag的容器 + +-l: 表示列出可选资源列表，目前只有docker这一项 + +-i: 表示在更新时忽略哪一项，可选的有docker与meta，docker代表容器镜像，meta代表yocto-meta-openeuler仓 + +-e: 表示在更新时使能哪一项，可选范围与解释同上 + +执行更新操作如下命令： + +``` +oebuild update +``` + +oebuild执行构建有两个必要的前提，一是构建需要的容器，二是主构建仓（yocto-meta-openeuler）。所以更新命令主要以这两部分展开 + +另外，如果我们有自己的oebuild适配仓，可以在`config`配置文件中修改（该文件在`/.oebuild`目录下），如果已经先执行过更新操作，然后再次执行`oebuild update`会将原有的`yocto-meta-openeuler`做备份，将在工作空间根目录下创建yocto-bak备份目录，然后将备份后的`yocto-meta-openeuler`移动到该目录。更改基础仓在config中的如下字段修改： + +``` +basic_repo: + yocto_meta_openeuler: + path: yocto-meta-openeuler + remote_url: https://gitee.com/openeuler/yocto-meta-openeuler.git + branch: master +``` + +basic_repo与yocto-meta-openeuler是两个key键，不可以更改，remote_url与branch可以更改成自己已经适配的`yocto-meta-openeuler`仓的参数 + +注：如果我们不输入任何参数，即直接执行`oebuild update`，则默认更新容器镜像和基础仓 + +##### oebuild generate + +创建配置文件指令，而该命令就是用来产生配置文件的。 + +``` +oebuild generate [-p platform] [-f features] [-t toolchain_dir] [-d build_directory] [-l list] +``` + +-p：cpu类型参数，全称`platform`，生成配置文件需要的一个参数，默认为aarch64-std + +-f：特性参数，全称`feature`，生成配置文件需要的一个参数，没有默认值 + +-t：外部编译链参数，全称`toolchain_dir`，生成配置文件需要的一个参数，没有默认值，该值表示如果我们不需要系统提供的交叉编译链而选择自己的交叉编译链，则可以选择该参数。 + +-d：初始化的编译目录，如果不设置该参数，则初始化的编译目录和-p参数保持一致 + +-l: list参数，有两个可选范围，platform和feature，platform则会列出支持的platform列表，feature则会列出支持的feature列表 + +oebuild在构建时依赖compile.yaml配置文件来完成构建操作，创建配置文件指令已经属于构建指令内容，该操作将会检查`yocto-meta-openeuler`是否适配了oebuild，检查是否适配的规则便是是否在`yocto-meta-openeuler`根目录创建了`.oebuild`隐藏目录，而`-p`则会解析`.oebuild/platform`下相应的平台配置文件，`-f`参数则会解析`.oebuild/feature`下相应的配置文件，该参数是可以多值传入的，例如如下范例： + +``` +oebuild generate -p aarch64-std -f systemd -f openeuler-qt +``` + +则生成的构建配置文件会涵盖`systemd openeuler-qt`两者的特性 + +最终会在编译目录下（在执行完`oebuild generate`后按提示给出的路径即为编译目录）生成构建配置文件`compile.yaml`,关于该配置文件的详细介绍请参考配置文件介绍中的`compile.yaml`。在下一步的构建流程会解析该配置文件，在此之前，用户可以根据自身特定场景环境来修改配置文件，因为按该`oebuild generate`指令生成的配置文件仅算作一个参考模板，目的是给用户一个最基本的模板参考用，减少用户学习的成本，使用户能够快速上手。 + +##### oebuild bitbake + +构建指令，该指令会解析`compile.yaml`(通过`oebuild generate`指令生成的)，然后完成构建环境的初始化工作。该命令参数如下： + +-h：帮助命令，通过执行 + +``` +oebuild bitbake -h +``` + +来获取帮助 + +一般来说，启动后的容器挂在的目录映射关系如下： + +``` +/src:/usr1/openeuler/src +/build/:/usr1/openeuler/build +``` + +如果在`compile.yaml`中有`toolchain_dir`参数，即有用户自定义外部工具链，则会增加一个挂载目录，如下： + +``` +：/usr1/openeuler/native_gcc +``` + +##### oebuild upgrade + +在线升级指令，该指令会请求远程oebuild版本信息，通过和本地oebuild版本对比来判断是否进行升级 + +由于oebuild二进制包存放在gitee仓库中，因此oebuild在升级时会先克隆最新的二进制仓到用户根目录，并以一个随机文件名命名，然后执行`sudo pip install `来完成升级，在这之中会要求用户输入root密码，在完成升级后会自动删除oebuild二进制包 + +#### 配置文件介绍 + +oebuild在生成后有多个配置文件，每个配置文件的作用域不同，下面将介绍各配置文件存放位置以及内容 + +##### config + +oebuild在外围环境的配置文件,该配置文件存放在oebuild项目根目录下的.oebuild目录中，该配置文件结构如下： + +``` +docker: + repo_url: swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container + tag_map: + openEuler-22.09: '22.09' + openEuler-22.03-lts-sp1: 22.03-lts-sp1 + master: latest +basic_repo: + yocto_meta_openeuler: + path: yocto-meta-openeuler + remote_url: https://gitee.com/openeuler/yocto-meta-openeuler.git + branch: master +``` + +**docker**: 表示构建容器相关信息，在该字段下面所列的容器镜像，在执行`oebuild update`后会下载相应的容器 + +​ repo_url: 表示openEuler Embedded的docker远程仓地址 + +​ tag_map: 表示每个openEuler Embedded版本对用的docker构建容器tag + +**basic_repo**:表示基础的repo仓，顾名思义，表示在构建之前是作为底座的角色存在的，在执行`oebuild update`时会解析config配置文件，然后下载相应的构建代码仓 + +​ yocto-meta-openeuler: 目前oebuild唯一的基础仓 + +​ path: 该仓下载的路径名称 + +​ remote_url: 该仓的远程地址 + +​ branch: 该仓的分支 + +##### .env + + 编译目录配置文件结构如下： + +``` +container: + remote: xxxxxxxxxxxx + branch: xxxxxxxxxxxx + short_id: xxxxxxxxxx + volumns: + - /xxxxxxxxxxx + - /xxxxxxxxxxx +``` + +container:表示容器相关配置信息 + +​ remote: 表示`yocto-meta-openeuler`远程url + +​ branch: 表示`yocto-meta-openeuler`分支信息 + +​ short_id: 表示容器ID + +​ volumns: 表示容器挂在的目录映射 + +oebuild在执行构建过程中，会解析`.env`配置文件，通过对比环境中的其他参数确定是否重新创建一个新的容器还是启用旧容器，比对的内容包括（remote，branch，volumns)只有这三个参数与要构建的对应参数一致，才会继续拉起旧容器，否则就会创建一个新的容器。另外oebuild也会检查设置的short_id对用的容器是否存在，不存在也会创建一个新的容器。在创建新的容器后，新的配置信息会重新写入到`.env`中 + +##### compile.yaml + +构建配置文件，该配置文件结构如下： + +``` +platform: aarch64-std +machine: qemu-aarch64 +toolchain_type: EXTERNAL_TOOLCHAIN_aarch64 +sdk_dir: +toolchain_dir: +repos: + yocto-poky: + url: https://gitee.com/openeuler/yocto-poky.git + path: yocto-poky + refspec: openEuler-22.09 + + yocto-meta-openembedded: + url: https://gitee.com/openeuler/yocto-meta-openembedded.git + path: yocto-meta-openembedded + refspec: dev_hardknott + + yocto-meta-ros: + url: https://gitee.com/openeuler/yocto-meta-ros.git + path: yocto-meta-ros + refspec: dev_hardknott +local_conf: | +- xxx +- xxx +layers: +- xxx +- xxxx +``` + +platform：表示cpu架构， + +machine：表示机器类型 + +toolchain_type: 表示编译链类型 + +sdk_dir: 保留字段 + +toolchain_dir：表示自定义外部编译链路径，如果在`oebuild generate`设置了该参数`-t`，则会在`compile.yaml`存在该字段 + +repos: 表示在初始化构建环境时需要用到的仓 + +​ url: 表示仓的远程地址 + +​ path: 表示仓在本地的地址 + +​ refspec:表示仓的版本分支 + +local_conf：local.conf替换内容，该值在oebuild执行完oe_init后将替换`build/conf/local.conf`中匹配到的内容 + +layers: meta层，该值在oebuild执行完oe_init后将通过调用`bitbake-layers add-layer`来添加meta层 + +#### 开发者帮助 + +oebuild项目欢迎广大爱好开发者参与贡献oebuild的发展，为了使开发者更快更好的参与到oebuild的开发工作中来，我们专门写了如下指导。 + +##### oebuild目录介绍 + +打开oebuild仓我们可以看到，oebuild一级目录有如下内容： + +``` +docs +src +.gitignore +MANIFEST.in +README.md +setup.py +``` + +docs：文档目录，该目录用于存放关于oebuild的介绍性信息 + +src：核心源码目录，我们真正运行oebuild的核心源码就存放在这里，后续介绍关于参与开发oebuild的详细流程将会详细介绍该目录内容 + +.gitignore：git提交忽略的文件，在该文件中通过设置的内容可以在git提交时自动忽略 + +MANIFEST.in：该文件为pip在打包时包含额外文件的配置文件，在该文件中的内容将在执行python打包时按规则进行包含 + +README.md：简要介绍性文件 + +setup.py：python打包入口文件 ，我们最终要打包wheel包就要通过该文件来完成 + + + +##### 如何使用setup.py进行调试或打包 + +在我们完成相关的开发性工作并进行调试时，将通过setup.py内的相关设置来完成该工作 + +打开setup.py文件，我们可以看到其内容如下： + +``` +# Copyright 2018 Open Source Foundries Limited. +# Copyright (c) 2020, Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +import os + +import setuptools + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +os.chdir(SCRIPT_DIR) + +with open('README.md', 'r') as f: + long_description = f.read() + +with open('src/oebuild/version.py', 'r') as f: + __version__ = None + exec(f.read()) + assert __version__ is not None + +version = os.environ.get('OEBUILD_VERSION', __version__) + +setuptools.setup( + name='oebuild', + version=version, + author='alichinese', + author_email='', + description='', + long_description=long_description, + # http://docutils.sourceforge.net/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data + long_description_content_type="text/x-rst", + url='', + packages=setuptools.find_packages(where='src'), + package_dir={'': 'src'}, + include_package_data=True, + classifiers=[ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + ], + install_requires=[ + 'setuptools', + 'packaging', + 'PyYaml', + 'docker', + 'GitPython', + 'colorama', + 'ruamel.yaml' + ], + python_requires='>=3.8', + entry_points={'console_scripts': ('oebuild = oebuild.app.main:main',)}, +) + +``` + +可以看到引入的模块儿有`setuptools`,这个是打包的核心模块儿，关于其他的介绍我们暂且不管，因为对于开发者来说几乎没改动，这里我们着重介绍以下`install_requires`，该设置从字面意思理解就是依赖的必要安装，也就是说oebuild运行要依赖的第三方库，如果我们在后续的oebuild开发过程中有一些其他库的依赖，则需要在这里添加。 + +在进入oebuild目录后，我们可以执行以下命令进入调试状态： + +``` +pip install -e . +``` + +注：以上命令的运行如果以普通用户运行，需要先确认是否将本地执行路径添加到环境变量`PATH`中，如果以root用户运行则不需要考虑，这样我们可以直接运行oebuild相关指令 + +这样在后续开发与调试过程中，我们可以随时改代码随时生效 + +##### src源码介绍 + +正在完善中... \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..bad987e3fb2c5b7ead03ee6f1f9edf9302c1951a --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +import os + +import setuptools + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +os.chdir(SCRIPT_DIR) + +with open('README.md', 'r') as f: + long_description = f.read() + +with open('src/oebuild/version.py', 'r') as f: + __version__ = None + exec(f.read()) + assert __version__ is not None + +version = os.environ.get('OEBUILD_VERSION', __version__) + +setuptools.setup( + name='oebuild', + version=version, + author='alichinese', + author_email='', + description='', + long_description=long_description, + # http://docutils.sourceforge.net/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data + long_description_content_type="text/x-rst", + url='', + packages=setuptools.find_packages(where='src'), + package_dir={'': 'src'}, + include_package_data=True, + classifiers=[ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + ], + install_requires=[ + 'setuptools', + 'packaging', + 'PyYaml', + 'docker', + 'GitPython', + 'colorama', + 'ruamel.yaml', + 'dataclasses', + 'reprint' + ], + python_requires='>=3.8', + entry_points={'console_scripts': ('oebuild = oebuild.app.main:main',)}, +) diff --git a/src/oebuild/__init__.py b/src/oebuild/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/oebuild/__main__.py b/src/oebuild/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..5b2cfc395a0c282408d1c26ec08b7c48eefe55f6 --- /dev/null +++ b/src/oebuild/__main__.py @@ -0,0 +1,15 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +from oebuild.app.main import main + +main() diff --git a/src/oebuild/app/__init__.py b/src/oebuild/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/oebuild/app/conf/config.yaml b/src/oebuild/app/conf/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cde0e17e8ae3da12f3c5187223b9dc7d88b34d18 --- /dev/null +++ b/src/oebuild/app/conf/config.yaml @@ -0,0 +1,11 @@ +docker: + repo_url: swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container + tag_map: + openEuler-22.09: "22.09" + openEuler-22.03-lts-sp1: "22.03-lts-sp1" + master: "latest" +basic_repo: + yocto_meta_openeuler: + path: yocto-meta-openeuler + remote_url: https://gitee.com/openeuler/yocto-meta-openeuler.git + branch: master \ No newline at end of file diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78d45ccd76d485dc8e149b68fe82c7999476a979 --- /dev/null +++ b/src/oebuild/app/conf/plugins.yaml @@ -0,0 +1,16 @@ +plugins: +- name: init + class: Init + path: plugins/init/init.py +- name: update + class: Update + path: plugins/update/update.py +- name: generate + class: Generate + path: plugins/generate/generate.py +- name: bitbake + class: Bitbake + path: plugins/bitbake/bitbake.py +- name: upgrade + class: Upgrade + path: plugins/upgrade/upgrade.py \ No newline at end of file diff --git a/src/oebuild/app/conf/upgrade.yaml b/src/oebuild/app/conf/upgrade.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5c4339e5aed9422a9ce5fa29b9255c2490a1e9d --- /dev/null +++ b/src/oebuild/app/conf/upgrade.yaml @@ -0,0 +1,3 @@ +remote_url: https://gitee.com/alichinese/oebuild-bin.git +branch: master +ver_file: version \ No newline at end of file diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py new file mode 100644 index 0000000000000000000000000000000000000000..89142fd9a02c4f8b3770b7fc1dd253bb8e1ee32f --- /dev/null +++ b/src/oebuild/app/main.py @@ -0,0 +1,346 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import sys +import argparse +import pathlib +import shutil +import textwrap +from io import StringIO +from collections import OrderedDict +import getpass + +import colorama + +import oebuild.util as oebuild_util +from oebuild.version import __version__ +from oebuild.command import ExtCommand, extension_commands +from oebuild.my_log import MyLog as log + +APP = "app" + +class OebuildApp: + ''' + The execution body of the oebuild tool, and all oebuild + commands are resolved and executed by this body + ''' + def __init__(self): + self.base_oebuild_dir = oebuild_util.get_base_oebuild() + self.oebuild_parser = None + self.subparser_gen = None + self.cmd = None + try: + plugins_dir = pathlib.Path(self.base_oebuild_dir,'app/conf','plugins.yaml') + self.command_ext = self.get_command_ext(oebuild_util.read_yaml(plugins_dir)['plugins']) + self.command_spec = {} + except Exception as e_p: + raise e_p + + @staticmethod + def get_command_ext(plugins:list): + ''' + return command information object + ''' + command_ext = OrderedDict() + for app in plugins: + command_ext[app['name']] = ExtCommand( + name=app['name'], + class_name=app['class'], + path=app['path']) + return command_ext + + + def _load_extension_specs(self, ): + self.command_spec = extension_commands(APP, self.command_ext) + + + def _setup_parsers(self): + # Set up and install command-line argument parsers. + + oebuild_parser, subparser_gen = self.make_parsers() + + # Add sub-parsers for the command_ext commands. + for command_name in self.command_ext: + subparser_gen.add_parser(command_name, add_help=False) + + # Save the instance state. + self.oebuild_parser = oebuild_parser + self.subparser_gen = subparser_gen + + def make_parsers(self,): + ''' + Make a fresh instance of the top level argument parser + and subparser generator, and return them in that order. + The prog='oebuild' override avoids the absolute path of the + main.py script showing up when West is run via the wrapper + ''' + + parser = OebuildArgumentParser( + prog='oebuild', description='The openEuler Embedded meta-tool.', + epilog='''Run "oebuild -h" for help on each .''', + add_help=False, oebuild_app=self + ) + + parser.add_argument('-h', '--help', action=OebuildHelpAction, nargs=0, + help='get help for oebuild or a command') + + parser.add_argument('-v', '--version', action='version', + version=f'Oebuild version: v{__version__}', + help='print the program version and exit') + + subparser_gen = parser.add_subparsers(metavar='', + dest='command') + + return parser, subparser_gen + + + def _check_command(self, args): + if args.help or \ + args.command is None or \ + args.command not in self.command_ext or \ + args.command == 'help': + self.help() + return False + + return True + + + def run_command(self, argv): + ''' + Parse command line arguments and run the OebuildCommand. + If we're running an extension, instantiate it from its + spec and re-parse arguments before running. + ''' + args, unknown = self.oebuild_parser.parse_known_args(args=argv) + + if not self._check_command(args=args): + return + + # Finally, run the command. + self._run_extension(args.command, unknown) + + def _run_extension(self, name, unknown): + # Check a program invariant. + self.cmd =self.command_spec[name].factory() + + args = self.cmd.add_parser(self.subparser_gen) + + self.cmd.run(args, unknown) + + + def run(self, argv): + ''' + the function will be exec first + ''' + self._load_extension_specs() + # Set up initial argument parsers. This requires knowing + # self.extensions, so it can't happen before now. + self._setup_parsers() + + # OK, we are all set. Run the command. + self.run_command(argv) + + def help(self,): + ''' + print help message + ''' + self.oebuild_parser.print_help(top_level=True) + + +class OebuildHelpAction(argparse.Action): + ''' + set argparse help is true + ''' + def __call__(self, parser, namespace, values, option_string=None): + # Just mark that help was requested. + namespace.help = True + + +class OebuildArgumentParser(argparse.ArgumentParser): + ''' + The argparse module is infuriatingly coy about its parser and + help formatting APIs, marking almost everything you need to + customize help output an "implementation detail". Even accessing + the parser's description and epilog attributes as we do here is + technically breaking the rules. + + Even though the implementation details have been pretty stable + since the module was first introduced in Python 3.2, let's avoid + possible headaches by overriding some "proper" argparse APIs + here instead of monkey-patching the module or breaking + abstraction barriers. This is duplicative but more future-proof. + ''' + + def __init__(self, *args, **kwargs): + # The super constructor calls add_argument(), so this has to + # come first as our override of that method relies on it. + self.oebuild_optionals = [] + self.oebuild_app = kwargs.pop('oebuild_app', None) + super(OebuildArgumentParser, self).__init__(*args, **kwargs) + + def print_help(self, file=None, top_level=False): + print(self.format_help(top_level=top_level), end='', + file=file or sys.stdout) + + def format_help(self, top_level=False): + # When top_level is True, we override the parent method to + # produce more readable output, which separates commands into + # logical groups. In order to print optionals, we rely on the + # data available in our add_argument() override below. + # + # If top_level is False, it's because we're being called from + # one of the subcommand parsers, and we delegate to super. + + if not top_level: + return super(OebuildArgumentParser, self).format_help() + + # Format the help to be at most 75 columns wide, the maximum + # generally recommended by typographers for readability. + # + # If the terminal width (COLUMNS) is less than 75, use width + # (COLUMNS - 2) instead, unless that is less than 30 columns + # wide, which we treat as a hard minimum. + width = min(75, max(shutil.get_terminal_size().columns - 2, 30)) + + with StringIO() as sio: + + def append(*strings): + for s_t in strings: + print(s_t, file=sio) + + append(self.format_usage(), + self.description, + '') + + append('optional arguments:') + for w_o in self.oebuild_optionals: + self._format_oebuild_optional(append, w_o, width) + + append('') + for _,command in self.oebuild_app.command_spec.items(): + self._format_command(append, command, width) + + if self.epilog: + append(self.epilog) + + return sio.getvalue() + + def _format_oebuild_optional(self, append, w_o, width): + metavar = w_o['metavar'] + options = w_o['options'] + help_msg = w_o.get('help') + + # Join the various options together as a comma-separated list, + # with the metavar if there is one. That's our "thing". + if metavar is not None: + opt_str = ' ' + ', '.join(f'{o} {metavar}' for o in options) + else: + opt_str = ' ' + ', '.join(options) + + # Delegate to the generic formatter. + self._format_thing_and_help(append, opt_str, help_msg, width) + + def _format_command(self, append, command, width): + thing = f' {command.name}:' + self._format_thing_and_help(append, thing, command.help, width) + + def _format_extension_spec(self, append, spec, width): + self._format_thing_and_help(append, ' ' + spec.name + ':', + spec.help, width) + + def _format_thing_and_help(self, append, thing, help_msg, width): + # Format help for some "thing" (arbitrary text) and its + # corresponding help text an argparse-like way. + help_offset = min(max(10, width - 20), 24) + help_indent = ' ' * help_offset + + thinglen = len(thing) + + if help_msg is None: + # If there's no help string, just print the thing. + append(thing) + else: + # Reflow the lines in help to the desired with, using + # the help_offset as an initial indent. + help_msg = ' '.join(help_msg.split()) + help_lines = textwrap.wrap(help_msg, width=width, + initial_indent=help_indent, + subsequent_indent=help_indent) + + if thinglen > help_offset - 1: + # If the "thing" (plus room for a space) is longer + # than the initial help offset, print it on its own + # line, followed by the help on subsequent lines. + append(thing) + append(*help_lines) + else: + # The "thing" is short enough that we can start + # printing help on the same line without overflowing + # the help offset, so combine the "thing" with the + # first line of help. + help_lines[0] = thing + help_lines[0][thinglen:] + append(*help_lines) + + def add_argument(self, *args, **kwargs): + # Track information we want for formatting help. The argparse + # module calls kwargs.pop(), so can't call super first without + # losing data. + optional = {'options': [], 'metavar': kwargs.get('metavar', None)} + need_metavar = (optional['metavar'] is None and + kwargs.get('action') in (None, 'store')) + for arg in args: + if not arg.startswith('-'): + break + optional['options'].append(arg) + # If no metavar was given, the last option name is + # used. By convention, long options go last, so this + # matches the default argparse behavior. + if need_metavar: + optional['metavar'] = arg.lstrip('-').translate( + {ord('-'): '_'}).upper() + optional['help'] = kwargs.get('help') + self.oebuild_optionals.append(optional) + + # Let argparse handle the actual argument. + super().add_argument(*args, **kwargs) + + def error(self, message): + # if (self.oebuild_app and + # self.oebuild_app.mle and + # isinstance(self.oebuild_app.mle, + # ManifestVersionError) and + # self.oebuild_app.cmd): + # self.oebuild_app.cmd.die(mve_msg(self.west_app.mle)) + super().error(message=message) + +def check_user(): + ''' + check execute user must in normal user + ''' + if getpass.getuser() == "root": + log.err("can not use oebuild in root") + return False + return True + +def main(argv=None): + ''' + oebuild main entrypoint + ''' + if not check_user(): + return + + colorama.init() + app = OebuildApp() + app.run(argv or sys.argv[1:]) + +if __name__ == "__main__": + main() diff --git a/src/oebuild/app/plugins/bitbake/__init__.py b/src/oebuild/app/plugins/bitbake/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py new file mode 100644 index 0000000000000000000000000000000000000000..545ecc2deff88310de22e3ab85bdc2be622679d3 --- /dev/null +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -0,0 +1,75 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' +import os + +import oebuild.util as oebuild_util +from oebuild.local_conf import LocalConf +from oebuild.bblayers import BBLayers +import oebuild.app.plugins.bitbake.const as bitbake_const + +class BaseBuild: + ''' + The class provides the basic methods that the build class inherits + ''' + def _set_tmpfile_content(self, content: str): + while True: + tmp_file = oebuild_util.generate_random_str(6) + if os.path.exists(tmp_file): + continue + with open(tmp_file, 'w', encoding="utf-8") as w_f: + w_f.write(content) + break + return tmp_file + + def replace_local_conf(self, parse_compile, local_dir, src_dir = None): + ''' + replace some param in local.conf, the LocalConf will be instantiated + and exec update + ''' + local_conf = LocalConf(local_conf_dir=local_dir) + local_conf.update(parse_compile=parse_compile, src_dir=src_dir) + + def add_bblayers(self, bblayers_dir: str, pre_dir: str, base_dir: str, layers: list): + ''' + add_layers has two main functions, one is to initialize + the compilation directory, and the other is to add the + bblayer layer so that subsequent build directory file + replacement operations can be successfully executed + ''' + bblayers = BBLayers(bblayers_dir=bblayers_dir, + base_dir=base_dir) + pre_dir = os.path.join(pre_dir, 'yocto-poky/..') + bblayers.add_layer(pre_dir=pre_dir, layers=layers) + + def _restore_bashrc_content(self, old_content): + new_content = '' + for line in old_content.split('\n'): + line: str = line + if line.endswith(bitbake_const.BASH_END_FLAG) or line.replace(" ", '') == '': + continue + new_content = new_content + line + '\n' + return new_content + + def _add_bashrc(self, content: str, line: str): + if not content.endswith('\n'): + content = content + '\n' + content = content + line + bitbake_const.BASH_END_FLAG + '\n' + + return content + + def _init_bashrc_content(self, old_content, init_command: list): + new_content = self._restore_bashrc_content(old_content=old_content) + + for command in init_command: + new_content = new_content + command + bitbake_const.BASH_END_FLAG + '\n' + + return new_content diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py new file mode 100644 index 0000000000000000000000000000000000000000..f2e5aafa44a20ec2841b35a51700f0a3ca3e0534 --- /dev/null +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -0,0 +1,112 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +import argparse +import textwrap + +from oebuild.command import OebuildCommand +from oebuild.configure import Configure +from oebuild.parse_compile import ParseCompile, CheckCompileError +from oebuild.parse_env import ParseEnv +import oebuild.util as oebuild_util +from oebuild.my_log import MyLog as log +from oebuild.app.plugins.bitbake.in_container import InContainer +from oebuild.app.plugins.bitbake.in_host import InHost +from oebuild.parse_template import BUILD_IN_HOST +from docker.errors import DockerException + +class Bitbake(OebuildCommand): + ''' + Bitbake instructions can enter the build interactive environment + and then directly run bitbake-related instructions,or run bitbake + command directly, for example: `oebuild bitbake busybox` + ''' + def __init__(self): + self.compile_conf_dir = os.path.join(os.getcwd(), 'compile.yaml') + self.configure = Configure() + + super().__init__( + 'bitbake', + 'execute bitbake command', + textwrap.dedent(''' + Bitbake instructions can enter the build interactive environment and then directly run bitbake-related instructions, + or run bitbake command directly, for example: `oebuild bitbake busybox` + ''') + ) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s [command] +''') + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + ''' + The BitBake execution logic is: + the first step is to prepare the code that initializes + the environment dependency, + the second step to build the configuration file to the object, + the third step to handle the container needed for compilation, + and the fourth step to enter the build environment + ''' + if '-h' in unknown or '--help' in unknown: + args.parse_args(unknown) + return + + command = self._get_command(unknow=unknown) + + if not self.check_support_bitbake(): + log.warning("please do it in compile workspace which contain compile.yaml") + return + + if not os.path.exists('.env'): + os.mknod('.env') + + try: + parse_compile = ParseCompile(self.compile_conf_dir) + except CheckCompileError as c_e: + log.err(str(c_e)) + return + parse_compile.pull_repos(self.configure.source_dir()) + parse_env = ParseEnv(env_dir='.env') + + if parse_compile.build_in == BUILD_IN_HOST: + in_host = InHost(self.configure) + in_host.exec(parse_compile=parse_compile, command=command) + return + try: + oebuild_util.check_docker() + except DockerException as d_e: + log.err(str(d_e)) + return + in_container = InContainer(self.configure) + in_container.exec(parse_env=parse_env, + parse_compile=parse_compile, + command=command) + + def check_support_bitbake(self,): + ''' + The execution of the bitbake instruction mainly relies + on compile.yaml, which is initialized by parsing the file + ''' + return os.path.exists(os.path.join(os.getcwd(), 'compile.yaml')) + + def _get_command(self, unknow: list): + if len(unknow) == 0: + return None + + return 'bitbake ' + ' '.join(unknow) diff --git a/src/oebuild/app/plugins/bitbake/const.py b/src/oebuild/app/plugins/bitbake/const.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce2db52472eb29e6392cbe5c59dceabff443786 --- /dev/null +++ b/src/oebuild/app/plugins/bitbake/const.py @@ -0,0 +1,22 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +BASH_END_FLAG = " ###!!!###" +CONTAINER_SRC = '/usr1/openeuler/src' +CONTAINER_BUILD = '/home/openeuler/build' +CONTAINER_USER = "openeuler" + +BASH_BANNER = ''' + Welcome to the openEuler Embedded build environment, + where you can run 'bitbake openeuler-image' to build + standard images +''' diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py new file mode 100644 index 0000000000000000000000000000000000000000..cce3213994610b1c5282f97d7275cd0d969a6f84 --- /dev/null +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -0,0 +1,327 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' +import re +import os + +from docker.models.containers import Container + +from oebuild.local_conf import NATIVE_GCC_DIR, SSTATE_CACHE +from oebuild.parse_env import ParseEnv, EnvContainer +from oebuild.docker_proxy import DockerProxy +from oebuild.configure import Configure +from oebuild.ogit import OGit +from oebuild.parse_compile import ParseCompile +from oebuild.my_log import MyLog as log +import oebuild.app.plugins.bitbake.const as bitbake_const +from oebuild.app.plugins.bitbake.base_build import BaseBuild + +class InContainer(BaseBuild): + ''' + bitbake command execute in container + ''' + + def __init__(self, configure: Configure): + self.configure = configure + self.client = DockerProxy() + self.container_id = None + + def __del__(self): + if self.container_id is None: + return + # try: + # container = self.client.get_container(self.container_id) + # self.client.stop_container(container=container) + # except Exception as e_p: + # raise e_p + + def exec(self, parse_env: ParseEnv, parse_compile: ParseCompile, command): + ''' + execute bitbake command + ''' + yocto_dir = os.path.join( + self.configure.source_dir(), "yocto-meta-openeuler") + remote, branch = OGit.get_repo_info(yocto_dir) + self.deal_env_container( + env=parse_env, + remote=remote, + branch=branch, + toolchain_dir=parse_compile.toolchain_dir, + sstate_cache=parse_compile.sstate_cache) + + self.exec_compile(parse_compile=parse_compile, command=command) + + def deal_env_container(self, + env: ParseEnv, + remote: str, + branch: str, + toolchain_dir=None, + sstate_cache = None): + ''' + This operation realizes the processing of the container, + controls how the container is processed by parsing the env + variable, if the container does not exist, or the original + environment and the current environment that needs to be set + are inconsistent, you need to create a new container, otherwise + directly enable the sleeping container + ''' + volumns = [] + volumns.append(self.configure.source_dir() + ':' + bitbake_const.CONTAINER_SRC) + volumns.append(self.configure.build_dir() + ':' + bitbake_const.CONTAINER_BUILD) + if toolchain_dir is not None: + volumns.append(toolchain_dir + ":" + NATIVE_GCC_DIR) + + if sstate_cache is not None: + volumns.append(sstate_cache + ":" + SSTATE_CACHE) + + try: + env_container = EnvContainer( + remote=remote, + branch=branch, + volumns=volumns, + short_id=None + ) + check_container = env.is_same_container(data=env_container) + except Exception as e_p: + raise e_p + + if not check_container \ + or env.container.short_id is None \ + or not self.client.is_container_exists(env.container.short_id): + # judge which container + config = self.configure.parse_oebuild_config() + container_config = config.docker + # here use default mater branch bind container and fix next + image_name = container_config.repo_url + ":" + \ + container_config.tag_map.get('master') + container = self.client.container_run_simple( + image=image_name, + volumes=volumns) + + env_container.short_id = container.short_id + env.set_env_container(env_container) + env.export_env() + + self.container_id = env.container.short_id + container = self.client.get_container(self.container_id) + if not self.client.is_container_running(container): + self.client.start_container(container) + + def exec_compile(self, parse_compile: ParseCompile = None, command: str = "" or None): + ''' + execute compile task + ''' + container = self.client.get_container(self.container_id) + + self.init_bash(container=container, + build_dir=os.path.basename(os.getcwd())) + + try: + self.init_bitbake(container=container) + except ValueError as v_e: + log.err(str(v_e)) + return + + # add bblayers, this action must before replace local_conf + bblayers_dir = os.path.join(os.getcwd(), "conf", "bblayers.conf") + self.add_bblayers( + bblayers_dir=bblayers_dir, + pre_dir=bitbake_const.CONTAINER_SRC, + base_dir=self.configure.source_dir(), + layers=parse_compile.layers) + + # replace local_conf + local_dir = os.path.join(os.getcwd(), 'conf', 'local.conf') + self.replace_local_conf( + parse_compile=parse_compile, local_dir=local_dir) + + # add auto execute command for example: bitbake busybox + if command is not None and command != "": + content = self._get_bashrc_content(container=container) + new_content = self._add_bashrc(content=content, line=command) + self.update_bashrc(container=container, content=new_content) + res = self.client.container_exec_command( + container=container, + command="bash .bashrc", + user=bitbake_const.CONTAINER_USER, + work_space=f"/home/{bitbake_const.CONTAINER_USER}") + + for line in res.output: + log.info(line.decode().strip('\n')) + else: + content = self._get_bashrc_content(container=container) + for b_s in bitbake_const.BASH_BANNER.split('\n'): + b_s = f"echo {b_s}{bitbake_const.BASH_END_FLAG}" + content = self._add_bashrc(content=content, line=b_s) + self.update_bashrc(container=container, content=content) + os.system( + f"docker exec -it -u {bitbake_const.CONTAINER_USER} {container.short_id} bash") + + self.restore_bashrc(container=container) + + def init_bitbake(self, container: Container): + ''' + init_bitbake will start a container with pty and then check + bblayers.conf and local.conf if exists in 10 seconds, otherwise + raise init bitbake faild + ''' + self._check_change_ugid(container=container) + self._install_sudo(container=container) + + res = self.client.container_exec_command( + container=container, + command=f"bash /home/{bitbake_const.CONTAINER_USER}/.bashrc", + user=bitbake_const.CONTAINER_USER, + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False) + if res.exit_code != 0: + raise ValueError(res.output.decode()) + # raise ValueError("bitbake init faild") + + def _check_change_ugid(self, container: Container): + res = self.client.container_exec_command( + container=container, + user='root', + command=f"id {bitbake_const.CONTAINER_USER}", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False) + if res.exit_code != 0: + raise ValueError("check docker user id faild") + + res_cont = res.output.decode() + + cuids = res_cont.split(' ') + # get uid from container in default user + pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + bitbake_const.CONTAINER_USER + r'\))') + match_uid = pattern.search(cuids[0]) + if match_uid: + cuid = match_uid.group() + else: + raise ValueError(f"can not get container {bitbake_const.CONTAINER_USER} uid") + # get gid from container in default user + pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + bitbake_const.CONTAINER_USER + r'\))') + match_gid = pattern.search(cuids[1]) + if match_gid: + cgid = match_gid.group() + else: + raise ValueError(f"can not get container {bitbake_const.CONTAINER_USER} gid") + + # judge host uid and gid are same with container uid and gid + # if not same and change container uid and gid equal to host's uid and gid + if os.getuid() != cuid: + self._change_container_uid(container=container, uid=os.getuid()) + if os.getgid() != cgid: + self._change_container_gid(container=container, gid=os.getgid()) + + def _change_container_uid(self, container: Container, uid: int): + self.client.container_exec_command( + container=container, + user='root', + command=f"usermod -u {uid} {bitbake_const.CONTAINER_USER}", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False) + + def _change_container_gid(self, container: Container, gid: int): + self.client.container_exec_command( + container=container, + user='root', + command=f"groupmod -g {gid} {bitbake_const.CONTAINER_USER}", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False) + + def _install_sudo(self, container: Container): + self._replace_yum_mirror(container=container) + + resp = self.client.container_exec_command( + container=container, + user='root', + command="which sudo", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False + ) + if resp.exit_code != 0: + log.info( + "=========================install sudo===============================") + self._install_software(container=container, software="sudo") + + def _replace_yum_mirror(self, container: Container): + self.client.container_exec_command( + container=container, + user='root', + command=r"sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repos.d/openEuler.repo", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=False + ) + + def _install_software(self, container: Container, software: str): + resp = self.client.container_exec_command( + container=container, + user='root', + command=f"yum install {software} -y", + work_space=f"/home/{bitbake_const.CONTAINER_USER}", + stream=True + ) + for line in resp.output: + log.info(line.decode().strip('\n')) + + def init_bash(self, container: Container, build_dir): + ''' + Bitbake will initialize the compilation environment by reading + the user initialization script first, then making directional + substitutions, and finally writing the initialization script + ''' + # read container default user .bashrc content + content = self._get_bashrc_content(container=container) + + init_sdk_command = '. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux' + set_template = f'export TEMPLATECONF="{bitbake_const.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild"' + init_oe_comand = f'. {bitbake_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + {bitbake_const.CONTAINER_BUILD}/{build_dir}' + init_command = [init_sdk_command, set_template, init_oe_comand] + new_content = self._init_bashrc_content(content, init_command) + + self.update_bashrc(container=container, content=new_content) + + def update_bashrc(self, container: Container, content: str): + ''' + update user initialization script by replace file, first create + a file and writed content and copy it to container's .bashrc, finally + remove it + ''' + tmp_file = self._set_tmpfile_content(content) + self.client.copy_to_container( + container=container, + source_path=tmp_file, + to_path=f'/home/{bitbake_const.CONTAINER_USER}') + container.exec_run( + cmd=f"mv /home/{bitbake_const.CONTAINER_USER}/{tmp_file} /home/{bitbake_const.CONTAINER_USER}/.bashrc", + user="root" + ) + os.remove(tmp_file) + + def restore_bashrc(self, container: Container): + ''' + Restoring .bashrc will strip out the command line + content added during bitbake initialization + ''' + old_content = self._get_bashrc_content(container=container) + self.update_bashrc(container=container, + content=self._restore_bashrc_content(old_content=old_content)) + + def _get_bashrc_content(self, container: Container): + content = self.client.container_exec_command( + container=container, + command=f"cat /home/{bitbake_const.CONTAINER_USER}/.bashrc", + user="root", + work_space=None, + stream=False).output + + return content.decode() diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py new file mode 100644 index 0000000000000000000000000000000000000000..ad0bf1a3be43f631951df10b9ea382e411e9fb63 --- /dev/null +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -0,0 +1,143 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' +import os +import subprocess +import pty +import shutil + +from oebuild.local_conf import NativesdkNotExist, NativesdkNotValid +from oebuild.configure import Configure +from oebuild.parse_compile import ParseCompile +from oebuild.my_log import MyLog as log +import oebuild.app.plugins.bitbake.const as bitbake_const +from oebuild.app.plugins.bitbake.base_build import BaseBuild + +class InHost(BaseBuild): + ''' + bitbake command execute in host + ''' + + def __init__(self, configure: Configure): + self.configure = configure + self.container_id = None + + def exec(self, parse_compile: ParseCompile, command): + ''' + execute bitbake commands + ''' + self._init_build_sh(build_dir=os.getcwd()) + self.mk_build_sh(nativesdk_dir=parse_compile.nativesdk_dir, build_dir=os.getcwd()) + self.init_bitbake() + + # add bblayers, this action must before replace local_conf + bblayers_dir = os.path.join(os.getcwd(), "conf", "bblayers.conf") + self.add_bblayers( + bblayers_dir=bblayers_dir, + pre_dir=self.configure.source_dir(), + base_dir=self.configure.source_dir(), + layers=parse_compile.layers) + + local_dir = os.path.join(os.getcwd(), 'conf', 'local.conf') + try: + self.replace_local_conf( + parse_compile=parse_compile, + local_dir=local_dir, + src_dir=self.configure.source_dir()) + except NativesdkNotExist as n_e: + log.err(str(n_e)) + log.err("please set valid nativesdk directory") + return + except NativesdkNotValid as n_e: + log.err(str(n_e)) + log.err(''' +The nativesdk path must be valid, it is recommended +that you download the nativesdk script and then perform +initialization operations''') + return + + if command is not None and command != "": + self._append_build_sh(str_list=[command], build_dir= os.getcwd()) + with subprocess.Popen('bash build.sh', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8") as s_p: + if s_p.returncode is not None and s_p.returncode != 0: + err_msg = '' + for line in s_p.stderr: + err_msg.join(line) + raise ValueError(err_msg) + + for line in s_p.stdout: + log.info(line.strip('\n')) + else: + # run in Interactive mode + banner_list = [] + for b_s in bitbake_const.BASH_BANNER.split('\n'): + b_s = f"echo {b_s}{bitbake_const.BASH_END_FLAG}" + banner_list.append(b_s) + self._append_build_sh(str_list=banner_list, build_dir= os.getcwd()) + append_str = f"sed -i '/{bitbake_const.BASH_END_FLAG}/d' $HOME/.bashrc" + self._append_build_sh(str_list = [append_str], build_dir= os.getcwd()) + + build_sh_dir = os.path.join(os.getcwd(), 'build.sh') + source_build_str = f"source {build_sh_dir}" + content = self._get_bashrc_content() + content = self._restore_bashrc_content(old_content=content) + new_content = self._add_bashrc(content, line=source_build_str) + self.update_bashrc(new_content) + pty.spawn("bash") + + def mk_build_sh(self, nativesdk_dir, build_dir): + ''' + xxx + ''' + init_sdk_command = f'. {nativesdk_dir}/environment-setup-x86_64-pokysdk-linux' + set_template = f'export TEMPLATECONF="{self.configure.source_dir()}/yocto-meta-openeuler/.oebuild"' + init_oe_command = f'. {self.configure.source_dir()}/yocto-poky/oe-init-build-env {build_dir}' + + self._append_build_sh(str_list= [init_sdk_command, set_template, init_oe_command], + build_dir=build_dir) + + def _init_build_sh(self, build_dir): + build_sh_dir = os.path.join(build_dir, 'build.sh') + if os.path.exists(build_sh_dir): + os.remove(build_sh_dir) + os.mknod(build_sh_dir) + + def _append_build_sh(self, str_list:list, build_dir): + build_sh_dir = os.path.join(build_dir, 'build.sh') + if not os.path.exists(build_sh_dir): + raise ValueError("build.sh not exists") + + with open(build_sh_dir, 'a', encoding='utf-8') as w_f: + w_f.write('\n') + w_f.write('\n'.join(str_list)) + + def init_bitbake(self,): + ''' + Bitbake will initialize the compilation environment by reading + the user initialization script first, then making directional + substitutions, and finally writing the initialization script + ''' + subprocess.getoutput("bash build.sh") + + def _get_bashrc_content(self,): + return subprocess.getoutput('cat $HOME/.bashrc') + + def update_bashrc(self, content: str): + ''' + update user initialization script by replace file, first create + a file and writed content and mv it to host's .bashrc + ''' + tmp_file = self._set_tmpfile_content(content) + shutil.move(tmp_file, os.path.join(os.environ['HOME'], '.bashrc')) diff --git a/src/oebuild/app/plugins/demo/demo.py b/src/oebuild/app/plugins/demo/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..df3b6f3415c82e52c299a5ef8ac1267ca8adbd47 --- /dev/null +++ b/src/oebuild/app/plugins/demo/demo.py @@ -0,0 +1,36 @@ +import argparse +import textwrap +import logging + +from oebuild.command import OebuildCommand +from oebuild.util import * +from oebuild.configure import Configure + +logger = logging.getLogger() + +class Demo(OebuildCommand): + + def __init__(self): + self.configure = Configure() + super().__init__( + name='{}', + help='this is your help mesasge', + description=textwrap.dedent('''\ + this is your description message +''' + )) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s [-m URL] [--mr REVISION] [--mf FILE] [directory] + %(prog)s -l [--mf FILE] directory +''') + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + args = args.parse_args(unknown) + pass \ No newline at end of file diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..b66004482596b64a0e3000ed3c79095fb8c6e7a4 --- /dev/null +++ b/src/oebuild/app/plugins/generate/generate.py @@ -0,0 +1,288 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import textwrap +import os +import sys +import pathlib + +from oebuild.command import OebuildCommand +import oebuild.util as oebuild_util +from oebuild.configure import Configure +from oebuild.parse_template import BaseParseTemplate, ParseTemplate, BUILD_IN_DOCKER, BUILD_IN_HOST +from oebuild.my_log import MyLog as log + +class Generate(OebuildCommand): + ''' + compile.yaml is generated according to different command parameters by generate + ''' + def __init__(self): + self.configure = Configure() + self.nativesdk_dir = None + self.toolchain_dir = None + self.sstate_cache = None + super().__init__( + 'generate', + 'help to mkdir build directory and generate compile.yaml', + textwrap.dedent('''\ + compile.yaml is generated according to different command parameters by generate +''' + )) + + def do_add_parser(self, parser_adder): + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s [-p platform] [-f features] [-t toolchain_dir] [-d build_directory] [-l list] [-b_in build_in] +''') + + parser.add_argument('-l', dest='list', choices=['platform', 'feature'], + help=''' + with platform will list support archs, with feature will list support features + ''' + ) + + parser.add_argument('-p', dest='platform', default="aarch64-std", + help=''' + this param is for arch, for example aarch4-std, aarch64-pro and so on + ''' + ) + + parser.add_argument('-s', dest='sstate_cache', + help=''' + this param is for arch, for example aarch4-std, aarch64-pro and so on + ''' + ) + + parser.add_argument('-f', dest='features', action='append', + help=''' + this param is feature, it's a reuse command + ''' + ) + + parser.add_argument('-d', dest='directory', + help=''' + this param is build directory, the default is same to platform + ''' + ) + + parser.add_argument('-t', dest='toolchain_dir', default = '', + help=''' + this param is for external toolchain dir, if you want use your own toolchain + ''' + ) + + parser.add_argument('-n', dest='nativesdk_dir', default = '', + help=''' + this param is for external nativesdk dir, the param will be useful when you want to build in host + ''' + ) + + parser.add_argument('-b_in', dest='build_in', choices=[BUILD_IN_DOCKER, BUILD_IN_HOST], + default = BUILD_IN_DOCKER, help=''' + This parameter marks the mode at build time, and is built in the container by docker + ''' + ) + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + args = args.parse_args(unknown) + + if not self.configure.is_oebuild_dir(): + log.err('your current directory had not finishd init') + sys.exit(-1) + + build_in = BUILD_IN_DOCKER + if args.build_in == BUILD_IN_HOST: + try: + self._check_param_in_host(args=args) + except ValueError as v_e: + log.err(str(v_e)) + return + self.nativesdk_dir = args.nativesdk_dir + build_in = BUILD_IN_HOST + + if args.toolchain_dir != '': + self.toolchain_dir = args.toolchain_dir + + if args.sstate_cache is not None: + self.sstate_cache = args.sstate_cache + + yocto_dir = self.configure.source_yocto_dir() + if not self.check_support_oebuild(yocto_dir): + log.err('Currently, yocto-meta-openeuler does not support oebuild, \ + please modify .oebuild/config and re-execute `oebuild update`') + return + + if args.list is not None: + self.list_info(args=args) + return + + build_dir = self._init_build_dir(args=args) + if build_dir is None: + return + + parser_template = ParseTemplate(yocto_dir=yocto_dir) + + yocto_oebuild_dir = os.path.join(yocto_dir, '.oebuild') + + try: + self._add_platform_template(args=args, + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) + except BaseParseTemplate as b_t: + log.err(str(b_t)) + return + except ValueError as v_e: + log.err(str(v_e)) + return + else: + pass + + try: + self._add_features_template(args=args, + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) + except BaseParseTemplate as b_t: + log.err(str(b_t)) + self._list_feature() + return + except ValueError as v_e: + log.err(str(v_e)) + return + else: + pass + + if os.path.exists(os.path.join(build_dir,'compile.yaml')): + os.remove(os.path.join(build_dir,'compile.yaml')) + + out_dir = pathlib.Path(os.path.join(build_dir,'compile.yaml')) + oebuild_util.write_yaml(out_dir, parser_template.generate_template( + nativesdk_dir= self.nativesdk_dir, + toolchain_dir= self.toolchain_dir, + build_in=build_in, + sstate_cache= self.sstate_cache + )) + + log.info("=============================================") + log.successful("generate compile.yaml successful") + log.info("please run `oebuild bitbake` in directory next") + format_dir = f''' + +cd {build_dir} + +''' + log.info(format_dir) + log.info("=============================================") + + def _check_param_in_host(self, args): + if args.toolchain_dir == '': + raise ValueError("build in host must points toolchain directory in '-t' param") + + if args.nativesdk_dir == '': + raise ValueError("build in host must points nativesdk directory in '-n' param") + + def _add_platform_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): + if args.platform + '.yaml' in os.listdir(os.path.join(yocto_oebuild_dir, 'platform')): + try: + parser_template.add_template( + os.path.join(yocto_oebuild_dir, + 'platform', + args.platform+'.yaml')) + except BaseParseTemplate as e_p: + raise e_p + else: + raise ValueError("wrong platform, please run\ + `oebuild generate -l platform` to view support platform") + + def _add_features_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): + if args.features: + for feature in args.features: + if feature + '.yaml' in os.listdir(os.path.join(yocto_oebuild_dir, 'features')): + try: + parser_template.add_template(os.path.join(yocto_oebuild_dir, + 'features', + feature+'.yaml')) + except BaseParseTemplate as b_t: + raise b_t + else: + raise ValueError("wrong platform, please run \ + `oebuild generate -l feature` to view support feature") + + def _init_build_dir(self, args): + if not os.path.exists(self.configure.build_dir()): + os.makedirs(self.configure.build_dir()) + + if args.directory is None or args.directory == '': + build_dir = os.path.join(self.configure.build_dir(), args.platform) + else: + build_dir = os.path.join(self.configure.build_dir(), args.directory) + + if not os.path.abspath(build_dir).startswith(self.configure.build_dir()): + log.err("build path must in oebuild workspace") + return None + + if not os.path.exists(build_dir): + os.makedirs(build_dir) + return build_dir + + def list_info(self, args): + ''' + print platform list or feature list + ''' + if args.list == 'platform': + self._list_platform() + return + if args.list == 'feature': + self._list_feature() + return + + def _list_platform(self): + log.info("=============================================") + yocto_dir = self.configure.source_yocto_dir() + yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") + list_platform = os.listdir(os.path.join(yocto_oebuild_dir, 'platform')) + log.info("the platform list is:") + for platform in list_platform: + if platform.endswith('.yml'): + log.info(platform.rstrip('.yml')) + if platform.endswith('.yaml'): + log.info(platform.rstrip('.yaml')) + + def _list_feature(self,): + log.info("=============================================") + yocto_dir = self.configure.source_yocto_dir() + yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") + list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) + log.info("the feature list is:") + for feature in list_feature: + if feature.endswith('.yml'): + log.info(feature.rstrip('.yaml')) + if feature.endswith('.yaml'): + log.info(feature.rstrip('.yaml')) + feat = oebuild_util.read_yaml(pathlib.Path(os.path.join(yocto_oebuild_dir, + 'features', + feature))) + if "support" in feat: + log.info(f" support arch: {feat.get('support')}") + else: + log.info(" support arch: all") + + def check_support_oebuild(self, yocto_dir): + ''' + Determine if OeBuild is supported by checking if .oebuild + exists in the yocto-meta-openeuler directory + ''' + return os.path.exists(os.path.join(yocto_dir, '.oebuild')) diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py new file mode 100644 index 0000000000000000000000000000000000000000..42750062acb2183f1158e5336e1b5f87ff4334e5 --- /dev/null +++ b/src/oebuild/app/plugins/init/init.py @@ -0,0 +1,157 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import shutil +import os +import textwrap +import sys + +from oebuild.command import OebuildCommand +import oebuild.util as oebuild_util +from oebuild.configure import Configure, YOCTO_META_OPENEULER, ConfigBasicRepo, CONFIG +from oebuild.my_log import MyLog as log + +class Init(OebuildCommand): + ''' + Directory initialization directive, mainly used to initialize + the OEbuild project directory, running this directive needs + to be followed by the directory name to be initialized + ''' + + def __init__(self): + self.configure = Configure() + self.oebuild_dir = None + self.src_dir = None + + super().__init__( + 'init', + 'Initialize an OEBUILD working directory', + textwrap.dedent('''\ + Initialize an OEBUILD working directory, and execute + all other OEbuild instructions in the initialized directory +''' + )) + + def do_add_parser(self, parser_adder): + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s [directory] [-u yocto_remote_url] [-b branch] +''') + + parser.add_argument('-u', dest = 'yocto_remote_url', + help='''Specifies the remote of yocto-meta-openeuler''') + + parser.add_argument('-b', dest = 'branch', + help='''Specifies the branch of yocto-meta-openeuler''') + + parser.add_argument( + 'directory', nargs='?', default=None, + help='''The name of the directory that will be initialized''') + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + ''' + detach target dicrectory if finished init, if inited, just put out err msg and exit + ''' + iargs = args + args = args.parse_args(unknown) + + if self.configure.is_oebuild_dir(): + log.err(f'The "{os.path.dirname(self.configure.oebuild_dir())}" \ + has already been initialized, please change other directory') + sys.exit(-1) + + if args.directory is None: + log.err("'oebuild init' need param directory") + log.info("\noebuild init help:") + self.print_help(iargs) + return + + if not self.init_workspace(args.directory): + log.err(f"mkdir {args.directory} faild") + return + + os.chdir(args.directory) + oebuild_config = self.configure.parse_oebuild_config() + + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo.get(YOCTO_META_OPENEULER) + if args.yocto_remote_url is not None: + yocto_config.remote_url = args.yocto_remote_url + if args.branch is not None: + yocto_config.branch = args.branch + oebuild_config.basic_repo[YOCTO_META_OPENEULER] = yocto_config + + self.configure.update_oebuild_config(oebuild_config) + + log.successful(f"init {args.directory} successful") + format_msg = f''' +please execute the follow commands next + + cd {os.path.abspath(os.getcwd())} + oebuild update + ''' + log.info(format_msg) + + def init_workspace(self, directory): + ''' + init workspace will copy config file and make new src directory + ''' + try: + os.mkdir(directory) + except FileExistsError: + return False + + self.oebuild_dir = self.create_oebuild_directory(directory) + self.copy_config_file(self.oebuild_dir) + self.src_dir = self.create_src_directory(directory) + return True + + @staticmethod + def create_oebuild_directory(updir : str): + ''' + create oebuild config directory + ''' + try: + oebuild_dir = os.path.join(updir, ".oebuild") + os.mkdir(oebuild_dir) + return oebuild_dir + except FileExistsError: + log.err("mkdir .oebuild faild") + return None + + @staticmethod + def create_src_directory(updir : str): + ''' + this is desctiption + ''' + try: + src_dir = os.path.join(updir, "src") + os.makedirs(src_dir) + return src_dir + except FileExistsError: + log.err("mkdir src faild") + return None + + @staticmethod + def copy_config_file(updir : str): + ''' + copy oebuild config to some directory + ''' + try: + config = oebuild_util.get_config_yaml_dir() + shutil.copyfile(config, os.path.join(updir, CONFIG)) + except FileNotFoundError: + log.err("mkdir config faild") diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py new file mode 100644 index 0000000000000000000000000000000000000000..62e12043a494999a61cc98e59050decec76e1f0f --- /dev/null +++ b/src/oebuild/app/plugins/update/update.py @@ -0,0 +1,201 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import os +import textwrap +import sys +import shutil + +import oebuild.util as oebuild_util +from oebuild.command import OebuildCommand +from oebuild.configure import Configure, ConfigBasicRepo, YOCTO_META_OPENEULER +from oebuild.docker_proxy import DockerProxy +from oebuild.ogit import OGit +from docker.errors import DockerException + +from oebuild.my_log import MyLog as log, INFO_COLOR + +class Update(OebuildCommand): + ''' + The update command will prepare the basic environment + related to the build, such as container images, build base repositories, etc + ''' + def __init__(self): + self.configure = Configure() + + super().__init__( + 'update', + 'Update the basic environment required for the build', + textwrap.dedent(''' + Update the base environment required at build time, such as + updating the necessary docker images and yocto-meta-openeuler repositories + ''') + ) + + def do_add_parser(self, parser_adder): + parser = self._parser( + parser_adder, + usage=''' + %(prog)s [-t docker_tag] [-l list] [-i ignore] [-e enable] +''') + parser.add_argument('-t', dest = 'docker_tag', + help='''specifying the -t parameter will update the corresponding docker image''') + + parser.add_argument('-l',dest = 'list',choices=['docker'], + help='''specifying the -l parameter lists the specified modules''') + + parser.add_argument('-i', dest='ignore', choices=['docker', 'meta'], action='append', + help=''' + specify the -i parameter to ignore the corresponding setting when updating, + when the -e parameter is used at the same time, the -i parameter no longer takes effect + ''' + ) + + parser.add_argument('-e', dest='enable', choices=['docker', 'meta'], action='append', + help=''' + specify the -e parameter to enable the corresponding setting when updating, + when the -e parameter is used at the same time, the -i parameter no longer takes effect + ''' + ) + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + ''' + update action rely on directory which has initd, so check it first + ''' + args = args.parse_args(unknown) + + if not self.configure.is_oebuild_dir(): + log.err('your current directory had not finishd init') + sys.exit(-1) + + if args.list is not None: + if args.list == "docker": + self.list_image_tag() + return + + update_docker, update_meta = True, True + if args.enable is not None: + if "docker" not in args.enable: + update_docker = False + if "meta" not in args.enable: + update_meta = False + elif args.ignore is not None: + if "docker" in args.ignore: + update_docker = False + if "meta" in args.ignore: + update_meta = False + + if update_meta: + self.get_basic_repo() + + if update_docker: + try: + oebuild_util.check_docker() + self.docker_image_update(args.docker_tag) + except DockerException as d_e: + log.err(str(d_e)) + return + + def list_image_tag(self,): + ''' + print compile docker image tag list + ''' + oebuild_config = self.configure.parse_oebuild_config() + docker_config = oebuild_config.docker + log.info("the openeuler embedded docker image repo url:") + log.info(" " + docker_config.repo_url) + log.info("the openeuler embedded docker tag list:") + for tag in docker_config.tag_map.values(): + log.info(" "+tag) + + def get_basic_repo(self,): + ''' + note: get_basic_repo is to download or update basic repo in config + which set in keys basic_repo, the rule is that when the + embedded/src/yocto-meta-openeuler exists, so check whether its + remote is equal or not with config's setting, if equal and run git + pull else mv yocto-meta-openeuler to embedded/bak/yocto-meta-openeuler + and rename yocto-meta-openeuler with a random string suffix. if + embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. + ''' + oebuild_config = self.configure.parse_oebuild_config() + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo.get(YOCTO_META_OPENEULER) + + local_dir = os.path.join(self.configure.source_dir(), yocto_config.path) + if os.path.exists(local_dir): + remote_url, _ = OGit.get_repo_info(local_dir) + if remote_url != yocto_config.remote_url: + if not os.path.exists(self.configure.yocto_bak_dir()): + os.makedirs(self.configure.yocto_bak_dir()) + bak_dir = os.path.join(self.configure.yocto_bak_dir(), + yocto_config.path + "_" + oebuild_util.get_time_stamp()) + log.warning(f"yocto-meta-openeuler remote is changed, \ + bak yocto-meta-openeuler to {bak_dir}") + shutil.move(local_dir, bak_dir) + + log.info(f"clone or pull {yocto_config.remote_url}:{yocto_config.branch} ...") + yocto_repo = OGit(repo_dir=local_dir, + remote_url=yocto_config.remote_url, + branch=yocto_config.branch) + yocto_repo.clone_or_pull_repo() + log.info(f"clone or pull {yocto_config.remote_url}:{yocto_config.branch} finish") + + def docker_image_update(self, docker_tag = None): + ''' + The container update logic is to update the corresponding tag + container image if tag is specified, otherwise it is determined + according to the yocto-meta-openeuler version branch in config, + and if the version branch does not correspond to it, it will enter + interactive mode, which is selected by the user + ''' + oebuild_config = self.configure.parse_oebuild_config() + docker_config = oebuild_config.docker + + if docker_tag is not None and docker_tag not in docker_config.tag_map.values(): + warn_msg = "please select valid docker_tag follow list" + log.warning(warn_msg) + for tag in docker_config.tag_map.keys(): + log.warning(docker_config.tag_map.get(tag)) + return + if docker_tag is None: + basic_config = oebuild_config.basic_repo + yocto_config: ConfigBasicRepo = basic_config.get(YOCTO_META_OPENEULER) + if yocto_config.branch in docker_config.tag_map: + docker_tag = docker_config.tag_map[yocto_config.branch] + else: + input_msg = "please select follow docker_tag:\n" + key_list = [] + for index, key in enumerate(docker_config.tag_map): + input_msg += f"{index+1}, {docker_config.repo_url}:\ + {docker_config.tag_map[key]}\n" + key_list.append(key) + input_msg += "please enter index number(enter q will exit):" + while True: + i = input(INFO_COLOR + input_msg) + if i == 'q': + sys.exit() + + i = int(i) + if i <= 0 or i > len(key_list): + log.warning("enter wrong") + continue + docker_tag = docker_config.tag_map[key_list[i-1]] + break + + docker_image = docker_config.repo_url + ":" + docker_tag + client = DockerProxy() + log.info(f"pull {docker_image} ...") + client.pull_image_with_progress(docker_image) + log.info(f"finishd pull {docker_image} ...") diff --git a/src/oebuild/app/plugins/upgrade/upgrade.py b/src/oebuild/app/plugins/upgrade/upgrade.py new file mode 100644 index 0000000000000000000000000000000000000000..7cbda832a05766592a75eafd1cd98df080f2c6b3 --- /dev/null +++ b/src/oebuild/app/plugins/upgrade/upgrade.py @@ -0,0 +1,96 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import textwrap +import argparse +import pathlib +import os +import getpass +import shutil + +import requests + +from oebuild.command import OebuildCommand +import oebuild.util as oebuild_util +from oebuild.ogit import OGit +from oebuild.my_log import MyLog as log + +class Upgrade(OebuildCommand): + ''' + the command for oebuild upgrade + ''' + def __init__(self): + super().__init__( + 'upgrade', + 'the command to upgrade oebuild', + textwrap.dedent('''\ + you can run this command to upgrade oebuild +''' + )) + + def do_add_parser(self, parser_adder): + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s +''') + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + ''' + Implement the online upgrade function by comparing + the local version and the remote version, and perform + the online upgrade function if there is inconsistency. + The upgrade method is to download the remote oebuild + binary warehouse locally, then execute 'pip install *.whl', + and then delete the local binary warehouse + ''' + args.parse_args(unknown) + + upgrade_conf_dir = oebuild_util.get_upgrade_yaml_dir() + upgrade_conf = oebuild_util.read_yaml(pathlib.Path(upgrade_conf_dir)) + ver_url = os.path.join(upgrade_conf['remote_url'], + 'raw', + upgrade_conf['branch'], + upgrade_conf['ver_file']) + response = requests.get(url=ver_url, timeout=5) + if response.status_code == 200: + log.info("the oebuild is latest version") + else: + log.err("download faild") + return + + version = oebuild_util.get_oebuild_version() + if version != response.content.decode(): + self.download_binary(upgrade_conf['remote_url'], upgrade_conf['branch']) + + def download_binary(self, remote_url, branch): + ''' + download oebuild binary package and install it + ''' + while True: + random_str = oebuild_util.generate_random_str(6) + random_repo_dir = os.path.join('/home',getpass.getuser(), random_str) + if os.path.exists(random_repo_dir): + continue + break + ogit = OGit(repo_dir=random_repo_dir, remote_url=remote_url, branch= branch) + ogit.clone_or_pull_repo() + + for f_name in os.listdir(random_repo_dir): + if f_name.endswith("whl"): + oebuild_file = f_name + os.system(f"pip install {os.path.join(random_repo_dir,oebuild_file)}") + if os.path.exists(random_repo_dir): + shutil.rmtree(random_repo_dir) diff --git a/src/oebuild/bb/__init__.py b/src/oebuild/bb/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac107ea6780f3eebb2345aa686a9240240083a4 --- /dev/null +++ b/src/oebuild/bb/utils.py @@ -0,0 +1,364 @@ +# Copyright (C) 2004 Michael Lauer +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import re + +def preserved_envvars_exported(): + """Variables which are taken from the environment and placed in and exported + from the metadata""" + return [ + 'BB_TASKHASH', + 'HOME', + 'LOGNAME', + 'PATH', + 'PWD', + 'SHELL', + 'USER', + 'LC_ALL', + 'BBSERVER', + ] + +def preserved_envvars(): + """Variables which are taken from the environment and placed in the metadata""" + v = [ + 'BBPATH', + 'BB_PRESERVE_ENV', + 'BB_ENV_WHITELIST', + 'BB_ENV_EXTRAWHITE', + ] + return v + preserved_envvars_exported() + +def approved_variables(): + """ + Determine and return the list of whitelisted variables which are approved + to remain in the environment. + """ + if 'BB_PRESERVE_ENV' in os.environ: + return os.environ.keys() + approved = [] + if 'BB_ENV_WHITELIST' in os.environ: + approved = os.environ['BB_ENV_WHITELIST'].split() + approved.extend(['BB_ENV_WHITELIST']) + else: + approved = preserved_envvars() + if 'BB_ENV_EXTRAWHITE' in os.environ: + approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split()) + if 'BB_ENV_EXTRAWHITE' not in approved: + approved.extend(['BB_ENV_EXTRAWHITE']) + return approved + +def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): + """Edit lines from a recipe or config file and modify one or more + specified variable values set in the file using a specified callback + function. Lines are expected to have trailing newlines. + Parameters: + meta_lines: lines from the file; can be a list or an iterable + (e.g. file pointer) + variables: a list of variable names to look for. Functions + may also be specified, but must be specified with '()' at + the end of the name. Note that the function doesn't have + any intrinsic understanding of _append, _prepend, _remove, + or overrides, so these are considered as part of the name. + These values go into a regular expression, so regular + expression syntax is allowed. + varfunc: callback function called for every variable matching + one of the entries in the variables parameter. The function + should take four arguments: + varname: name of variable matched + origvalue: current value in file + op: the operator (e.g. '+=') + newlines: list of lines up to this point. You can use + this to prepend lines before this variable setting + if you wish. + and should return a four-element tuple: + newvalue: new value to substitute in, or None to drop + the variable setting entirely. (If the removal + results in two consecutive blank lines, one of the + blank lines will also be dropped). + newop: the operator to use - if you specify None here, + the original operation will be used. + indent: number of spaces to indent multi-line entries, + or -1 to indent up to the level of the assignment + and opening quote, or a string to use as the indent. + minbreak: True to allow the first element of a + multi-line value to continue on the same line as + the assignment, False to indent before the first + element. + To clarify, if you wish not to change the value, then you + would return like this: return origvalue, None, 0, True + match_overrides: True to match items with _overrides on the end, + False otherwise + Returns a tuple: + updated: + True if changes were made, False otherwise. + newlines: + Lines after processing + """ + + var_res = {} + if match_overrides: + override_re = r'(_[a-zA-Z0-9-_$(){}]+)?' + else: + override_re = '' + for var in variables: + if var.endswith('()'): + var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + else: + var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) + + updated = False + varset_start = '' + varlines = [] + newlines = [] + in_var = None + full_value = '' + var_end = '' + + def handle_var_end(): + prerun_newlines = newlines[:] + op = varset_start[len(in_var):].strip() + (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines) + changed = (prerun_newlines != newlines) + + if newvalue is None: + # Drop the value + return True + elif newvalue != full_value or (newop not in [None, op]): + if newop not in [None, op]: + # Callback changed the operator + varset_new = "%s %s" % (in_var, newop) + else: + varset_new = varset_start + + if isinstance(indent, int): + if indent == -1: + indentspc = ' ' * (len(varset_new) + 2) + else: + indentspc = ' ' * indent + else: + indentspc = indent + if in_var.endswith('()'): + # A function definition + if isinstance(newvalue, list): + newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue))) + else: + if not newvalue.startswith('\n'): + newvalue = '\n' + newvalue + if not newvalue.endswith('\n'): + newvalue = newvalue + '\n' + newlines.append('%s {%s}\n' % (varset_new, newvalue)) + else: + # Normal variable + if isinstance(newvalue, list): + if not newvalue: + # Empty list -> empty string + newlines.append('%s ""\n' % varset_new) + elif minbreak: + # First item on first line + if len(newvalue) == 1: + newlines.append('%s "%s"\n' % (varset_new, newvalue[0])) + else: + newlines.append('%s "%s \\\n' % (varset_new, newvalue[0])) + for item in newvalue[1:]: + newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append('%s"\n' % indentspc) + else: + # No item on first line + newlines.append('%s " \\\n' % varset_new) + for item in newvalue: + newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append('%s"\n' % indentspc) + else: + newlines.append('%s "%s"\n' % (varset_new, newvalue)) + return True + else: + # Put the old lines back where they were + newlines.extend(varlines) + # If newlines was touched by the function, we'll need to return True + return changed + + checkspc = False + + for line in meta_lines: + if in_var: + value = line.rstrip() + varlines.append(line) + if in_var.endswith('()'): + full_value += '\n' + value + else: + full_value += value[:-1] + if value.endswith(var_end): + if in_var.endswith('()'): + if full_value.count('{') - full_value.count('}') >= 0: + continue + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + else: + skip = False + for (varname, var_re) in var_res.items(): + res = var_re.match(line) + if res: + isfunc = varname.endswith('()') + if isfunc: + splitvalue = line.split('{', 1) + var_end = '}' + else: + var_end = res.groups()[-1] + splitvalue = line.split(var_end, 1) + varset_start = splitvalue[0].rstrip() + value = splitvalue[1].rstrip() + if not isfunc and value.endswith('\\'): + value = value[:-1] + full_value = value + varlines = [line] + in_var = res.group(1) + if isfunc: + in_var += '()' + if value.endswith(var_end): + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + skip = True + break + if not skip: + if checkspc: + checkspc = False + if newlines and newlines[-1] == '\n' and line == '\n': + # Squash blank line if there are two consecutive blanks after a removal + continue + newlines.append(line) + return (updated, newlines) + +def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): + """Edit bblayers.conf, adding and/or removing layers + Parameters: + bblayers_conf: path to bblayers.conf file to edit + add: layer path (or list of layer paths) to add; None or empty + list to add nothing + remove: layer path (or list of layer paths) to remove; None or + empty list to remove nothing + edit_cb: optional callback function that will be called after + processing adds/removes once per existing entry. + Returns a tuple: + notadded: list of layers specified to be added but weren't + (because they were already in the list) + notremoved: list of layers that were specified to be removed + but weren't (because they weren't in the list) + """ + + import fnmatch + + def remove_trailing_sep(pth): + if pth and pth[-1] == os.sep: + pth = pth[:-1] + return pth + + approved = approved_variables() + def canonicalise_path(pth): + pth = remove_trailing_sep(pth) + if 'HOME' in approved and '~' in pth: + pth = os.path.expanduser(pth) + return pth + + def layerlist_param(value): + if not value: + return [] + elif isinstance(value, list): + return [remove_trailing_sep(x) for x in value] + else: + return [remove_trailing_sep(value)] + + addlayers = layerlist_param(add) + removelayers = layerlist_param(remove) + + # Need to use a list here because we can't set non-local variables from a callback in python 2.x + bblayercalls = [] + removed = [] + plusequals = False + orig_bblayers = [] + + def handle_bblayers_firstpass(varname, origvalue, op, newlines): + bblayercalls.append(op) + if op == '=': + del orig_bblayers[:] + orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()]) + return (origvalue, None, 2, False) + + def handle_bblayers(varname, origvalue, op, newlines): + updated = False + bblayers = [remove_trailing_sep(x) for x in origvalue.split()] + if removelayers: + for removelayer in removelayers: + for layer in bblayers: + if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)): + updated = True + bblayers.remove(layer) + removed.append(removelayer) + break + if addlayers and not plusequals: + for addlayer in addlayers: + if addlayer not in bblayers: + updated = True + bblayers.append(addlayer) + del addlayers[:] + + if edit_cb: + newlist = [] + for layer in bblayers: + res = edit_cb(layer, canonicalise_path(layer)) + if res != layer: + newlist.append(res) + updated = True + else: + newlist.append(layer) + bblayers = newlist + + if updated: + if op == '+=' and not bblayers: + bblayers = None + return (bblayers, None, 2, False) + else: + return (origvalue, None, 2, False) + + with open(bblayers_conf, 'r') as f: + (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass) + + if not bblayercalls: + raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf) + + # Try to do the "smart" thing depending on how the user has laid out + # their bblayers.conf file + if bblayercalls.count('+=') > 1: + plusequals = True + + removelayers_canon = [canonicalise_path(layer) for layer in removelayers] + notadded = [] + for layer in addlayers: + layer_canon = canonicalise_path(layer) + if layer_canon in orig_bblayers and not layer_canon in removelayers_canon: + notadded.append(layer) + notadded_canon = [canonicalise_path(layer) for layer in notadded] + addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon] + + (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers) + if addlayers: + # Still need to add these + for addlayer in addlayers: + newlines.append('BBLAYERS += "%s"\n' % addlayer) + updated = True + + if updated: + with open(bblayers_conf, 'w') as f: + f.writelines(newlines) + + notremoved = list(set(removelayers) - set(removed)) + + return (notadded, notremoved) diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py new file mode 100644 index 0000000000000000000000000000000000000000..ab7f0987fa448cc4c7596a36c9ca11781c9c1e99 --- /dev/null +++ b/src/oebuild/bblayers.py @@ -0,0 +1,83 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os + +import oebuild.bb.utils as bb_utils + +class BBLayers: + ''' + The BBlayer class implements the layer added in the + container environment in the physical environment, + and the add operation references bitbake-related code + ''' + def __init__(self, bblayers_dir: str, base_dir: str): + self._base_dir = base_dir + self._bblayers_dir = bblayers_dir + + @property + def base_dir(self): + ''' + Returns base_dir value + ''' + return self._base_dir + + @property + def bblayers_dir(self): + ''' + Returns bblayers_dir value + ''' + return self._bblayers_dir + + def add_layer(self, pre_dir: str, layers: str or list): + ''' + Add a layer layer to bblayers.conf, but our layer + layer verification is done on the host, + and the added path is written as a path in the container + args: + pre_dir (str): when added layer with path, for example + pre_dir/layer + layers (str or list): needed to add to bblayers.conf + ''' + try: + self.check_layer_exist(layers=layers) + except Exception as e_p: + raise e_p + + if isinstance(layers, str): + layers = [os.path.join(pre_dir, layers)] + else: + tmp_layers = [] + for layer in layers: + tmp_layers.append(os.path.join(pre_dir, layer)) + layers = tmp_layers + + bb_utils.edit_bblayers_conf(self.bblayers_dir, add=layers, remove=None) + + def check_layer_exist(self, layers:str or list): + ''' + To check if it is legitimate to add a layer, + the main thing is to verify the existence of layer.conf + args: + layers (str or list): needed to add to bblayers.conf + ''' + if isinstance(layers, str): + layers = [layers] + + for layer in layers: + layer_dir = os.path.join(self.base_dir, layer) + if not os.path.exists(layer_dir): + raise ValueError("layer does not exists") + + layer_conf_dir = os.path.join(layer_dir, 'conf', 'layer.conf') + if not os.path.exists(layer_conf_dir): + raise ValueError("invalid layer") diff --git a/src/oebuild/command.py b/src/oebuild/command.py new file mode 100644 index 0000000000000000000000000000000000000000..ed0076e2ae1e3aa4f9e245663465500a53268b53 --- /dev/null +++ b/src/oebuild/command.py @@ -0,0 +1,219 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +from abc import ABC, abstractmethod +import argparse +import importlib +from collections import OrderedDict +import os +from dataclasses import dataclass +import sys +from typing import List +import colorama + +INF_COLOR = colorama.Fore.LIGHTGREEN_EX + +WRN_COLOR = colorama.Fore.LIGHTYELLOW_EX + +ERR_COLOR = colorama.Fore.LIGHTRED_EX + +class OebuildCommand(ABC): + '''Abstract superclass for a oebuild command.''' + + def __init__(self, name, help_msg, description): + self.name = name + self.help_msg = help_msg + self.description = description + self.parser = None + + def run(self, args: argparse.Namespace, unknown: List[str]): + ''' + The executing body, each inherited class will + register the executor with the executor body for execution + ''' + self.do_run(args=args, unknown=unknown) + + def add_parser(self, parser_adder): + ''' + Registers a parser for this command, and returns it. + The parser object is stored in a ``parser`` attribute. + :param parser_adder: The return value of a call to + ``argparse.ArgumentParser.add_subparsers()`` + ''' + parser = self.do_add_parser(parser_adder) + + if parser is None: + raise ValueError('do_add_parser did not return a value') + + self.parser = parser + return self.parser + + @abstractmethod + def do_add_parser(self, parser_adder): + ''' + The directive registers the interface, which the successor needs to implement + ''' + + @abstractmethod + def do_run(self, args: argparse.Namespace, unknown: List[str]): + ''' + Subclasses must implement; called to run the command. + :param args: ``argparse.Namespace`` of parsed arguments + :param unknown: If ``accepts_unknown_args`` is true, a + sequence of un-parsed argument strings. + ''' + + def print_help(self, args: argparse.Namespace): + ''' + print help message + ''' + args = args.parse_args(['-h']) + + def _parser(self, parser_adder, **kwargs): + # Create and return a "standard" parser. + + kwargs['help'] = self.help_msg + kwargs['description'] = self.description + kwargs['formatter_class'] = argparse.RawDescriptionHelpFormatter + return parser_adder.add_parser(self.name, **kwargs) + + +class CommandError(RuntimeError): + ''' + Indicates that a command failed. + ''' + def __init__(self, returncode=1): + super().__init__() + self.returncode = returncode + + +class CommandContextError(CommandError): + '''Indicates that a context-dependent command could not be run.''' + + +class ExtensionCommandError(CommandError): + '''Exception class indicating an extension command was badly + defined and could not be created.''' + + + def __init__(self, **kwargs): + self.hint = kwargs.pop('hint', None) + super(ExtensionCommandError, self).__init__(**kwargs) + +@dataclass +class _CmdFactory: + + py_file: str + name: str + attr: str + + def __call__(self): + # Append the python file's directory to sys.path. This lets + # its code import helper modules in a natural way. + py_dir = os.path.dirname(self.py_file) + sys.path.append(py_dir) + + # Load the module containing the command. Convert only + # expected exceptions to ExtensionCommandError. + try: + mod = _commands_module_from_file(self.py_file, self.attr) + except ImportError as i_e: + raise ExtensionCommandError( + hint=f'could not import {self.py_file}') from i_e + + # Get the attribute which provides the OebuildCommand subclass. + try: + cls = getattr(mod, self.attr) + except AttributeError as a_e: + raise ExtensionCommandError( + hint=f'no attribute {self.attr} in {self.py_file}') from a_e + + # Create the command instance and return it. + try: + return cls() + except Exception as e_p: + raise ExtensionCommandError( + hint='command constructor threw an exception') from e_p + + +@dataclass +class OebuildExtCommandSpec: + ''' + An object which allows instantiating a oebuild extension. + ''' + + # Command name, as known to the user + name: str + + description: str + + help: str + + # This returns a OebuildCommand instance when called. + # It may do some additional steps (like importing the definition of + # the command) before constructing it, however. + factory: _CmdFactory + + +@dataclass +class ExtCommand: + ''' + record extern command basic info, it's useful when app initialize + ''' + name: str + + class_name: str + + path: str + + +def extension_commands(pre_dir, commandlist:OrderedDict): + ''' + Get descriptions of available extension commands. + The return value is an ordered map from project paths to lists of + OebuildExtCommandSpec objects, for projects which define extension + commands. The map's iteration order matches the manifest.projects + order. + ''' + specs = OrderedDict() + for key, value in commandlist.items(): + specs[key] = _ext_specs(pre_dir, value) + + return specs + + +def _ext_specs(pre_dir, command_ext: ExtCommand): + + py_file = os.path.join(os.path.dirname(__file__), pre_dir, command_ext.path) + + factory = _CmdFactory(py_file=py_file, name=command_ext.name, attr=command_ext.class_name) + + return OebuildExtCommandSpec( + name=command_ext.name, + description=factory().description, + help=factory().help_msg, + factory=factory) + + +def _commands_module_from_file(file, mod_name): + ''' + Python magic for importing a module containing oebuild extension + commands. To avoid polluting the sys.modules key space, we put + these modules in an (otherwise unpopulated) oebuild.commands.ext + package. + ''' + + spec = importlib.util.spec_from_file_location(mod_name, file) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + return mod diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py new file mode 100644 index 0000000000000000000000000000000000000000..5abbc4ae9dd24a2e46f3fa7029d141546c3f2893 --- /dev/null +++ b/src/oebuild/configure.py @@ -0,0 +1,207 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +from typing import Dict, Optional, Union +import pathlib +from dataclasses import dataclass + +import oebuild.util as oebuild_util + +PathType = Union[str, os.PathLike] + +YOCTO_META_OPENEULER = "yocto_meta_openeuler" +CONFIG = "config" + +class OebuildNotFound(RuntimeError): + '''Neither the current directory nor any parent has a oebuild workspace.''' + + +@dataclass +class ConfigContainer: + ''' + container object in config + ''' + # repo_url is for container's repo url + repo_url: str + + # tag_mag is for branch to container tag map + tag_map: Dict + +@dataclass +class ConfigBasicRepo: + ''' + basic repo object in config + ''' + # path is for repo's path that will downloaded + path: str + + # remote url is for repo's remote url + remote_url: str + + # branch is for repo's branch + branch: str + +@dataclass +class Config: + ''' + config object container docker and basic_repo + ''' + docker: ConfigContainer + + basic_repo: Dict + +class Configure: + ''' + Configure object is to contain some generally param or function about oebuild + ''' + + @staticmethod + def oebuild_topdir(start: Optional[PathType] = None, + fall_back: bool = True): + ''' + Like oebuild_dir(), but returns the path to the parent directory of the .oebuild/ + directory instead, where project repositories are stored + ''' + cur_dir = pathlib.Path(start or os.getcwd()) + + while True: + if (cur_dir / '.oebuild').is_dir(): + return os.fspath(cur_dir) + + parent_dir = cur_dir.parent + if cur_dir == parent_dir: + # At the root. Should we fall back? + if fall_back: + return Configure.oebuild_topdir(fall_back=False) + + raise OebuildNotFound('Could not find a oebuild workspace ' + 'in this or any parent directory') + cur_dir = parent_dir + + @staticmethod + def oebuild_dir(start: Optional[PathType] = None): + '''Returns the absolute path of the workspace's .oebuild directory. + + Starts the search from the start directory, and goes to its + parents. If the start directory is not specified, the current + directory is used. + + Raises OebuildNotFound if no .oebuild directory is found. + ''' + return os.path.join(Configure.oebuild_topdir(start), '.oebuild') + + @staticmethod + def is_oebuild_dir(): + ''' + Determine whether OEBuild is initialized + ''' + try: + Configure.oebuild_topdir() + return True + except OebuildNotFound: + return False + + @staticmethod + def source_dir(): + ''' + returns src directory base on topdir, the openEuler Embedded meta layers + will be in here when you run oebuild update + ''' + return os.path.join(Configure.oebuild_topdir(), 'src') + + @staticmethod + def source_yocto_dir(): + ''' + return src/yocto-meta-openeuler path + ''' + config = Configure.parse_oebuild_config() + basic_config = config.basic_repo + yocto_config:ConfigBasicRepo = basic_config.get(YOCTO_META_OPENEULER) + yocto_dir = yocto_config.path + return os.path.join(Configure.source_dir(), yocto_dir) + + @staticmethod + def yocto_bak_dir(): + ''' + returns yocto_bak directory base on topdir, the openEuler Embedded meta layers + will be in here when you run oebuild update + ''' + return os.path.join(Configure.oebuild_topdir(), 'yocto_bak') + + @staticmethod + def build_dir(): + ''' + returns build absolute path which the build result will be in + ''' + return os.path.join(Configure.oebuild_topdir(), 'build') + + @staticmethod + def env_dir(): + ''' + returns env path + ''' + return os.path.join(Configure.build_dir(), '.env') + + @staticmethod + def parse_oebuild_config(): + ''' + just parse oebuild config and return a json object, + the file path is {WORKSPACE}.oebuild/config + ''' + + config = oebuild_util.read_yaml(yaml_dir = pathlib.Path(Configure.oebuild_dir(), CONFIG)) + + tag_map = {} + for key, value in config['docker']['tag_map'].items(): + tag_map[key] = value + docker_config = ConfigContainer(repo_url=config['docker']['repo_url'], tag_map=tag_map) + + basic_config = {} + for key, repo in config['basic_repo'].items(): + basic_config[key] = ConfigBasicRepo(path=repo['path'], + remote_url=repo['remote_url'], + branch=repo['branch']) + + config = Config(docker=docker_config, basic_repo=basic_config) + + return config + + @staticmethod + def update_oebuild_config(config: Config): + ''' + update {WORKSPACE}/.oebuild/config + ''' + data = {} + + docker_config = config.docker + data['docker'] = {} + data['docker']['repo_url'] = docker_config.repo_url + tag_map = {} + for key, value in docker_config.tag_map.items(): + tag_map[key] = value + data['docker']['tag_map'] = tag_map + + basic_config = config.basic_repo + data['basic_repo'] = {} + for key, repo in basic_config.items(): + repo:ConfigBasicRepo = repo + data['basic_repo'][key] = {'path': repo.path, + 'remote_url': repo.remote_url, + 'branch': repo.branch} + + try: + oebuild_util.write_yaml(yaml_dir = pathlib.Path(Configure.oebuild_dir(), CONFIG), + data=data) + return True + except TypeError: + return False diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..7c9022df3560dec0f7d6d3eeeae05dbacc22c4ca --- /dev/null +++ b/src/oebuild/docker_proxy.py @@ -0,0 +1,289 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +from io import BytesIO +import tarfile +from queue import Queue +import threading + +import docker +from docker.errors import ImageNotFound, NotFound +from docker.models.containers import Container +from reprint import output + +class DockerProxy: + ''' + a object just be wrapper again to run docker command easily + ''' + def __init__(self): + self._docker = docker.from_env() + + def is_image_exists(self, image_name): + ''' + determize if image exist + args: + image_name (str): docker image name + ''' + try: + self._docker.images.get(image_name) + return True + except ImageNotFound: + return False + + def is_container_exists(self, container_id): + ''' + determize if container exists + args: + container_id (str): docker container short_id or id + ''' + try: + self._docker.containers.get(container_id=container_id) + return True + except NotFound: + return False + + def pull_image(self, image_name : str): + ''' + pull image like command 'docker pull' + args: + image_name (str): docker image name, if no tag, the default tag is latest + ''' + repository,tag = self._get_image_name_tag(image_name=image_name) + + self._docker.images.pull(repository=repository, tag=tag) + + def pull_image_with_progress(self, image_name: str): + ''' + pull docker image and print progress + ''' + def flush_print(in_q:Queue): + with output(output_type='dict') as output_lines: + while True: + data = in_q.get() + if data == "over": + break + if 'foot_msg' in data: + output_lines.append(data['foot_msg']) + continue + output_lines[data['id']] = data['message'] + + client = docker.APIClient() + repository,tag = self._get_image_name_tag(image_name=image_name) + p_q = Queue() + f_p = threading.Thread(target=flush_print, args=(p_q,)) + f_p.start() + resp = client.pull(repository=repository, tag=tag, stream=True, decode=True) + for line in resp: + if 'id' in line: + if "progressDetail" in line: + tmp_data = { + 'id': line['id'], + 'message': f"{line['status']}" + } + if line['progressDetail'] != {}: + tmp_data['message'] = f"{tmp_data['message']} {line['progress']}" + p_q.put(tmp_data) + else: + if 'status' in line: + if 'status' in line: + tmp_data = {'foot_msg': line['status']} + p_q.put(tmp_data) + + p_q.put("over") + + def _get_image_name_tag(self, image_name: str): + repository = image_name.split(':')[0] + tag = "latest" + if len(image_name.split(':'))==2: + tag = image_name.split(':')[1] + return repository, tag + + def get_image(self, image_name): + ''' + get a docker image object + args: + image_name (str): docker image name + ''' + return self._docker.images.get(image_name) + + + def get_container(self, container_id): + ''' + get a docker container object + args: + container_id (str): docker container short_id or id + ''' + return self._docker.containers.get(container_id=container_id) + + @staticmethod + def stop_container(container: Container): + ''' + stop a container if running like command 'docker stop' + args: + container (Container): container object + ''' + container.stop() + + @staticmethod + def delete_container(container: Container): + ''' + rm a container which not running like command 'docker rm' + args: + container (Container): container object + ''' + container.remove() + + @staticmethod + def start_container(container: Container): + ''' + start a container like command 'docker start' + args: + container (Container): container object + ''' + container.start() + + @staticmethod + def is_container_running(container : Container): + ''' + determize if a container in running state + args: + container (Container): container object + ''' + if container.status == "running": + return True + return False + + @staticmethod + def add_tar(path_dir): + ''' + add a path to tar + args: + path_dir (str): the directory that will added to a tar + ''' + if os.path.exists(path = path_dir): + pw_tarstream = BytesIO() + with tarfile.TarFile(fileobj=pw_tarstream, mode='w') as pw_tar: + pw_tar.add(name = path_dir, arcname=os.path.basename(path_dir)) + pw_tarstream.seek(0) + return pw_tarstream + return None + + def copy_to_container(self, container: Container, source_path, to_path): + ''' + copy file that tar before to container + args: + container (Container): docker container object + source_path (str): which copied file path + to_path (str): will copy to docker container path + ''' + tar = self.add_tar(source_path) + return container.put_archive(path = to_path,data = tar) + + def copy_from_container(self, container: Container, from_path, dst_path): + ''' + copy file from container to local + args: + container (Container): docker container object + from_path (str): which copied file path + dst_path (str): will copy from docker container path + ''' + pw_tarstream = BytesIO() + bits, _ = container.get_archive(from_path) + for trunk in bits: + pw_tarstream.write(trunk) + pw_tarstream.seek(0) + with tarfile.open(fileobj = pw_tarstream) as tar: + res = tar.extractall(path = dst_path) + return res is None + + def container_exec_command(self, + container: Container, + command: str or list, + user: str = '', + work_space = None, + stream = True): + ''' + run command like 'docker run exec', other param + will be default and just use a little params + returns a data stream + ''' + res = container.exec_run( + cmd=command, + user=user, + workdir=work_space, + stderr=True, + stdout=True, + stream=stream + ) + + return res + + def container_run_command(self, + image:str, + command: str, + user: str, + volumes: list, + work_space: str): + ''' + run command like 'docker run' with tty being true + to keep container alive and then run command in + docker container + ''' + container = self._docker.containers.run( + image=image, + command="bash", + volumes=volumes, + detach=True, + tty=True + ) + + res = self.container_exec_command( + container=container, + command=command, + user=user, + work_space=work_space) + + return container, res.output + + def container_run_simple(self, image:str, volumes: list): + ''' + it's just create a tty docker container to do some thing next + ''' + container = self._docker.containers.run( + image=image, + command="bash", + volumes=volumes, + detach=True, + tty=True + ) + return container + + def container_exec_with_tty(self, + container: Container, + user:str, + work_space: str): + ''' + run docker container with tty, you can has a tty terminal + ''' + cli = container.exec_run( + cmd="bash", + stdout=True, + stderr=True, + stdin=True, + user=user, + workdir=work_space, + tty=True, + socket=True, + ).output + + return cli.fileno() diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..a5600777b9a95f73d7142950b2dfe251113a0fa3 --- /dev/null +++ b/src/oebuild/local_conf.py @@ -0,0 +1,230 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +import re + +from oebuild.parse_compile import ParseCompile +from oebuild.parse_template import BUILD_IN_DOCKER, BUILD_IN_HOST + +class BaseLocalConf(ValueError): + ''' + basic error about parse_template + ''' +class NativesdkNotExist(BaseLocalConf): + ''' + nativesdk directory not exist + ''' +class NativesdkNotValid(BaseLocalConf): + ''' + nativesdk directory not valid + ''' + +NATIVESDK_DIR_NAME = "OPENEULER_NATIVESDK_SYSROOT" +NATIVESDK_SYSROOT = "sysroots/x86_64-pokysdk-linux" +OPENEULER_SP_DIR = "OPENEULER_SP_DIR" +NATIVESDK_ENVIRONMENT = "environment-setup-x86_64-pokysdk-linux" +SSTATE_MIRRORS = "SSTATE_MIRRORS" + +NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' +SSTATE_CACHE = '/usr1/openeuler/sstate-cache' + +def match_and_add(new_str: str, content: str): + ''' + math line in content when the new_str not exist and added + ''' + for line in content.split('\n'): + if new_str.strip() != line.strip(): + continue + return content + + content = content + '\n' + content = content + new_str + content = content + '\n' + return content + +def match_and_replace(pre: str, new_str: str, content: str): + ''' + math line in content when the new_str exist and replace + ''' + for line in content.split('\n'): + ret = re.match(f'^({pre})', line) + if ret is None: + continue + return content.replace(line, new_str) + + content = content + '\n' + content = content + new_str + content = content + '\n' + return content + +class LocalConf: + ''' + LocalConf corresponds to the local.conf configuration + file, which can be modified by specifying parameters + ''' + + def __init__(self, local_conf_dir: str): + self.local_dir = local_conf_dir + + def update(self, parse_compile: ParseCompile, src_dir = None): + ''' + update local.conf by ParseCompile + ''' + local_dir = self.local_dir + if not os.path.exists(local_dir): + raise ValueError(f'{local_dir} not exists') + + if parse_compile.local_conf is None: + return + + with open(local_dir, 'r', encoding='utf-8') as r_f: + content = r_f.read() + + # replace machine + content = match_and_replace( + pre='MACHINE ', + new_str=f'MACHINE = "{parse_compile.machine}"', + content=content) + + # replace platform + content = match_and_replace( + pre="OPENEULER_PLATFORM ", + new_str=f'OPENEULER_PLATFORM = "{parse_compile.platform}"', + content=content + ) + + # replace toolchain + if parse_compile.toolchain_dir is not None: + replace_toolchain_str = parse_compile.toolchain_type + ' = "' + if parse_compile.build_in == BUILD_IN_DOCKER: + replace_toolchain_str += NATIVE_GCC_DIR + '"' + else: + replace_toolchain_str += parse_compile.toolchain_dir + replace_toolchain_str += '"' + content = match_and_replace( + pre=parse_compile.toolchain_type, + new_str=replace_toolchain_str, + content=content + ) + + # replace nativesdk OPENEULER_SP_DIR + if parse_compile.build_in == BUILD_IN_HOST: + self.check_nativesdk_valid(parse_compile.nativesdk_dir) + nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, NATIVESDK_SYSROOT) + content = match_and_replace( + pre=NATIVESDK_DIR_NAME, + new_str=NATIVESDK_DIR_NAME + ' = "' + nativesdk_sys_dir + '"', + content=content + ) + + content = match_and_replace( + pre=OPENEULER_SP_DIR, + new_str=OPENEULER_SP_DIR + ' = "' + src_dir + '"', + content=content + ) + + # replace sstate_cache + if parse_compile.sstate_cache is not None: + if os.path.islink(parse_compile.sstate_cache): + new_str= f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" + else: + if parse_compile.build_in == BUILD_IN_DOCKER: + new_str= f"file://.* file://{SSTATE_CACHE}/PATH" + else: + new_str= f"file://.* file://{parse_compile.sstate_cache}/PATH" + content = match_and_replace( + pre=SSTATE_MIRRORS, + new_str= SSTATE_MIRRORS + ' = "' + new_str + '"', + content=content + ) + + content = self.match_lib_param(content=content) + + content = self._match_lib(parse_compile=parse_compile, content=content) + + content = self.replace_param(parse_compile=parse_compile, content=content) + + with open(local_dir, 'w', encoding="utf-8") as r_f: + r_f.write(content) + + def match_lib_param(self, content: str): + ''' + add params LIBC, TCMODE-LIBC, crypt + ''' + new_content = '' + for line in content.split('\n'): + ret = re.match('^(LIBC)', line.strip()) + if ret is not None: + continue + ret = re.match('^(TCMODE-LIBC)', line.strip()) + if ret is not None: + continue + ret = re.match('^(crypt)', line.strip()) + if ret is not None: + continue + new_content = new_content + line + '\n' + return new_content + + def _match_lib(self, parse_compile: ParseCompile, content: str): + if parse_compile.toolchain_dir is not None and 'musl' in parse_compile.toolchain_dir: + content = content + 'LIBC = "musl"\n' + content = content + 'TCMODE-LIBC = "musl"\n' + content = content + 'crypt = "musl"\n' + content.replace('aarch64-openeuler-linux-gnu', 'aarch64-openeuler-linux-musl') + else: + content = content + 'LIBC = "glibc"\n' + content = content + 'TCMODE-LIBC = "glibc-external"\n' + content = content + 'crypt = "libxcrypt-external"\n' + content.replace('aarch64-openeuler-linux-musl', 'aarch64-openeuler-linux-gnu') + return content + + def replace_param(self, parse_compile: ParseCompile, content:str): + ''' + match and replace param by ParseCompile.local_conf + ''' + for line in parse_compile.local_conf.split('\n'): + ret = re.match(r'^([A-Z0-9_]+)(append)(\s)', line) + if ret is not None: + content = match_and_add(line, content) + continue + ret = re.match(r'^([A-Z0-9_]+)([a-z_/]+)(\s)', line) + if ret is not None: + content = match_and_replace(ret.group(), line, content) + continue + ret = re.match(r'^(require)(\s)', line) + if ret is not None: + content = match_and_add(line, content) + continue + return content + + def check_nativesdk_valid(self, nativesdk_dir): + ''' + Check whether the set nativesdk is valid, check whether + the path exists, and then check whether the internal + OECORE_NATIVE_SYSROOT variables are consistent with the set nativesdk + ''' + if not os.path.exists(nativesdk_dir): + raise NativesdkNotExist(f"nativesdk directory: {nativesdk_dir} not exist") + + nativesdk_environment_dir = os.path.join(nativesdk_dir, NATIVESDK_ENVIRONMENT) + + with open(nativesdk_environment_dir, 'r', encoding='utf-8') as r_f: + for line in r_f.readlines(): + line = line.strip('\n') + if not line.startswith("export OECORE_NATIVE_SYSROOT="): + continue + oecore_sysroot_dir = line.lstrip('export OECORE_NATIVE_SYSROOT=').strip('"') + if not oecore_sysroot_dir.startswith(nativesdk_dir): + raise NativesdkNotValid(f"nativesdk directory: {nativesdk_dir} are not valid") + return + \ No newline at end of file diff --git a/src/oebuild/my_log.py b/src/oebuild/my_log.py new file mode 100644 index 0000000000000000000000000000000000000000..54ce026075a1c5cca0b88eaf65ef428d5bc92ddb --- /dev/null +++ b/src/oebuild/my_log.py @@ -0,0 +1,58 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import colorama + +#: Color used (when applicable) for printing with successful() +INFO_COLOR = colorama.Fore.WHITE + +#: Color used (when applicable) for printing with successful() +SUCCESS_COLOR = colorama.Fore.LIGHTGREEN_EX + +#: Color used (when applicable) for printing with wrn() +WRN_COLOR = colorama.Fore.LIGHTYELLOW_EX + +#: Color used (when applicable) for printing with err() and die() +ERR_COLOR = colorama.Fore.LIGHTRED_EX + +class MyLog: + ''' + Simple log output is implemented, including info, successful, warning, err four output types + ''' + + @staticmethod + def info(msg): + ''' + normal message print + ''' + print(INFO_COLOR + msg) + + @staticmethod + def successful(msg): + ''' + successful message print + ''' + print(SUCCESS_COLOR + msg) + + @staticmethod + def warning(msg): + ''' + warning messaage print + ''' + print(WRN_COLOR + msg) + + @staticmethod + def err(msg): + ''' + err message print + ''' + print(ERR_COLOR + msg) diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py new file mode 100644 index 0000000000000000000000000000000000000000..c169860eeb600199f4b26e07898ea69209f3caed --- /dev/null +++ b/src/oebuild/ogit.py @@ -0,0 +1,135 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os + +import git + +from oebuild.my_log import MyLog as log + +( + BEGIN, + END, + COUNTING, + COMPRESSING, + WRITING, + RECEIVING, + RESOLVING, + FINDING_SOURCES, + CHECKING_OUT, +) = [1 << x for x in range(9)] + +class OGit: + ''' + owner git to print progress in clone action + ''' + def __init__(self, repo_dir, remote_url, branch) -> None: + self._repo_dir = repo_dir + self._remote_url = remote_url + self._branch = branch + self._last_code = 0 + _, self._screen_width = os.popen('stty size', 'r').read().split() + + @property + def repo_dir(self): + ''' + return repo dir + ''' + return self._repo_dir + + @property + def remote_url(self): + ''' + return remote url + ''' + return self._remote_url + + @property + def branch(self): + ''' + return branch + ''' + return self.branch + + def clone_or_pull_repo(self, ): + ''' + clone or pull git repo + ''' + if os.path.exists(self._repo_dir): + try: + repo = git.Repo(self._repo_dir) + remote = repo.remote() + if repo.active_branch.name != self._branch: + log.info(f"Fetching into '{self._repo_dir}'...") + remote.fetch(progress=self.clone_process) + repo.git.checkout(self._branch) + log.info(f"Pulling into '{self._repo_dir}'...") + remote.pull(progress=self.clone_process) + except Exception as e_p: + raise e_p + else: + try: + log.info(f"Cloning into '{self._repo_dir}'...") + git.Repo.clone_from( + url=self._remote_url, + to_path=self._repo_dir, + branch=self._branch, + progress=self.clone_process) + except Exception as e_p: + raise e_p + + def clone_process(self, op_code, cur_count, max_count, message): + ''' + print clone or pull progress + ''' + if op_code % 2 == BEGIN: + print("") + return + op_title = '' + pmsg = '' + if op_code == COUNTING: + op_title = "remote: Counting objects" + pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ + ({cur_count}/{max_count}), {message}" + + elif op_code == COMPRESSING: + op_title = "remote: Compressing objects" + pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ + ({cur_count}/{max_count}), {message}" + + elif op_code == RECEIVING: + op_title = "Receiving objects" + pmsg = f"{op_title}: {int(cur_count/max_count*100)}% \ + ({cur_count}/{max_count}), {message}" + + elif op_code == RESOLVING: + op_title = "Resolving deltas" + pmsg = f"{op_title}: {int(cur_count/max_count*100)}% ({cur_count}/{max_count})" + else: + return + + pmsg = "\r" + pmsg + pmsg = pmsg.ljust(int(self._screen_width), ' ') + print(pmsg, end='', flush=True) + + @staticmethod + def get_repo_info(repo_dir: str): + ''' + return git repo info: remote_url, branch + ''' + try: + repo = git.Repo(repo_dir) + remote_url = repo.remote().url + branch = repo.active_branch.name + return remote_url, branch + except git.GitError: + return "","" diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py new file mode 100644 index 0000000000000000000000000000000000000000..1cacf4f792aa6910468ac650ffb81cd994fdd366 --- /dev/null +++ b/src/oebuild/parse_compile.py @@ -0,0 +1,196 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +from dataclasses import dataclass +import pathlib + +import oebuild.util as oebuild_util +from oebuild.ogit import OGit +from oebuild.parse_template import PlatformTemplate,ParseTemplate,BUILD_IN_DOCKER + +@dataclass +class Compile(PlatformTemplate): + ''' + Compile is the parsed object of compile.yaml and is used to manipulate the build file + ''' + toolchain_dir: str + + nativesdk_dir: str + + not_use_repos: bool + + build_in: str + + sstate_cache: str + +class BaseParseCompileError(ValueError): + ''' + parse compile basic error + ''' + +class CheckCompileError(BaseParseCompileError): + ''' + compile.yaml parse check faild error + ''' + +class ParseCompile: + ''' + This class is used to parse compile.yaml and + download the relevant code repository + ''' + def __init__(self, compile_conf_dir): + self.compile = None + self.init_parse(compile_conf_dir) + + def init_parse(self, compile_conf_dir): + ''' + The initialization operation is used to parse the compile.yaml + file and perform a series of checks before parsing + ''' + if not os.path.exists(compile_conf_dir): + raise ValueError('compile.yaml is not exists') + + compile_conf_dir = pathlib.Path(compile_conf_dir) + data = oebuild_util.read_yaml(compile_conf_dir) + + try: + self.check_compile_conf(data=data) + except Exception as e_p: + raise e_p + + self.compile = Compile( + build_in=BUILD_IN_DOCKER if 'build_in' not in data else data['build_in'], + platform=data['platform'], + machine=data['machine'], + toolchain_type=data['toolchain_type'], + toolchain_dir=None if 'toolchain_dir' not in data else data['toolchain_dir'], + nativesdk_dir=None if 'nativesdk_dir' not in data else data['nativesdk_dir'], + sstate_cache=None if 'sstate_cache' not in data else data['sstate_cache'], + not_use_repos=False if 'not_use_repos' not in data else data['not_use_repos'], + repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), + local_conf=None if "local_conf" not in data else data['local_conf'], + layers=None if "layers" not in data else data['layers'] + ) + + @property + def build_in(self): + ''' + return attr of buildin + ''' + return self.compile.build_in + + @property + def platform(self): + ''' + return attr of platform + ''' + return self.compile.platform + + @property + def machine(self): + ''' + return attr of machine + ''' + return self.compile.machine + + @property + def toolchain_type(self): + ''' + return attr of toolchain_type + ''' + return self.compile.toolchain_type + + @property + def toolchain_dir(self): + ''' + return attr of toolchain_dir + ''' + return self.compile.toolchain_dir + + @property + def nativesdk_dir(self): + ''' + return attr of nativesdk_dir + ''' + return self.compile.nativesdk_dir + + @property + def local_conf(self): + ''' + return attr of local_conf path + ''' + return self.compile.local_conf + + @property + def layers(self): + ''' + return attr of layers + ''' + return self.compile.layers + + @property + def not_use_repos(self): + ''' + return attr of not_use_repos + ''' + return self.compile.not_use_repos + + @property + def sstate_cache(self): + ''' + return attr of sstate_cache + ''' + return self.compile.sstate_cache + + def pull_repos(self, base_dir): + ''' + Download the repos set in compile.yaml based on the given base path + ''' + if not self.compile.not_use_repos: + repos = self.compile.repos + for _, repo in repos.items(): + repo_dir = os.path.join(base_dir, repo.path) + try: + repo_git = OGit(repo_dir=repo_dir, remote_url=repo.url, branch=repo.refspec) + repo_git.clone_or_pull_repo() + except Exception as e_p: + raise e_p + + @staticmethod + def check_compile_conf(data): + ''' + Check whether the compile.yaml content is compliant + ''' + + if "platform" not in data: + raise CheckCompileError("the key platform is None") + + if "machine" not in data: + raise CheckCompileError("the key machine is None") + + if "toolchain_type" not in data: + raise CheckCompileError("the key toolchain_type is None") + + if "toolchain_dir" in data and data['toolchain_dir'] is not None: + if not os.path.exists(data['toolchain_dir']): + raise CheckCompileError(f"the toolchain_dir {data['toolchain_dir']} is not exist") + + if "repos" in data: + for _, repo in data['repos'].items(): + if "url" not in repo: + raise CheckCompileError("the key url is None") + if "path" not in repo: + raise CheckCompileError("the key path is None") + if "refspec" not in repo: + raise CheckCompileError("the key refspec is None") + \ No newline at end of file diff --git a/src/oebuild/parse_env.py b/src/oebuild/parse_env.py new file mode 100644 index 0000000000000000000000000000000000000000..4bf638c8faa00dd68da855f371d8fd5c1fce5b39 --- /dev/null +++ b/src/oebuild/parse_env.py @@ -0,0 +1,143 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +from dataclasses import dataclass +import pathlib + +import oebuild.util as oebuild_util + +@dataclass +class EnvContainer: + ''' + the container object in env object + ''' + remote: str + + branch: str + + short_id: str + + volumns: list + +@dataclass +class Env: + ''' + the env object + ''' + container: EnvContainer + +class ParseEnv: + ''' + This class is used to parse env.yaml and + update env.yaml + ''' + def __init__(self, env_dir): + self.env_dir = pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir + self.env = Env(container=None) + self.parse_env() + + @property + def container(self): + ''' + return container object + ''' + return self.env.container + + def parse_env(self): + ''' + parse env.yaml to env object + ''' + data = oebuild_util.read_yaml(self.env_dir) + if data is None: + return + + if "container" in data: + env_container = data['container'] + self.env.container = EnvContainer( + remote=oebuild_util.add_git_suffix(env_container['remote']), + branch=env_container['branch'], + short_id=env_container['short_id'], + volumns=env_container['volumns'] + ) + + def is_same_container(self, data: EnvContainer): + ''' + judge if container same with container in env.yaml + ''' + if data.remote is None: + raise ValueError("the key remote is lack") + + if data.branch is None: + raise ValueError("the key branch is lack") + + if data.volumns is None: + raise ValueError("the key volumns is lack") + + if self.env.container is None: + return False + + if self.env.container.remote != data.remote: + return False + + if self.env.container.branch != data.branch: + return False + + if len(self.env.container.volumns) != len(data.volumns): + return False + + a_gather = set(self.env.container.volumns) + b_gather = set(data.volumns) + + c_gather = a_gather.symmetric_difference(b_gather) + if len(c_gather) != 0: + return False + + return True + + def set_env_container(self, env_container: EnvContainer): + ''' + set ParseEnv's container object + ''' + self.env.container = env_container + + def export_env(self): + ''' + export env object to env.yaml + ''' + data = {} + if self.env.container is not None: + container = self.env.container + data['container'] = { + 'remote': container.remote, + 'branch': container.branch, + 'short_id': container.short_id, + 'volumns': container.volumns + } + + oebuild_util.write_yaml(pathlib.Path(self.env_dir), data=data) + + @staticmethod + def check_env_container(env_container: EnvContainer): + ''' + Check that the env.yaml content is compliant + ''' + if "remote" not in env_container: + raise ValueError("the key remote is lack") + + if "branch" not in env_container: + raise ValueError("the key branch is lack") + + if "short_id" not in env_container: + raise ValueError("the key short_id is lack") + + if "volumns" not in env_container: + raise ValueError("the key volumns is lack") diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py new file mode 100644 index 0000000000000000000000000000000000000000..f29e17f793c99350b71b6d7e69ae77b52fb691c4 --- /dev/null +++ b/src/oebuild/parse_template.py @@ -0,0 +1,258 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +from dataclasses import dataclass +from typing import Dict +import pathlib +import os + +from ruamel.yaml.scalarstring import LiteralScalarString + +import oebuild.util as oebuild_util + +PLATFORM = 'platform' +BUILD_IN_DOCKER = "docker" +BUILD_IN_HOST = "host" + +@dataclass +class OebuildRepo: + ''' + object repo is to record template repo info, repo struct is: + repo_name: + url: str + path: str + refspec: str + object repo transfer string to struct to use it next easily + ''' + + repo_name: str + + url: str + + path: str + + refspec: str + +@dataclass +class Template: + ''' + basic template for paltform and feature + ''' + repos: Dict[str, 'OebuildRepo'] + + layers: list + + local_conf: LiteralScalarString + +@dataclass +class PlatformTemplate(Template): + ''' + the object will be parsed by platform config + ''' + platform: LiteralScalarString + + machine: LiteralScalarString + + toolchain_type: LiteralScalarString + +@dataclass +class FeatureTemplate(Template): + ''' + the object will be parsed by feature config + ''' + feature_name: LiteralScalarString + + support: list + +class BaseParseTemplate(ValueError): + ''' + basic error about parse_template + ''' + +class ConfigPathNotExists(BaseParseTemplate): + ''' + config path not exists + ''' + +class PlatformNotAdd(BaseParseTemplate): + ''' + platform not add first + ''' + +class FeatureNotSupport(BaseParseTemplate): + ''' + feature not support + ''' + +class CommonNotFound(BaseParseTemplate): + ''' + common param not found + ''' + +class ParseTemplate: + ''' + ParseTemplate is to add platform template and feature template and export compile.yaml finially + ''' + def __init__(self, yocto_dir:str): + self.yocto_dir = yocto_dir + self.build_in = None + self.platform_template = None + self.feature_template = [] + + def add_template(self, config_dir): + ''' + this method is to add Template, note: the template has two type for board and application, + and the deal is difference, when adding board template it will set board_template as the + board_template unset, or replace as the board_template had setted. but feature_template + will append to feature_template anywhere, the feature_template adding must after + board_templiate, else throw exception + ''' + if not isinstance(config_dir, pathlib.Path): + config_dir = pathlib.Path(config_dir) + if not os.path.exists(config_dir): + raise ConfigPathNotExists(f'{config_dir} is not exists') + + try: + data = oebuild_util.read_yaml(config_dir) + repo_dict = None if 'repos' not in data else self.parse_oebuild_repo(data['repos']) + + layers = None if 'layers' not in data else data['layers'] + local_conf = None if 'local_conf' not in data else data['local_conf'] + + config_type = data['type'] + config_name = os.path.basename(config_dir) + if config_type == PLATFORM: + self.platform_template = PlatformTemplate( + platform=os.path.splitext(config_name)[0], + machine=data['machine'], + toolchain_type=data['toolchain_type'], + repos=repo_dict, + local_conf=local_conf, + layers=layers) + return + + if self.platform_template is None: + raise PlatformNotAdd('please add platform template first') + + support_arch = [] + if 'support' in data: + support_arch = data['support'].split('|') + if self.platform_template.platform not in support_arch: + raise FeatureNotSupport(f'your arch is {self.platform_template.platform}, \ + the feature is not supported, please check your \ + application support archs') + + self.feature_template.append(FeatureTemplate( + feature_name=os.path.splitext(config_name)[0], + repos=repo_dict, + support=support_arch, + local_conf=local_conf, + layers=layers + )) + + except Exception as e_p: + raise e_p + + def generate_template(self, + nativesdk_dir = None, + toolchain_dir = None, + build_in: str = BUILD_IN_DOCKER, + sstate_cache = None): + ''' + first param common yaml + ''' + common_yaml_dir = os.path.join(self.yocto_dir, '.oebuild', 'common.yaml') + if not os.path.exists(common_yaml_dir): + raise CommonNotFound('can not find .oebuild/common.yaml in yocto-meta-openeuler') + + if self.platform_template is None: + raise PlatformNotAdd('please set platform template first') + + common_yaml_dir = pathlib.Path(common_yaml_dir) + data = oebuild_util.read_yaml(common_yaml_dir) + + repos = {} + if 'repos' in data : + repos.update(data['repos']) + layers = [] + if 'layers' in data: + layers.extend(data['layers']) + local_conf = LiteralScalarString('') + if 'local_conf' in data: + local_conf += LiteralScalarString(data['local_conf']) + + if self.platform_template.repos is not None: + for repo_name, oebuild_repo in self.platform_template.repos.items(): + if repo_name in repos: + continue + repos[repo_name] = { + 'url': oebuild_repo.url, + 'path': oebuild_repo.path, + 'refspec': oebuild_repo.refspec + } + + if self.platform_template.layers is not None: + self.platform_template.layers.extend(layers) + layers = self.platform_template.layers + + if self.platform_template.local_conf is not None: + local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf ) + + for feature in self.feature_template: + feature:FeatureTemplate = feature + if feature.repos is not None: + for repo_name, oebuild_repo in feature.repos.items(): + if repo_name in repos: + continue + repos[repo_name] = { + 'url': oebuild_repo.url, + 'path': oebuild_repo.path, + 'refspec': oebuild_repo.refspec + } + if feature.layers is not None: + layers.extend(feature.layers) + + if feature.local_conf is not None: + local_conf = LiteralScalarString(feature.local_conf + local_conf) + + compile_conf = { + 'build_in': build_in, + 'platform': self.platform_template.platform, + 'machine': self.platform_template.machine, + 'toolchain_type': self.platform_template.toolchain_type} + + if nativesdk_dir is not None: + compile_conf['nativesdk_dir'] = nativesdk_dir + if toolchain_dir is not None: + compile_conf['toolchain_dir'] = toolchain_dir + if sstate_cache is not None: + compile_conf['sstate_cache'] = sstate_cache + + compile_conf['repos'] = repos + compile_conf['local_conf'] = local_conf + compile_conf['layers'] = layers + return compile_conf + + @staticmethod + def parse_oebuild_repo(repos): + ''' + parse repo json object to OebuildRepo + ''' + repo_cict = {} + for name, repo in repos.items(): + repo_cict[name] = OebuildRepo( + repo_name=name, + url=repo['url'], + path=repo['path'], + refspec=repo['refspec']) + + return repo_cict diff --git a/src/oebuild/util.py b/src/oebuild/util.py new file mode 100644 index 0000000000000000000000000000000000000000..328a74b83ba98f309f9c3d0c066128186299da76 --- /dev/null +++ b/src/oebuild/util.py @@ -0,0 +1,125 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import pathlib +import os +import time +import random +import getpass + +from ruamel.yaml import YAML +from oebuild.docker_proxy import DockerProxy +from docker.errors import DockerException + +from oebuild.version import __version__ + +CONFIG_YAML = 'config.yaml' +UPGRADE_YAML = 'upgrade.yaml' + +def read_yaml(yaml_dir : pathlib.Path): + ''' + read yaml file and parse it to object + ''' + if not os.path.exists(yaml_dir.absolute()): + raise ValueError(f"yaml_dir can not find in :{yaml_dir.absolute()}") + + try: + with open(yaml_dir.absolute(), 'r', encoding='utf-8') as r_f: + yaml = YAML() + data = yaml.load(r_f.read()) + return data + except Exception as e_p: + raise e_p + + +def write_yaml(yaml_dir : pathlib.Path, data): + ''' + write data to yaml file + ''' + if not os.path.exists(yaml_dir.absolute()): + os.mknod(yaml_dir) + + with open(yaml_dir, 'w', encoding='utf-8') as w_f: + yaml = YAML() + yaml.dump(data, w_f) + +def get_git_repo_name(remote_url : str): + ''' + return repo name + ''' + url = remote_url.replace(".git","") + return os.path.basename(url) + +def add_git_suffix(remote : str): + ''' + add .git suffix to remote if needed + ''' + if remote.endswith(".git"): + return remote + + return remote + ".git" + +def get_base_oebuild(): + ''' + return oebuild base dir + ''' + return os.path.abspath(os.path.dirname(__file__)) + +def get_config_yaml_dir(): + ''' + return config yaml dir + ''' + return os.path.join(get_base_oebuild(), 'app/conf', CONFIG_YAML) + +def get_upgrade_yaml_dir(): + ''' + return upgrade yaml dir + ''' + return os.path.join(get_base_oebuild(), 'app/conf', UPGRADE_YAML) + +def generate_random_str(randomlength=16): + ''' + generate a random string by length + ''' + random_str = '' + base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789' + length = len(base_str) - 1 + for _ in range(randomlength): + random_str += base_str[random.randint(0, length)] + return random_str + +def get_time_stamp(): + ''' + get current timestamp + ''' + return time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) + +def get_oebuild_version(): + ''' + return oebuild version + ''' + return __version__ + +def check_docker(): + ''' + check docker had be installed or not + ''' + try: + DockerProxy() + except DockerException as exc: + raise ValueError(f''' +please install docker first, and run follow commands in root: +1, groupadd docker +2, usermod -a -G docker {getpass.getuser()} +3, systemctl daemon-reload && systemctl restart docker +4, chmod o+rw /var/run/docker.sock +''') from exc diff --git a/src/oebuild/version.py b/src/oebuild/version.py new file mode 100644 index 0000000000000000000000000000000000000000..dbe2c45d10808cd830f2b604babd857d077659b8 --- /dev/null +++ b/src/oebuild/version.py @@ -0,0 +1,13 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +__version__ = '0.0.1'