From d74d8577a0f60afae0809cbdd224a1bd928611db Mon Sep 17 00:00:00 2001 From: lisimin Date: Mon, 28 Mar 2022 19:35:17 +0800 Subject: [PATCH 1/2] support scp from building host Signed-off-by: lisimin --- script/tools/copy_embedded_img.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/script/tools/copy_embedded_img.sh b/script/tools/copy_embedded_img.sh index 2c5396b..f5a4e52 100644 --- a/script/tools/copy_embedded_img.sh +++ b/script/tools/copy_embedded_img.sh @@ -11,6 +11,7 @@ function cp_embedded_img() dest_ip=$5 ssh_key=$6 latest_iso_dir=$7 + local copydir="${source_dir}" res=$(ssh -i ${ssh_key} -o StrictHostKeyChecking=no -o LogLevel=ERROR -o ServerAliveInterval=60 root@${dest_ip} " cd ${latest_iso_dir} if [ ! -d embedded_img ];then @@ -23,10 +24,16 @@ fi if [ -n "$res" ];then echo $res fi - mkdir tmpdir + echo source_user="${source_user}" + if [[ -z "${source_user}" ]];then + echo "copy files at this host" + ret=0 + else + copydir="tmpdir" + mkdir "${copydir}" expect <<-END1 set timeout 3600 - spawn scp -o StrictHostKeyChecking=no -o LogLevel=ERROR -o ServerAliveInterval=60 -r ${source_user}@${source_ip}:${source_dir}/* ./tmpdir/ + spawn scp -o StrictHostKeyChecking=no -o LogLevel=ERROR -o ServerAliveInterval=60 -r ${source_user}@${source_ip}:${source_dir}/* ./${copydir} expect { -re "\[P|p]assword:" { send "${source_pwd}\r" @@ -41,11 +48,15 @@ fi exit [lindex \${result} 3] } END1 - if [ $? -ne 0 ];then + ret=$? + fi + if [ $ret -ne 0 ];then echo "[ERROR]: scp failed." exit 1 else - scp -i ${ssh_key} -o StrictHostKeyChecking=no -o LogLevel=ERROR -o ServerAliveInterval=60 -r tmpdir/* root@${dest_ip}:${latest_iso_dir}/embedded_img/ + find ${copydir}/ -type f + echo "==================================" + scp -v -i ${ssh_key} -o StrictHostKeyChecking=no -o LogLevel=ERROR -o ServerAliveInterval=60 -r "${copydir}"/* root@${dest_ip}:${latest_iso_dir}/embedded_img/ if [ $? -ne 0 ];then echo "[ERROR]: scp embedded_img to dailybuild failed." exit 1 @@ -56,4 +67,4 @@ fi fi } -cp_embedded_img $@ +cp_embedded_img "$@" -- Gitee From a898cd25e5737e9d0cd8500bd9fa24ccbc2ebe0b Mon Sep 17 00:00:00 2001 From: lisimin Date: Fri, 29 Apr 2022 10:29:08 +0800 Subject: [PATCH 2/2] add scripts to compile openeuler embedded version Signed-off-by: lisimin --- script/embedded/compile/build.sh | 107 +++++++++++++++++++ script/embedded/compile/download.sh | 153 ++++++++++++++++++++++++++++ script/embedded/crontab.txt | 2 + 3 files changed, 262 insertions(+) create mode 100755 script/embedded/compile/build.sh create mode 100755 script/embedded/compile/download.sh create mode 100644 script/embedded/crontab.txt diff --git a/script/embedded/compile/build.sh b/script/embedded/compile/build.sh new file mode 100755 index 0000000..7b63de4 --- /dev/null +++ b/script/embedded/compile/build.sh @@ -0,0 +1,107 @@ +#!/bin/bash +SCRIPTS_DIR=$(cd $(dirname $0);pwd) +set -x +set -e + +set_env() +{ +PLATFORM="$1" +SRC_DIR="/usr1/openeuler/src" +[[ -z "${PLATFORM}" ]] && PLATFORM="aarch64-std" +#PLATFORM="qemu-arm" +#PLATFORM="qemu-aarch64" +BUILD_DIR="${SRC_DIR}/build/build-${PLATFORM}" +if [ "$(whoami)" == "root" ];then + useradd -m openeuler || echo "" + username="openeuler" + chown -R ${username}:users "${SRC_DIR}" +fi +chown -R ${username}:users "${SRC_DIR}" +ROOTFS_DIR="/usr1/openeuler/rootfs/${PLATFORM}" +OUTPUT_DIR="/usr1/output/${PLATFORM}" +TOP_OUTPUT_DIR="/usr1/output/" + case $PLATFORM in + "aarch64-std" | "aarch64-pro" | "raspberrypi4-64") + group="arm64";; + "arm-std") + group="arm32";; + *) + echo "unknown machine";exit 1 + esac +} + +create_checksum_for_release() +{ + local releasedir="$1" + pushd "${releasedir}" + local filelist="$(find . -type f | grep -Ev "\.sha256sum")" + for fname in $filelist + do + sha256sum "${fname}" | awk -F"[ /]" '{print $1,$NF}' > "${fname}".sha256sum + done + popd + +} + +main() +{ +whoami +local archlist="$1" +Is_delete_worspace="$2" +local is_download_code="$3" +local is_install_tools="$4" +[[ -z "$is_download_code" ]] && { is_download_code="yes"; } +[[ -z "$is_install_tools" ]] && { is_install_tools="no"; } +chmod a+r "${SCRIPTS_DIR}"/*.sh +##download codes and install tools +local datetime="$(date +%Y%m%d%H%M%S)" +[[ "${is_download_code}" == "yes" ]] && sh -x "${SCRIPTS_DIR}"/download.sh "$is_install_tools" +#[[ -z "${archlist}" ]] && archlist="qemu-arm qemu-aarch64 raspberrypi4-64" +[[ -z "${archlist}" ]] && archlist="arm-std aarch64-std aarch64-pro raspberrypi4-64" +#delete log file from dnf +rm -f /tmp/hawkey.log +for arch in $archlist +do + set_env "$arch" + arch="$(echo $arch|sed "s|qemu-||g")" + mkdir -p "${TOP_OUTPUT_DIR}/${datetime}/source-list/" + ##compile + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + chown -R "${username}":users "${BUILD_DIR}" + if [[ "${PLATFORM}" == "aarch64-std" ]];then + sh "${SRC_DIR}"/yocto-meta-openeuler/scripts/compile.sh "dsoftbus" + fi + cat > "${BUILD_DIR}"/build.sh << EOF +#!/bin/bash +set -e +whoami || echo "" +rm -f "${BUILD_DIR}"/source.log +export DATETIME="$datetime" +source "${SRC_DIR}"/yocto-meta-openeuler/scripts/compile.sh "${PLATFORM}" "${BUILD_DIR}" > "${BUILD_DIR}"/source.log +bitbake_opt="\$(grep "You can now run " ${BUILD_DIR}/source.log | awk -F"'" '{print \$2}')" +#\${bitbake_opt} +echo "bitbake openeuler-image" +bitbake openeuler-image +EOF + + sudo -E -u "${username}" sh -x "${BUILD_DIR}"/build.sh || exit 1 + local destdir="${TOP_OUTPUT_DIR}/${datetime}/${group}/${arch}" + mkdir -p "$destdir" || exit 1 + cp -a "${BUILD_DIR}"/output/${datetime}/* "$destdir" || exit 1 + [[ "${Is_delete_worspace}" == "yes" ]] && rm -rf "${BUILD_DIR}" +done + +pushd "${TOP_OUTPUT_DIR}/" +test -d ./dailybuild || mkdir ./dailybuild +pushd ./dailybuild || exit 1 +rm -rf * +ln -sf ../"${datetime}" "${datetime}" +popd +popd +cp -a "${SRC_DIR}"/manifest.xml "${TOP_OUTPUT_DIR}/${datetime}/source-list/" +create_checksum_for_release "${TOP_OUTPUT_DIR}/${datetime}" +echo "INFO: ALL successfully!" +} + +main "$@" diff --git a/script/embedded/compile/download.sh b/script/embedded/compile/download.sh new file mode 100755 index 0000000..b38cba1 --- /dev/null +++ b/script/embedded/compile/download.sh @@ -0,0 +1,153 @@ +#!/bin/bash +SCRIPTS_DIR=$(cd $(dirname $0);pwd) +set -xe + +download_gcc() +{ + ls "${GCC_INSTALL_DIR}"/openeuler_gcc_arm32le "${GCC_INSTALL_DIR}"/openeuler_gcc_arm64le && return 0 + local url="https://gitee.com/openeuler/yocto-embedded-tools/attach_files/1003463/download/openeuler_gcc_arm32le.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" + local url="https://gitee.com/openeuler/yocto-embedded-tools/attach_files/1003462/download/openeuler_gcc_arm64le.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" + + return 0 + + local url="https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/aarch64-linux-gnu/gcc-linaro-7.3.1-2018.05-x86_64_aarch64-linux-gnu.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" + url="https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/aarch64-linux-gnu/sysroot-glibc-linaro-2.25-2018.05-aarch64-linux-gnu.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" + url="https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabi/sysroot-glibc-linaro-2.25-2018.05-arm-linux-gnueabi.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" + url="https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabi/gcc-linaro-7.3.1-2018.05-x86_64_arm-linux-gnueabi.tar.xz" + test -f "${TOOLS_DIR}/$(basename ${url})" || wget "${url}" -P "${TOOLS_DIR}" +} + +install_gcc_tar() +{ + local install_dir="$1" + [[ -z "${install_dir}" ]] && install_dir="/usr1/openeuler/gcc" + test -d "${install_dir}" || mkdir -p "${install_dir}" + pushd "${install_dir}" + for i in $(ls "${TOOLS_DIR}"/*.tar.*) + do + local unpackdir="$(tar -tf "$i" | awk -F/ '{print $1}' | uniq)" + test -d ./"${unpackdir}" || tar -xf "$i" + find ./"${unpackdir}" -type d | xargs chmod go+x + chmod go+r ./"${unpackdir}" -R + done + popd +} + +update_code_repo() +{ + local repo="$1" + local branch="-b $2" + local realdir="$3" + local pkg="$(basename ${repo})" + local branchname="$2" + [[ -z "${realdir}" ]] || pkg="$(basename ${realdir})" + [[ "${pkg}" == "kernel-5.10" ]] && local git_param="--depth 1" + pushd "${SRC_DIR}" + test -d ./"${pkg}"/.git || { rm -rf ./"${pkg}";git clone "${URL_PREFIX}/${repo}" ${branch} ${git_param} -v "${pkg}"; } + pushd ./"${pkg}" + git checkout origin/${branchname} -b ${branchname} || echo "" + git checkout -f ${branchname} + git branch | grep "^*" | grep " ${branchname}$" || exit 1 + git config pull.ff only + while true + do + git reset --hard HEAD^ || echo "" + git reset --hard HEAD + git clean -dfx + git pull + git status | grep "is up to date with" && break + done + local newest_commitid="$(git log --pretty=oneline -n1 | awk '{print $1}')" + echo "${repo} ${newest_commitid}" >> "${SRC_DIR}"/code.list + popd + popd +} + +download_code() +{ + mkdir -p "${SRC_DIR}"/ + rm -f "${SRC_DIR}"/code.list + update_code_repo openeuler/yocto-meta-openeuler openEuler-22.03-LTS +} + +install_python() +{ +wget https://www.python.org/ftp/python/3.9.2/Python-3.9.2.tgz +tar -xf Python-3.9.2.tgz +install_dir="/opt/buildtools/python-3.9.2/" +rm -rf "${install_dir}" +rm /usr/local/bin/python3 /usr/local/bin/python +cd Python-3.9.2 +./configure --prefix=/opt/buildtools/python-3.9.2 --enable-loadable-sqlite-extensions +make -j 8 && make install +ln -s /opt/buildtools/python-3.9.2/bin/python3 /usr/local/bin/python3 +ln -s /opt/buildtools/python-3.9.2/bin/python3 /usr/local/bin/python +#ModuleNotFoundError: No module named '_socket' +#export PYTHON_HOME="/opt/buildtools/python-3.9.2/" +export PYTHONPATH=/opt/buildtools/python-3.9.2/lib64/python3.9/lib-dynload/ +export PYTHONPATH="/opt/buildtools/python-3.9.2/lib/python3.9/site-packages/:/opt/buildtools/python-3.9.2/:/opt/buildtools/python-3.9.2/lib64/python3.9/lib-dynload/" +} + +install_ninja() +{ +"${TOOLS_INSTALL_DIR}"/ninja-1.10.1/bin/ninja --version && return 0 +wget https://distfiles.macports.org/ninja/ninja-1.10.1.tar.gz +tar -xf ninja-1.10.1.tar.gz +cd ninja-1.10.1 +ls -l +sed -ie '1c#!/usr/bin/env python3' *.py +./configure.py --bootstrap +mkdir -p "${TOOLS_INSTALL_DIR}"/ninja-1.10.1/bin || echo "" +install -m 0755 ./ninja "${TOOLS_INSTALL_DIR}"/ninja-1.10.1/bin +cd - +} + +install_sphinx() +{ +yum -y install sphinx + +#Installing: +# sphinx x86_64 2.2.11-1.oe1 everything 4.4 M +#Installing dependencies: +# apr x86_64 1.7.0-4.oe1 OS 109 k +# apr-util x86_64 1.6.1-14.oe1 OS 110 k +# gpdb x86_64 6.17.0-3.oe1 update 11 M +# net-tools x86_64 2.0-0.54.oe1 OS 198 k +# rsync x86_64 3.1.3-6.oe1 OS 323 k +# xerces-c x86_64 3.2.2-3.oe1 everything 916 k + +} + +#zypper install autoconf automake chrpath patch +TOOLS_DIR="/usr1/openeuler/src/tools" +SRC_DIR="/usr1/openeuler/src/" +GCC_INSTALL_DIR="/usr1/openeuler/gcc" +TOOLS_INSTALL_DIR="/opt/buildtools" +URL_PREFIX="https://gitee.com/" + +is_install_tools=$1 +if [[ "$is_install_tools" == "yes" ]];then +download_gcc +#install_ninja +install_gcc_tar "${GCC_INSTALL_DIR}" +#zypper --non-interactive install rpmbuild libtool patchelf autoconf automake chrpath patch +###depends by python +#zypper --non-interactive install libffi-devel sqlite-devel xz-devel +#for openeuler +yum -y install tar cmake gperf sqlite-devel +yum -y install chrpath gcc-c++ patch rpm-build flex autoconf automake m4 bison bc libtool gettext-devel createrepo_c rpcgen texinfo hostname python meson dosfstools mtools parted ninja-build autoconf-archive libmpc-devel gmp-devel numactl-devel make +#dsoftbus need python3 to build +which python +python --version | grep " 3" || ln -sf /usr/bin/python3 /usr/local/bin/python +fi + +download_code +#download kernel by tag, cannot change tag automatically +rm -rf "${SRC_DIR}"/kernel-5.10 +sh "${SRC_DIR}"/yocto-meta-openeuler/scripts/download_code.sh +exit $? diff --git a/script/embedded/crontab.txt b/script/embedded/crontab.txt new file mode 100644 index 0000000..4b0e20f --- /dev/null +++ b/script/embedded/crontab.txt @@ -0,0 +1,2 @@ +00 0 * * * cd /usr1/scripts/scripts && git pull >/usr1/build.log 2>&1 +05 0 * * * bash /usr1/scripts/scripts/build.sh "" yes >>/usr1/build.log 2>&1 -- Gitee