diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 9541ea3923d2086cb0aa8bf868545481a2ac9c5a..d7be84ef2d678e4d17c0c197a500437810fc90db 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -22,7 +22,7 @@ import logging import re -import urlparse +import urllib.parse as urlparse import requests import json import subprocess @@ -537,4 +537,4 @@ class ReleaseTagsFactory(object): return: class """ release_tags = ReleaseTagsFactory.VERSION_CTRL_GETTER_MAPPING.get(version_control, DefaultReleaseTags) - return release_tags(version_control) \ No newline at end of file + return release_tags(version_control) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index d4d9af9e72f6beb6d8a3eb7ac42354bfc128e17b..da04991c38809f129cd8d3f30ff415023d72871c 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -82,7 +82,7 @@ class CheckSCA(BaseCheck): # 保存详细结果到web server if not result: self.save_scanoss_result(scan.html) - logger.warning("click to view scanoss detail: {}".format(self._scanoss_result_repo_path)) + logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) return SUCCESS if result else FAILED diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 6502e91481a280afcc6158b4cad149a9b4d9d786..289969b4ed1b6c470fa2e7aeb986fc874a8bebe0 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -123,7 +123,7 @@ class CheckSpec(BaseCheck): if not homepage: return SUCCESS - for _ in xrange(retrying): + for _ in range(retrying): if 0 == do_requests("get", homepage, timeout=timeout): return SUCCESS time.sleep(interval) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 3432d4f579c83e9305b5f8eead33b17c835d510e..0f4527f0efaa78328b8da2cfe961890b759cbf8e 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -139,7 +139,7 @@ class GiteeRepo(object): if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue - for leading in xrange(max_leading + 1): + for leading in range(max_leading + 1): logger.debug("try dir {} -p{}".format(patch_dir, leading)) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 9beda0be37b02d3113688f19281e695b52723cc7..da9042783db5786a3c3e01d95732807a7ad9cb6c 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -136,7 +136,7 @@ class AC(object): :return: """ for filename in os.listdir(acl_dir): - if os.path.isdir(os.path.join(acl_dir, filename)): + if filename != "__pycache__" and os.path.isdir(os.path.join(acl_dir, filename)): self._ac_check_elements[filename] = {} # don't worry, using default when checking def load_check_elements_from_conf(self, conf_file, community): @@ -223,7 +223,7 @@ if "__main__" == __name__: logger = logging.getLogger("ac") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.community, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.community, args.repo)) logger.info("clone depth 4") logger.info("checking out pull request {}".format(args.pr)) @@ -254,7 +254,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -266,9 +265,7 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.scm.etime") dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) logger.info("fetch finished -") sys.exit(-1) else: @@ -299,6 +296,4 @@ if "__main__" == __name__: ac.save(args.output) dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 56ef4c09374fa97801f6fbaebcfed82061359eb1..442ebbbe4250f592dca24e1d713be61594d52293 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -315,9 +315,7 @@ if "__main__" == __name__: logging.getLogger("kafka").setLevel(logging.WARNING) # upload to es - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - #ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 98cccaa8270d9d0f1044d9df6f6b7d938ea668f3..eede4c63bafc0d3d110a3a3e3f7edb164bf61714 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -66,7 +66,7 @@ class SinglePackageBuild(object): 需要构建obs repo列表 :return: list """ - return OBSProxy.list_repos_of_arch(project, self._package, self._arch) + return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) def build_obs_repos(self, project, repos, work_dir, code_dir): """ @@ -251,7 +251,7 @@ if "__main__" == __name__: logger = logging.getLogger("build") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.owner, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.owner, args.repo)) logger.info("clone depth 1") logger.info("checking out pull request {}".format(args.pr)) @@ -272,7 +272,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -291,7 +290,6 @@ if "__main__" == __name__: query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(-1) else: @@ -311,6 +309,5 @@ if "__main__" == __name__: # upload to es query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(rs) diff --git a/src/dockerfile/README.md b/src/dockerfile/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b4af7377b2e475be71e561d7bccbd39df2697a35 --- /dev/null +++ b/src/dockerfile/README.md @@ -0,0 +1,31 @@ +# docker images +## openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts + +## openjdk 11-jdk-stretch based on openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts + +> Dockerfile: openjdk-openeuler + +## openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts + +> Dockerfile: ci-common + +## src-openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:base + +> Dockerfile: ci-soe-base + +> choose an openjdk image that from stable openeuler version + +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:{version} diff --git a/src/dockerfile/obs-base b/src/dockerfile/ci-base-ubuntu similarity index 100% rename from src/dockerfile/obs-base rename to src/dockerfile/ci-base-ubuntu diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv new file mode 100644 index 0000000000000000000000000000000000000000..b4169bbd874008fba4fe043443904a78e1b74852 --- /dev/null +++ b/src/dockerfile/ci-bisheng-riscv @@ -0,0 +1,106 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# Install qemu 5.2.0 +RUN set -eux; \ + yum update; yum clean all; \ + yum -y groupinstall "Development Tools"; \ + yum -y install pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ + cd /usr/src; \ + wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ + unzip ninja-linux.zip; \ + cp ninja /usr/bin; \ + wget -q http://121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ + tar xJf qemu-5.2.0.tar.xz; \ + cd qemu-5.2.0 && mkdir build && cd build; \ + ../configure --target-list=riscv64-linux-user,riscv64-softmmu; \ + make install -j; \ + rm -rf /usr/src/qemu-5.2.0* /usr/src/ninja*; + +# RISCV toolchain +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/riscv_toolchain_linux64.tar.xz; \ + tar xJf riscv_toolchain_linux64.tar.xz; \ + rm riscv_toolchain_linux64.tar.xz; + +# fedora sysroot +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/fedora28_riscv_smlroot.tar.xz; \ + tar xJf fedora28_riscv_smlroot.tar.xz; \ + ln -s /opt/fedora28_riscv_root/usr/lib64/ld-linux-riscv64-lp64d.so.1 /lib/ld-linux-riscv64-lp64d.so.1; \ + rm fedora28_riscv_smlroot.tar.xz; + +# Packages for openjdk build +RUN set -eux; \ + yum -y install java-1.8.0-openjdk-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXi-devel alsa-lib-devel libffi-devel elfutils-libelf-devel libstdc++-static libatomic + +# Test tool packages +RUN set -eux; \ + yum -y install gcc gcc-c++ unzip zlib-devel perl-devel libcurl-devel openssl-devel mercurial perl perl-Test-Simple xorg-x11-server-Xorg xorg-x11-server-Xvfb + +# JTREG +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-4.2.0-tip.tar.gz; \ + mkdir jtreg-4; tar -xf jtreg-4.2.0-tip.tar.gz -C jtreg-4 --strip-components 1; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-5.1-b01.tar.gz; \ + mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -C jtreg-5 --strip-components 1; \ + rm jtreg*.tar.gz; + +# AdoptOpenJDK 8/10/11/16 +RUN mkdir -p /usr/lib/jvm && cd /usr/lib/jvm; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK8U-jdk_x64_linux_hotspot_8u292b10.tar.gz -O jdk-8.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK10U-jdk_x64_linux_hotspot_10.0.2_13.tar.gz -O jdk-10.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK11U-jdk_x64_linux_hotspot_11.0.11_9.tar.gz -O jdk-11.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz -O jdk-16.tar.gz; \ + for tar in *.tar.gz; do tar xf $tar; done; \ + rm *.tar.gz; \ + ln -s jdk8* jdk-8; \ + ln -s jdk-10* jdk-10; \ + ln -s jdk-11* jdk-11; \ + ln -s jdk-16* jdk-16; + +ENV \ + JDK8_BOOT_DIR="/usr/lib/jvm/jdk-8" \ + JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \ + JDK11_BOOT_DIR="/usr/lib/jvm/jdk-11" \ + JDK16_BOOT_DIR="/usr/lib/jvm/jdk-16" \ + JTREG4_DIR="/opt/jtreg-4" \ + JTREG5_DIR="/opt/jtreg-5" + +# Jenkins Agent +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/jenkins-openeuler b/src/dockerfile/ci-common similarity index 74% rename from src/dockerfile/jenkins-openeuler rename to src/dockerfile/ci-common index 98ffa419b8a475cb358da0c1b71288e2162e4a9c..b99d662db71ce600fa0b4937cfb89491ecc30cc5 100644 --- a/src/dockerfile/jenkins-openeuler +++ b/src/dockerfile/ci-common @@ -1,5 +1,8 @@ # replace VERSION before run -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler-VERSION/openjdk:11-jdk-stretch +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace ARG VERSION=4.3 ARG user=jenkins @@ -13,7 +16,9 @@ RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins- && chmod 644 /usr/share/jenkins/agent.jar \ && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar -COPY jenkins-agent /usr/local/bin/jenkins-agent +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + RUN chmod a+rx /usr/local/openjdk-11 \ && chmod a+rx /usr/local/bin/jenkins-agent \ && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave @@ -33,5 +38,3 @@ WORKDIR ${AGENT_WORKDIR} ENTRYPOINT ["jenkins-agent"] - -# swr.cn-north-4.myhuaweicloud.com/openeuler/jenkins/openeuler:base diff --git a/src/dockerfile/ci-soe-base b/src/dockerfile/ci-soe-base new file mode 100644 index 0000000000000000000000000000000000000000..b6332ad5480ac339a5a788f7568d377ab8ebd834 --- /dev/null +++ b/src/dockerfile/ci-soe-base @@ -0,0 +1,47 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# golint not ready yet +RUN set -eux; \ + yum install -y curl sudo cpio bsdtar expect openssh shadow vim git strace python3 python3-pip build osc pylint + +# libabigail perl-XML-Structured not ready yet, use SP2 dailybuild +RUN set -eux; \ + arch="$(arch)"; \ + yum install -y http://121.36.53.23/SP2/libabigail-1.6-4.oe1.$arch.rpm; \ + yum install -y http://121.36.53.23/SP2/perl-XML-Structured-1.3-2.oe1.$arch.rpm + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/obs b/src/dockerfile/ci-ubuntu similarity index 100% rename from src/dockerfile/obs rename to src/dockerfile/ci-ubuntu diff --git a/src/dockerfile/openjdk-openeuler b/src/dockerfile/openjdk-openeuler index aaf591440970344bdbaf2421b89b9eb8caef92bc..7ca38a966437f405a91e46b429efeafd59e141a7 100644 --- a/src/dockerfile/openjdk-openeuler +++ b/src/dockerfile/openjdk-openeuler @@ -2,7 +2,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:VERSION RUN set -eux; \ - yum install -y tar wget python3 expect osc vim openssh shadow git + yum install -y tar wget # Default to UTF-8 file.encoding ENV LANG C.UTF-8 @@ -33,9 +33,11 @@ RUN set -eux; \ # this "case" statement is generated via "update.sh" case "$arch" in \ # arm64v8 - arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + arm64 | aarch64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_aarch64.tgz ;; \ # amd64 - amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ + amd64 | i386:x86-64 | x86_64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_x64.tgz ;; \ + #arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + #amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ # fallback *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 7ecab8a31bad19e651950d2bca4fab60e769149d..3978493b7cf94b3e3568e896d5e8a20d148b1472 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -68,7 +68,7 @@ class JenkinsJobs(object): """ batch = (len(target_jobs) + concurrency - 1) / concurrency _failed_jobs = [] - for index in xrange(batch): + for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) @@ -86,7 +86,7 @@ class JenkinsJobs(object): failed_jobs = run_once(real_jobs) - for index in xrange(retry): + for index in range(retry): if not failed_jobs: break logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index e2c644bc8f40d13540fe798d9263e0c06e6f09e9..87fe23294e550775cd673d8c38f17f7c03722f34 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -1,7 +1,24 @@ # -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: git api proxy +# ********************************************************************************** +""" + import os import logging -from cStringIO import StringIO +from io import StringIO import retrying from src.utils.shell_cmd import shell_cmd_live diff --git a/src/proxy/jenkins_patch.py b/src/proxy/jenkins_patch.py index 4fa3ef2b527651a66eb15be7476e3bed37cd75f9..1b73d44039ba7aabb985e946ade50b5778943758 100644 --- a/src/proxy/jenkins_patch.py +++ b/src/proxy/jenkins_patch.py @@ -1,5 +1,5 @@ # -*- encoding=utf-8 -*- -from urllib import quote as urlquote +from urllib.parse import quote from jenkinsapi.jenkinsbase import JenkinsBase @@ -11,7 +11,7 @@ def resolve_job_folders(self, jobs): jobs.remove(job) jobs += self.process_job_folder(job, self.baseurl) else: - job["url"] = '%s/job/%s' % (self.baseurl, urlquote(job['name'])) + job["url"] = '%s/job/%s' % (self.baseurl, quote(job['name'])) return jobs diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index 9ff7ff74e87da603a255570bf399a64f00c9fbeb..c7dcde44ca59280cd1bf6ba5a9591a6f853d0aa5 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -37,6 +37,7 @@ class KafkaProducerProxy(object): self._timeout = timeout self._kp = kafka.KafkaProducer(bootstrap_servers=brokers, + key_serializer=str.encode, value_serializer=lambda v:json.dumps(v).encode("utf-8")) def send(self, topic, key=None, value=None): diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index eee8ad139bcdf070d7565678d533e871052c8cc1..935d944c69e454b63159afbbb72ec8e0478586e8 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -51,14 +51,15 @@ class OBSProxy(object): return rs @staticmethod - def list_repos_of_arch(project, package, arch): + def list_repos_of_arch(project, package, arch, show_exclude=False): """ 获取包的repo列表 :param project: :param package: :return: """ - cmd = "osc results {} {} -a {}".format(project, package, arch) + cmd = "osc results {} {} {} -a {}".format( + "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: logger.debug("list obs repos of arch error, {}".format(ret)) @@ -120,7 +121,7 @@ class OBSProxy(object): :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean".format( + cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice".format( package_path, repo, arch, "--disable-debuginfo" if not debug else "") logger.info("osc build {} {} {} --no-verify --clean".format( diff --git a/src/requirements b/src/requirements index bb5b47e47ef3df8a08b269e06c174222d09b72cf..d3fd573f2d4135e8280d57b07b7cd8b4112a154b 100644 --- a/src/requirements +++ b/src/requirements @@ -3,11 +3,12 @@ jenkinsapi colorlog threadpool PyYAML -gevent==1.2.2 +gevent jsonpath mock tldextract chardet -kafka +kafka-python elasticsearch retrying +scanoss-scanner diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index d3bc10cc92b5b8cb2678e8bd20326d23724438cd..dec8843642ea5e16214d23e60f5979f0424da36d 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -42,6 +42,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False while True: line = p.stdout.readline() if line: + line = line.decode("utf-8") line = line.strip() no_fmt_logger.info(line) if verbose else no_fmt_logger.debug(line) if cap_out and line and line != "\n":