diff --git a/src/README.md b/src/README.md index dfca4a8f68c6725994c36297aa67d3841a758894..f371a1e68eac3059247fba7a2f00132c76c67b4b 100644 --- a/src/README.md +++ b/src/README.md @@ -63,3 +63,15 @@ 多arch支持 > docker manifest push时Registry Credentials? + +## 目录结构 +| 目录 | 描述 | +| --- | --- | +|ac/framework | 门禁框架 | +|ac/acl | 门禁任务，每个门禁项对应一个目录 | +|ac/common | 门禁通用代码 | +|build| 单包构建| +|jobs| jenkins任务管理| +|conf|配置| +|proxy|第三方接口代理| +|utils|通用代码，日志等| diff --git a/src/ac/acl/package_license/__init__.py b/src/ac/acl/package_license/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..36e95ff3db7d173ff656de1605ce299b4e77f17a --- /dev/null +++ b/src/ac/acl/package_license/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-10-16 +# Description: check spec file +# ********************************************************************************** +""" \ No newline at end of file diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py new file mode 100644 index 0000000000000000000000000000000000000000..b73cdc0d7380de4897cbb22df3b6728465bb569a --- /dev/null +++ b/src/ac/acl/package_license/check_license.py @@ -0,0 +1,116 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-10-16 +# Description: check spec file +# ********************************************************************************** +""" + +import logging +import time +import os +import yaml +import shutil + +from src.proxy.git_proxy import GitProxy +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.framework.ac_base import BaseCheck +from src.ac.common.rpm_spec_adapter import RPMSpecAdapter +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.acl.package_license.package_license import PkgLicense + +logger = logging.getLogger("ac") + +class CheckLicense(BaseCheck): + """ + check license in spec and src-code + """ + + def __init__(self, workspace, repo, conf=None): + super(CheckLicense, self).__init__(workspace, repo, conf) + + self._gp = GitProxy(self._work_dir) + self._work_tar_dir = os.path.join(workspace, "code") + self._gr = GiteeRepo(self._repo, self._work_dir, self._work_tar_dir) + if self._gr.spec_file: + self._spec = RPMSpecAdapter(os.path.join(self._work_dir, self._gr.spec_file)) + else: + self._spec = None + + self._pkg_license = PkgLicense() + self._license_in_spec = set() + self._license_in_src = set() + + def check_license_in_spec(self): + """ + check whether the license in spec file is in white list + :return + """ + if self._spec is None: + logger.warning("spec file not find") + return WARNING + self._license_in_spec = self._gr.scan_license_in_spec(self._spec) + self._license_in_spec = self._pkg_license.translate_license(self._license_in_spec) + if self._pkg_license.check_license_safe(self._license_in_spec): + return SUCCESS + else: + logger.warning("licenses in spec are not in white list") + return WARNING + + def check_license_in_src(self): + """ + check whether the license in src file is in white list + :return + """ + self._license_in_src = self._pkg_license.scan_licenses_in_license(self._work_tar_dir) + self._license_in_src = self._pkg_license.translate_license(self._license_in_src) + if not self._license_in_src: + logger.warning("can't find license in src code") + return WARNING + if self._pkg_license.check_license_safe(self._license_in_src): + return SUCCESS + else: + logger.warning("licenses in src code are not in white list") + return WARNING + + def check_license_is_same(self): + """ + check whether the license in spec file and in src file is same + :return + """ + if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src): + logger.info("licenses in src:{} and in spec:{} are same".format(self._license_in_src, + self._license_in_spec)) + return SUCCESS + else: + logger.warning("licenses in src:{} and in spec:{} are not same".format(self._license_in_src, + self._license_in_spec)) + return WARNING + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check {} license ...".format(self._repo)) + + _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) + self._gr.decompress_all() # decompress all compressed file into work_tar_dir + self._pkg_license.load_config() # load license config into instance variable + + try: + return self.start_check_with_order("license_in_spec", "license_in_src", "license_is_same") + finally: + shutil.rmtree(self._work_tar_dir) \ No newline at end of file diff --git a/src/ac/acl/package_license/config/license_list b/src/ac/acl/package_license/config/license_list new file mode 100644 index 0000000000000000000000000000000000000000..13861c91eeee86db0f8ca2ff84114d4b0d7b4e97 --- /dev/null +++ b/src/ac/acl/package_license/config/license_list @@ -0,0 +1,451 @@ +0BSD,white +AAL,white +ADSL,black +AFL, white +AFL-1.1,white +AFL-1.2,white +AFL-2.0,white +AFL-2.1,white +AFL-3.0,white +AFLv2.1, white +AGPL-1.0-only,black +AGPL-1.0-or-later,black +AGPL-3.0, white +AGPL-3.0-only,black +AGPL-3.0-or-later,black +AGPLv3, white +AGPLv3+, white +AMDPLPA,black +AML,black +AMPAS,black +AND, black +ANTLR-PD,black +APAFML,black +APL-1.0,black +APSL-1.0,black +APSL-1.1,black +APSL-1.2,black +APSL-2.0,white +ASL-1.0, white +ASL-1.1, white +ASL-2.0, white +Abstyles,black +Adobe-2006,black +Adobe-Glyph,black +Afmparse,black +Aladdin,black +Apache, white +Apache-1.1,white +Apache-2, white +Apache-2.0,white +Arphic, black +Artistic, white +Artistic-1.0,black +Artistic-1.0-Perl,black +Artistic-1.0-cl8,black +Artistic-2.0,white +BSD, white +BSD-1-Clause,white +BSD-2-Clause,white +BSD-2-Clause-Patent,white +BSD-3-Clause,white +BSD-3-Clause-Attribution,black +BSD-3-Clause-LBNL,black +BSD-3-Clause-No-Nuclear-License,black +BSD-3-Clause-No-Nuclear-License-2014,black +BSD-3-Clause-No-Nuclear-Warranty,black +BSD-3-Clause-Open-MPI,black +BSD-4-Clause-UC,black +BSD-Protection,black +BSD-Source-Code,black +BSD-licenced-3, white +BSD3, white +BSL-1.0,white +Bahyph,black +Barr,black +Beerware,black +BitTorrent-1.0,black +BlueOak-1.0.0,black +Boost, white +Borceux,black +CAL-1.0,black +CAL-1.0-Combined-Work-Exception,black +CATOSL-1.1,white +CC-BY, black +CC-BY-1.0,black +CC-BY-2.0,black +CC-BY-2.5,black +CC-BY-3.0,black +CC-BY-4.0,white +CC-BY-NC-1.0,black +CC-BY-NC-2.0,black +CC-BY-NC-2.5,black +CC-BY-NC-3.0,black +CC-BY-NC-4.0,black +CC-BY-NC-ND-1.0,black +CC-BY-NC-ND-2.0,black +CC-BY-NC-ND-2.5,black +CC-BY-NC-ND-3.0,black +CC-BY-NC-ND-4.0,black +CC-BY-NC-SA-1.0,black +CC-BY-NC-SA-2.0,black +CC-BY-NC-SA-2.5,black +CC-BY-NC-SA-3.0,black +CC-BY-NC-SA-4.0,black +CC-BY-ND-1.0,black +CC-BY-ND-2.0,black +CC-BY-ND-2.5,black +CC-BY-ND-3.0,black +CC-BY-ND-4.0,black +CC-BY-SA, black +CC-BY-SA-1.0,black +CC-BY-SA-2.0,black +CC-BY-SA-2.5,black +CC-BY-SA-3.0,black +CC-BY-SA-4.0,white +CC-PDDC,black +CC0, black +CC0-1.0,white +CDDL, white +CDDL-1.0,white +CDDL-1.1,black +CDLA-Permissive-1.0,black +CDLA-Sharing-1.0,black +CECILL-1.0,black +CECILL-1.1,black +CECILL-2.1,black +CERN-OHL-1.1,black +CERN-OHL-1.2,black +CERN-OHL-P-2.0,black +CERN-OHL-S-2.0,black +CERN-OHL-W-2.0,black +CNRI-Jython,black +CNRI-Python,white +CNRI-Python-GPL-Compatible,black +CPAL-1.0,white +CPL, white +CPL-1.0,white +CPOL-1.02,black +CUA-OPL-1.0,black +Caldera,black +Commons Clause,black +Commons Clause 1.0, white +Crossword,black +CrystalStacker,black +Cube,black +D-FSL-1.0,black +DMIT,black +DMTF, black +DOC,black +DSDP,black +Dotseqn,black +ECL-1.0,black +ECL-2.0,white +EFL-1.0,black +EFL-2.0,white +EPL, white +EPL-1.0,white +EPL-2.0,white +EUDatagrid,white +EUPL-1.0,black +EUPL-1.1,white +EUPL-1.2,white +Entessa,black +ErlPL-1.1,black +Eurosym,black +FSFAP, black +FSFUL,black +FSFULLR,black +FTL, black +Fair,white +Frameworx-1.0,black +FreeImage,black +GFDL-1.1, white +GL2PS,black +GPL, white +GPL+, white +GPL-1.0, white +GPL-1.0+, white +GPL-1.0-only,black +GPL-1.0-or-later,black +GPL-2, white +GPL-2.0, white +GPL-2.0+, white +GPL-2.0-only,white +GPL-2.0-or-later,white +GPL-3.0, white +GPL-3.0+, white +GPL-3.0-only,white +GPL-3.0-or-later,white +GPL2, white +GPLV2, white +GPLV2+, white +GPLv1+, white +GPLv2, white +GPLv2+, white +GPLv2-or-later, white +GPLv3, white +GPLv3+, white +Giftware,black +Glide,black +Glulxe,black +HPND,white +HPND-sell-variant,black +HaskellReport,black +Hippocratic-2.1,black +IBM, black +IBM-pibs,black +ICU,black +IEEE, black +IJG,white +IPA,white +IPL-1.0,white +ISC,white +ImageMagick,black +Info-ZIP,black +Intel,white +Intel-ACPI,black +Interbase, black +Interbase-1.0,black +JPNIC,black +JSON,black +JasPer-2.0,black +Knuth, black +LAL-1.2,black +LAL-1.3,black +LGPL-2.0, white +LGPL-2.0+, white +LGPL-2.0-only,white +LGPL-2.0-or-later,white +LGPL-2.1, white +LGPL-2.1+, white +LGPL-2.1-only,white +LGPL-2.1-or-later,white +LGPL-3.0+, white +LGPL-3.0-only,white +LGPL-3.0-or-later,white +LGPLLR,black +LGPLv2, white +LGPLv2+, white +LGPLv2.1, white +LGPLv2.1+, white +LGPLv3, white +LGPLv3+, white +LPL-1.0,black +LPL-1.02,white +LPPL, white +LPPL-1.0,black +LPPL-1.1,black +LPPL-1.3c,white +Latex2e,black +Leptonica,black +LiLiQ-P-1.1,black +LiLiQ-R-1.1,black +LiLiQ-Rplus-1.1,black +Liberation, black +Libpng,black +Licence-2.0, black +Linux-OpenIB,black +Lucida, black +MIT,white +MIT-0,black +MIT-CMU,black +MIT-advertising,black +MIT-enna,black +MIT-feh,black +MITNFA,black +MPL-1.0,white +MPL-1.1,white +MPL-2, white +MPL-2.0,white +MPL-2.0-no-copyleft-exception,black +MPL2, white +MPLv1.1, white +MPLv2.0, white +MS-PL,white +MS-RL,white +MTLL,black +MakeIndex,black +MirOS,white +Motosoto,white +Mulan 2.0, white +Mulan-PSL-1, white +Mulan-PSL-2, white +MulanPSL-1.0,black +MulanPSL-2.0,white +Multics,black +Mup,black +NASA-1.3,black +NBPL-1.0,black +NCGL-UK-2.0,black +NCSA,white +NGPL,white +NLOD-1.0,black +NLPL,black +NPOSL-3.0,black +NRL,black +NTP,white +NTP-0,black +Naumen,white +Net-SNMP,black +NetCDF,black +Netscape, black +Newsletr,black +Nmap, black +Nokia,white +Noweb,black +O-UDA-1.0,black +OCCT-PL,black +OCLC-2.0,black +ODC-By-1.0,black +OFL, white +OFL-1.0-RFN,black +OFL-1.0-no-RFN,black +OFL-1.1,white +OFL-1.1-RFN,black +OFL-1.1-no-RFN,black +OFSFDL, black +OGC-1.0,black +OGL-Canada-2.0,black +OGL-UK-1.0,black +OGL-UK-2.0,black +OGL-UK-3.0,black +OGTSL,black +OLDAP-1.1,black +OLDAP-1.2,black +OLDAP-1.3,black +OLDAP-1.4,black +OLDAP-2.0,black +OLDAP-2.0.1,black +OLDAP-2.1,black +OLDAP-2.2,black +OLDAP-2.2.1,black +OLDAP-2.2.2,black +OLDAP-2.4,black +OLDAP-2.5,black +OLDAP-2.6,black +OLDAP-2.8,black +OML,black +OPL-1.0,black +OSET-PL-2.1,black +OSL-1.0,white +OSL-2.0,white +OSL-2.1,white +OSL-3.0,white +OpenLDAP, black +OpenSSL,white +PDDL-1.0,black +PHP, white +PHP-3.0,white +PSF, black +PSF-2.0,black +Parity-6.0.0,black +Parity-7.0.0,black +Perl, white +Plexus,black +PolyForm-Noncommercial-1.0.0,black +PolyForm-Small-Business-1.0.0,black +PostgreSQL,white +Pubilc, black +Python, white +Python-2.0,white +QPL, white +QPL-1.0,white +Qhull,black +RHeCos-1.1,black +RPL-1.1,black +RPL-1.5,black +RPSL-1.0,white +RSA-MD,black +RSCPL,black +Rdisc,black +Ruby,white +SAX-PD,black +SCEA,black +SGI-B-1.0,black +SGI-B-1.1,black +SHL-0.5,black +SHL-0.51,black +SISSL,white +SISSL-1.2,black +SMPPL,black +SNIA,black +SPL-1.0,white +SSH-OpenSSH,black +SSH-short,black +SSLeay, black +SSPL-1.0,black +SWL,black +Saxpath,black +Sendmail,black +Sendmail-8.23,black +SimPL-2.0,black +Sleepycat,white +Spencer-86,black +Spencer-94,black +Spencer-99,black +SugarCRM-1.1.3,black +TAPR-OHL-1.0,black +TCGL, black +TCL,white +TCP-wrappers,black +TMate,black +TORQUE-1.1,black +TOSL,black +TTWL, black +TU-Berlin-1.0,black +TU-Berlin-2.0,black +UCD,white +UCL-1.0,black +UPL-1.0,white +Unicode, black +Unicode-DFS-2015,black +Unicode-DFS-2016,black +Unicode-TOU,black +Utopia, black +VOSTROM,black +VSL-1.0,white +Verbatim, black +Vim, black +W3C,white +W3C-19980720,black +W3C-20150513,black +WTFPL, black +Watcom-1.0,black +Wsuipa,black +XSkat,black +Xerox,black +Xnet,black +YPL-1.0,black +ZPL-1.1,black +ZPL-2.0,white +ZPL-2.1,white +ZPLv2.0, white +ZPLv2.1, white +Zed,black +Zimbra-1.4,black +Zlib,white +artistic,black +artistic-2.0,black +blessing,black +bzip2-1.0.5,black +bzip2-1.0.6,black +copyleft-next-0.3.0,black +copyleft-next-0.3.1,black +curl,black +diffmark,black +dvipdfm,black +eGenix,black +etalab-2.0,black +gSOAP-1.3b,black +gnuplot, black +libpng-2.0,black +libselinux-1.0,black +libtiff,black +mpich2,black +psfrag,black +psutils,black +xinetd, black +xpp,black +zlib, white +zlib-acknowledgement,white \ No newline at end of file diff --git a/src/ac/acl/package_license/config/license_translations b/src/ac/acl/package_license/config/license_translations new file mode 100644 index 0000000000000000000000000000000000000000..7472de792d7fc5b5b397fd18d65dc0613e1ec374 --- /dev/null +++ b/src/ac/acl/package_license/config/license_translations @@ -0,0 +1,457 @@ +(MIT), MIT +(mit), MIT +(per1), Artistic-1.0-Perl +2-clause, BSD-2-Clause +3dfx Glide License,Glide +AGPL-3, AGPL-3.0 +AMD's plpa_map.c License,AMDPLPA +ANTLR Software Rights Notice,ANTLR-PD +APL-2.0, Apache-2.0 +APL2, Apache-2.0 +APL2.0, Apache-2.0 +ASL 2.0, Apache-2.0 +ASL-2, Apache-2.0 +ASL-2.0, Apache-2.0 +Abstyles License,Abstyles +Academic Free License v1.1,AFL-1.1 +Academic Free License v1.2,AFL-1.2 +Academic Free License v2.0,AFL-2.0 +Academic Free License v2.1,AFL-2.1 +Academic Free License v3.0,AFL-3.0 +Academy of Motion Picture Arts and Sciences BSD,AMPAS +Adaptive Public License 1.0,APL-1.0 +Adobe Glyph List License,Adobe-Glyph +Adobe Postscript AFM License,APAFML +Adobe Systems Incorporated Source Code License Agreement,Adobe-2006 +Affero General Public License v1.0 only,AGPL-1.0-only +Affero General Public License v1.0 or later,AGPL-1.0-or-later +Afmparse License,Afmparse +Aladdin Free Public License,Aladdin +Amazon Digital Services License,ADSL +Apache, Apache-2.0 +Apache 2.0, Apache-2.0 +Apache License 1.1,Apache-1.1 +Apache License 2.0, Apache-2.0 +Apache License, Version 2.0, Apache-2.0 +Apache-2, Apache-2.0 +Apache2.0, Apache-2.0 +Apachev2, Apache-2.0 +Apple MIT License,AML +Apple Public Source License 1.0,APSL-1.0 +Apple Public Source License 1.1,APSL-1.1 +Apple Public Source License 1.2,APSL-1.2 +Apple Public Source License 2.0,APSL-2.0 +Artistic License, Artistic +Artistic License 1.0,Artistic-1.0 +Artistic License 1.0 (Perl),Artistic-1.0-Perl +Artistic License 1.0 w/clause 8,Artistic-1.0-cl8 +Artistic License 2.0,Artistic-2.0 +Artistic license,artistic +Artistic license 2.0,artistic-2.0 +Artistic-1.0+GPL-1.0, Artistic-1.0 GPL-1.0 +Attribution Assurance License,AAL +BSD, BSD +BSD 1-Clause License,BSD-1-Clause +BSD 2-Clause "Simplified" License,BSD-2-Clause +BSD 3-Clause "New" or "Revised" License,BSD-3-Clause +BSD 3-Clause No Nuclear License,BSD-3-Clause-No-Nuclear-License +BSD 3-Clause No Nuclear License 2014,BSD-3-Clause-No-Nuclear-License-2014 +BSD 3-Clause No Nuclear Warranty,BSD-3-Clause-No-Nuclear-Warranty +BSD 3-Clause Open MPI variant,BSD-3-Clause-Open-MPI +BSD Protection License,BSD-Protection +BSD Source Code Attribution,BSD-Source-Code +BSD Zero Clause License,0BSD +BSD with attribution,BSD-3-Clause-Attribution +BSD(3-clause), BSD-3-Clause +BSD-2-Clause Plus Patent License,BSD-2-Clause-Patent +BSD-4-Clause (University of California-Specific),BSD-4-Clause-UC +BSD_2_clause, BSD-2-Clause +BSD_3_clause, BSD-3-Clause +Bahyph License,Bahyph +Barr License,Barr +Beerware License,Beerware +BitTorrent Open Source License v1.0,BitTorrent-1.0 +Blue Oak Model License 1.0.0,BlueOak-1.0.0 +Boost, BSL-1.0 +Boost Software License 1.0,BSL-1.0 +Borceux license,Borceux +CC0, CC0-1.0 +CERN Open Hardware Licence Version 2 - Permissive,CERN-OHL-P-2.0 +CERN Open Hardware Licence Version 2 - Strongly Reciprocal,CERN-OHL-S-2.0 +CERN Open Hardware Licence Version 2 - Weakly Reciprocal,CERN-OHL-W-2.0 +CERN Open Hardware Licence v1.1,CERN-OHL-1.1 +CERN Open Hardware Licence v1.2,CERN-OHL-1.2 +CMU License,MIT-CMU +CNRI Jython License,CNRI-Jython +CNRI Python License,CNRI-Python +CNRI Python Open Source GPL Compatible License Agreement,CNRI-Python-GPL-Compatible +CPL, CPL-1.0 +CUA Office Public License v1.0,CUA-OPL-1.0 +Caldera License,Caldera +CeCILL Free Software License Agreement v1.0,CECILL-1.0 +CeCILL Free Software License Agreement v1.1,CECILL-1.1 +CeCILL Free Software License Agreement v2.1,CECILL-2.1 +Code Project Open License 1.02,CPOL-1.02 +Common Development and Distribution License 1.0,CDDL-1.0 +Common Development and Distribution License 1.1,CDDL-1.1 +Common Public Attribution License 1.0,CPAL-1.0 +Common Public License 1.0,CPL-1.0 +Commons Clause License v1.0,Commons Clause +Community Data License Agreement Permissive 1.0,CDLA-Permissive-1.0 +Community Data License Agreement Sharing 1.0,CDLA-Sharing-1.0 +Computer Associates Trusted Open Source License 1.1,CATOSL-1.1 +Creative Commons Attribution 1.0 Generic,CC-BY-1.0 +Creative Commons Attribution 2.0 Generic,CC-BY-2.0 +Creative Commons Attribution 2.5 Generic,CC-BY-2.5 +Creative Commons Attribution 3.0 Unported,CC-BY-3.0 +Creative Commons Attribution 4.0 International,CC-BY-4.0 +Creative Commons Attribution No Derivatives 1.0 Generic,CC-BY-ND-1.0 +Creative Commons Attribution No Derivatives 2.0 Generic,CC-BY-ND-2.0 +Creative Commons Attribution No Derivatives 2.5 Generic,CC-BY-ND-2.5 +Creative Commons Attribution No Derivatives 3.0 Unported,CC-BY-ND-3.0 +Creative Commons Attribution No Derivatives 4.0 International,CC-BY-ND-4.0 +Creative Commons Attribution Non Commercial 1.0 Generic,CC-BY-NC-1.0 +Creative Commons Attribution Non Commercial 2.0 Generic,CC-BY-NC-2.0 +Creative Commons Attribution Non Commercial 2.5 Generic,CC-BY-NC-2.5 +Creative Commons Attribution Non Commercial 3.0 Unported,CC-BY-NC-3.0 +Creative Commons Attribution Non Commercial 4.0 International,CC-BY-NC-4.0 +Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic,CC-BY-NC-ND-1.0 +Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic,CC-BY-NC-ND-2.0 +Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic,CC-BY-NC-ND-2.5 +Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported,CC-BY-NC-ND-3.0 +Creative Commons Attribution Non Commercial No Derivatives 4.0 International,CC-BY-NC-ND-4.0 +Creative Commons Attribution Non Commercial Share Alike 1.0 Generic,CC-BY-NC-SA-1.0 +Creative Commons Attribution Non Commercial Share Alike 2.0 Generic,CC-BY-NC-SA-2.0 +Creative Commons Attribution Non Commercial Share Alike 2.5 Generic,CC-BY-NC-SA-2.5 +Creative Commons Attribution Non Commercial Share Alike 3.0 Unported,CC-BY-NC-SA-3.0 +Creative Commons Attribution Non Commercial Share Alike 4.0 International,CC-BY-NC-SA-4.0 +Creative Commons Attribution Share Alike 1.0 Generic,CC-BY-SA-1.0 +Creative Commons Attribution Share Alike 2.0 Generic,CC-BY-SA-2.0 +Creative Commons Attribution Share Alike 2.5 Generic,CC-BY-SA-2.5 +Creative Commons Attribution Share Alike 3.0 Unported,CC-BY-SA-3.0 +Creative Commons Attribution Share Alike 4.0 International,CC-BY-SA-4.0 +Creative Commons Public Domain Dedication and Certification,CC-PDDC +Creative Commons Zero v1.0 Universal,CC0-1.0 +Crossword License,Crossword +Cryptographic Autonomy License 1.0,CAL-1.0 +Cryptographic Autonomy License 1.0 (Combined Work Exception),CAL-1.0-Combined-Work-Exception +CrystalStacker License,CrystalStacker +Cube License,Cube +DMIT License,DMIT +DOC License,DOC +DSDP License,DSDP +Deutsche Freie Software Lizenz,D-FSL-1.0 +Dotseqn License,Dotseqn +EU DataGrid Software License,EUDatagrid +Eclipse Public License, EPL +Eclipse Public License 1.0,EPL-1.0 +Eclipse Public License 2.0,EPL-2.0 +Educational Community License v1.0,ECL-1.0 +Educational Community License v2.0,ECL-2.0 +Eiffel Forum License v1.0,EFL-1.0 +Eiffel Forum License v2.0,EFL-2.0 +Enlightenment License (e16),MIT-advertising +Entessa Public License v1.0,Entessa +Erlang Public License v1.1,ErlPL-1.1 +Etalab Open License 2.0,etalab-2.0 +European Union Public License 1.0,EUPL-1.0 +European Union Public License 1.1,EUPL-1.1 +European Union Public License 1.2,EUPL-1.2 +Eurosym License,Eurosym +Expat, MIT +FSF Unlimited License,FSFUL +FSF Unlimited License (with License Retention),FSFULLR +Fair License,Fair +Frameworx Open License 1.0,Frameworx-1.0 +FreeImage Public License v1.0,FreeImage +GFDL1.1, GFDL-1.1 +GL2PS License,GL2PS +GNU Affero General Public License v3.0 only,AGPL-3.0-only +GNU Affero General Public License v3.0 or later,AGPL-3.0-or-later +GNU GENERAL PUBLIC LICENSE, GPL+ +GNU General Public License v1.0 only,GPL-1.0-only +GNU General Public License v1.0 or later,GPL-1.0-or-later +GNU General Public License v2.0 only,GPL-2.0-only +GNU General Public License v2.0 or later,GPL-2.0-or-later +GNU General Public License v3.0 only,GPL-3.0-only +GNU General Public License v3.0 or later,GPL-3.0-or-later +GNU LESSER GENERAL PUBLIC LICENSE, LGPL-2.1+ +GNU Lesser General Public License v2.1 only,LGPL-2.1-only +GNU Lesser General Public License v2.1 or later,LGPL-2.1-or-later +GNU Lesser General Public License v3.0 only,LGPL-3.0-only +GNU Lesser General Public License v3.0 or later,LGPL-3.0-or-later +GNU Library General Public License v2 only,LGPL-2.0-only +GNU Library General Public License v2 or later,LGPL-2.0-or-later +GPL(==-2), GPL-2.0 +GPL(>=-2), GPL-2.0+ +GPL(>=-2.0), GPL-2.0+ +GPL(>=-2.1), GPL-2.0 +GPL(>=-3), GPL-3.0 +GPL(>=2), GPL-2.0+ +GPL(>=3), GPL-3.0+ +GPL-2, GPL-2.0 +GPL-2+, GPL-2.0+ +GPL-2.0, GPL-2.0 +GPL-2.0+, GPL-2.0+ +GPL-2.0+LGPL-2.1, GPL-2.0 LGPL-2.1 +GPL-2.0-or-later, GPL-2.0+ +GPL-3, GPL-3.0 +GPL-3+, GPL-3.0 +GPL-3.0, GPL-3.0 +GPL-3.0+, GPL-3.0+ +GPL2, GPL-2.0 +GPL3, GPL-3.0 +GPLV2, GPL-2.0 +GPLV3, GPL-3.0 +GPLv2, GPL-2.0 +GPLv2+, GPL-2.0+ +GPLv3, GPL-3.0 +GPLv3+, GPL-3.0+ +Giftware License,Giftware +Glulxe License,Glulxe +Haskell Language Report License,HaskellReport +Hippocratic License 2.1,Hippocratic-2.1 +Historical Permission Notice and Disclaimer,HPND +Historical Permission Notice and Disclaimer - sell variant,HPND-sell-variant +IBM PowerPC Initialization and Boot Software,IBM-pibs +IBM Public License v1.0,IPL-1.0 +ICU License,ICU +IPA Font License,IPA +ISC License,ISC +ISCL , ISC +ImageMagick License,ImageMagick +Independent JPEG Group License,IJG +Info-ZIP License,Info-ZIP +Intel ACPI Software License Agreement,Intel-ACPI +Intel Open Source License,Intel +Interbase Public License v1.0,Interbase-1.0 +JSON License,JSON +Japan Network Information Center License,JPNIC +JasPer License,JasPer-2.0 +LGPL(>=-2), LGPL-2.0+ +LGPL(>=-2.1), LGPL-2.1 +LGPL(>=2), LGPL-2.0+ +LGPL-2, LGPL-2.0 +LGPL-2.0+, LGPL-2.0+ +LGPL-2.1+, LGPL-2.1+ +LGPL-2.1-or-later, LGPL-2.1+ +LGPL-3, LGPL-3.0 +LGPL-3+, LGPL-3.0+ +LGPLv2, LGPL-2.0 +LGPLv2+, LGPL-2.1+ +LGPLv2.1, LGPL-2.1 +LGPLv2.1+, LGPL-2.1+ +LGPLv3, LGPL-3.0 +LGPLv3+, LGPL-3.0+ +LaTeX Project Public License v1.0,LPPL-1.0 +LaTeX Project Public License v1.1,LPPL-1.1 +LaTeX Project Public License v1.3c,LPPL-1.3c +Latex2e License,Latex2e +Lawrence Berkeley National Labs BSD variant license,BSD-3-Clause-LBNL +Leptonica License,Leptonica +Lesser General Public License For Linguistic Resources,LGPLLR +Licence Art Libre 1.2,LAL-1.2 +Licence Art Libre 1.3,LAL-1.3 +Licence Libre du Québec - Permissive version 1.1,LiLiQ-P-1.1 +Licence Libre du Québec - Réciprocité forte version 1.1,LiLiQ-Rplus-1.1 +Licence Libre du Québec - Réciprocité version 1.1,LiLiQ-R-1.1 +Linux Kernel Variant of OpenIB.org license,Linux-OpenIB +Lucent Public License Version 1.0,LPL-1.0 +Lucent Public License v1.02,LPL-1.02 +MIT, MIT +MIT +no-false-attribs license,MITNFA +MIT License, MIT +MIT No Attribution,MIT-0 +MIT/X, MIT +MPL-2, MPL-2.0 +MPL2, MPL-2.0 +MPLv1.1, MPL-1.1 +MPLv2, MPL-2.0 +MPLv2.0, MPL-2.0 +MPLv2.0,, MPL-2.0 +MakeIndex License,MakeIndex +Matrix Template Library License,MTLL +Microsoft Public License,MS-PL +Microsoft Reciprocal License,MS-RL +Motosoto License,Motosoto +Mozilla Public License 1.0,MPL-1.0 +Mozilla Public License 1.1,MPL-1.1 +Mozilla Public License 2.0,MPL-2.0 +Mozilla Public License 2.0 (no copyleft exception),MPL-2.0-no-copyleft-exception +Mulan Permissive Software License v1,MulanPSL-1.0 +Mulan Permissive Software License, Version 1,MulanPSL-1.0 +Mulan Permissive Software License, Version 2,MulanPSL-2.0 +Multics License,Multics +Mup License,Mup +NASA Open Source Agreement 1.3,NASA-1.3 +NRL License,NRL +NTP License,NTP +NTP No Attribution,NTP-0 +Naumen Public License,Naumen +Net Boolean Public License v1,NBPL-1.0 +Net-SNMP License,Net-SNMP +NetCDF license,NetCDF +Nethack General Public License,NGPL +Newsletr License,Newsletr +No Limit Public License,NLPL +Nokia Open Source License,Nokia +Non-Commercial Government Licence,NCGL-UK-2.0 +Non-Profit Open Software License 3.0,NPOSL-3.0 +Norwegian Licence for Open Government Data,NLOD-1.0 +Noweb License,Noweb +OCLC Research Public License 2.0,OCLC-2.0 +ODC Public Domain Dedication & License 1.0,PDDL-1.0 +OFL, OFL +OGC Software License, Version 1.0,OGC-1.0 +OSET Public License version 2.1,OSET-PL-2.1 +Open CASCADE Technology Public License,OCCT-PL +Open Data Commons Attribution License v1.0,ODC-By-1.0 +Open Government Licence - Canada,OGL-Canada-2.0 +Open Government Licence v1.0,OGL-UK-1.0 +Open Government Licence v2.0,OGL-UK-2.0 +Open Government Licence v3.0,OGL-UK-3.0 +Open Group Test Suite License,OGTSL +Open LDAP Public License 2.2.2,OLDAP-2.2.2 +Open LDAP Public License v1.1,OLDAP-1.1 +Open LDAP Public License v1.2,OLDAP-1.2 +Open LDAP Public License v1.3,OLDAP-1.3 +Open LDAP Public License v1.4,OLDAP-1.4 +Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B),OLDAP-2.0 +Open LDAP Public License v2.0.1,OLDAP-2.0.1 +Open LDAP Public License v2.1,OLDAP-2.1 +Open LDAP Public License v2.2,OLDAP-2.2 +Open LDAP Public License v2.2.1,OLDAP-2.2.1 +Open LDAP Public License v2.4,OLDAP-2.4 +Open LDAP Public License v2.5,OLDAP-2.5 +Open LDAP Public License v2.6,OLDAP-2.6 +Open LDAP Public License v2.8,OLDAP-2.8 +Open Market License,OML +Open Public License v1.0,OPL-1.0 +Open Software License 1.0,OSL-1.0 +Open Software License 2.0,OSL-2.0 +Open Software License 2.1,OSL-2.1 +Open Software License 3.0,OSL-3.0 +Open Use of Data Agreement v1.0,O-UDA-1.0 +OpenSSL License,OpenSSL +PHP License v3.0,PHP-3.0 +PNG Reference Library version 2,libpng-2.0 +PSF, Python-2.0 +Perl, Artistic-1.0-Perl +Plexus Classworlds License,Plexus +PolyForm Noncommercial License 1.0.0,PolyForm-Noncommercial-1.0.0 +PolyForm Small Business License 1.0.0,PolyForm-Small-Business-1.0.0 +PostgreSQL License,PostgreSQL +Python, Python-2.0 +Python License 2.0,Python-2.0 +Python Software Foundation License 2.0,PSF-2.0 +Q Public License 1.0,QPL-1.0 +Qhull License,Qhull +RSA Message-Digest License,RSA-MD +Rdisc License,Rdisc +RealNetworks Public Source License v1.0,RPSL-1.0 +Reciprocal Public License 1.1,RPL-1.1 +Reciprocal Public License 1.5,RPL-1.5 +Red Hat eCos Public License v1.1,RHeCos-1.1 +Ricoh Source Code Public License,RSCPL +Ruby License,Ruby +SCEA Shared Source License,SCEA +SGI Free Software License B v1.0,SGI-B-1.0 +SGI Free Software License B v1.1,SGI-B-1.1 +SIL Open Font License 1.0 with Reserved Font Name,OFL-1.0-RFN +SIL Open Font License 1.0 with no Reserved Font Name,OFL-1.0-no-RFN +SIL Open Font License 1.1,OFL-1.1 +SIL Open Font License 1.1 with Reserved Font Name,OFL-1.1-RFN +SIL Open Font License 1.1 with no Reserved Font Name,OFL-1.1-no-RFN +SNIA Public License 1.1,SNIA +SQLite Blessing,blessing +SSH OpenSSH license,SSH-OpenSSH +SSH short notice,SSH-short +Sax Public Domain Notice,SAX-PD +Saxpath License,Saxpath +Scheme Widget Library (SWL) Software License Agreement,SWL +Secure Messaging Protocol Public License,SMPPL +Sendmail License,Sendmail +Sendmail License 8.23,Sendmail-8.23 +Server Side Public License, v 1,SSPL-1.0 +Simple Public License 2.0,SimPL-2.0 +Sleepycat License,Sleepycat +Solderpad Hardware License v0.5,SHL-0.5 +Solderpad Hardware License, Version 0.51,SHL-0.51 +Spencer License 86,Spencer-86 +Spencer License 94,Spencer-94 +Spencer License 99,Spencer-99 +SugarCRM Public License v1.1.3,SugarCRM-1.1.3 +Sun Industry Standards Source License v1.1,SISSL +Sun Industry Standards Source License v1.2,SISSL-1.2 +Sun Public License v1.0,SPL-1.0 +Sybase Open Watcom Public License 1.0,Watcom-1.0 +TAPR Open Hardware License v1.0,TAPR-OHL-1.0 +TCL/TK License,TCL +TCP Wrappers License,TCP-wrappers +TMate Open Source License,TMate +TORQUE v2.5+ Software License v1.1,TORQUE-1.1 +Technische Universitaet Berlin License 1.0,TU-Berlin-1.0 +Technische Universitaet Berlin License 2.0,TU-Berlin-2.0 +The MirOS Licence,MirOS +The Parity Public License 6.0.0,Parity-6.0.0 +The Parity Public License 7.0.0,Parity-7.0.0 +The Unicode Character Database,UCD +Trusster Open Source License,TOSL +Unicode License Agreement - Data Files and Software (2015),Unicode-DFS-2015 +Unicode License Agreement - Data Files and Software (2016),Unicode-DFS-2016 +Unicode Terms of Use,Unicode-TOU +Universal Permissive License v1.0,UPL-1.0 +University of Illinois/NCSA Open Source License,NCSA +Upstream Compatibility License v1.0,UCL-1.0 +VIM, Vim +VOSTROM Public License for Open Source,VOSTROM +Vovida Software License v1.0,VSL-1.0 +W3C Software Notice and Document License (2015-05-13),W3C-20150513 +W3C Software Notice and License (1998-07-20),W3C-19980720 +W3C Software Notice and License (2002-12-31),W3C +Wsuipa License,Wsuipa +X.Net License,Xnet +XPP License,xpp +XSkat License,XSkat +Xerox License,Xerox +Yahoo! Public License v1.0,YPL-1.0 +ZLIB, Zlib +ZPL, ZPL-2.0 +ZPL 2.1, ZPL-2.1 +Zed License,Zed +Zimbra Public License v1.4,Zimbra-1.4 +Zope Public License 1.1,ZPL-1.1 +Zope Public License 2.0,ZPL-2.0 +Zope Public License 2.1,ZPL-2.1 +apache, Apache-2.0 +artistic2, Artistic-2.0 +artistic_2, Artistic-2.0 +bzip2 and libbzip2 License v1.0.5,bzip2-1.0.5 +bzip2 and libbzip2 License v1.0.6,bzip2-1.0.6 +copyleft-next 0.3.0,copyleft-next-0.3.0 +copyleft-next 0.3.1,copyleft-next-0.3.1 +curl License,curl +diffmark license,diffmark +dvipdfm License,dvipdfm +eGenix.com Public License 1.1.0,eGenix +enna License,MIT-enna +feh License,MIT-feh +gSOAP Public License v1.3b,gSOAP-1.3b +gplv3, GPL-3.0 +http://creativecommons.org/licenses/BSD/, BSD-2-Clause +http://opensource.org/licenses/MIT, MIT +http://www.apache.org/licenses/LICENSE-2.0, Apache-2.0 +lgpl, LGPL-2.1 +libpng License,Libpng +libselinux public domain notice,libselinux-1.0 +libtiff License,libtiff +mpich2 License,mpich2 +perl, Artistic-1.0-Perl +psfrag License,psfrag +psutils License,psutils +version 3 of the GNU General Public License, GPL-3.0+ +w3c, W3C +zlib, Zlib +zlib License,Zlib +zlib/libpng, zlib-acknowledgement +zlib/libpng License with Acknowledgement,zlib-acknowledgement diff --git a/src/ac/acl/package_license/package_license.py b/src/ac/acl/package_license/package_license.py new file mode 100644 index 0000000000000000000000000000000000000000..cfac975f9b6fdacab8cf0c8170401d9c699aaf7b --- /dev/null +++ b/src/ac/acl/package_license/package_license.py @@ -0,0 +1,193 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-10-16 +# Description: check spec file +# ********************************************************************************** +""" + +import logging +import os +import re +import chardet + +from src.ac.common.pyrpm import Spec, replace_macros +from src.ac.common.rpm_spec_adapter import RPMSpecAdapter + +logger = logging.getLogger("ac") + +class PkgLicense(object): + """ + 解析获取软件包中源码、spec中的license + 进行白名单校验、一致性检查 + """ + + LICENSE_FILE_TARGET = ["apache-2.0", + "artistic", + "artistic.txt", + "libcurllicense", + "gpl.txt", + "gpl2.txt", + "gplv2.txt", + "notice", + "about_bsd.txt", + "mit", + "pom.xml", + "meta.yml"] + + LICENSE_TARGET_PAT = re.compile(r"^(copying)|(copyright)|(copyrights)|(licenses)|(licen[cs]e)(\.(txt|xml))?$") + + WHITE_LIST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "config", + "license_list") + LICENSE_TRANS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "config", + "license_translations") + + def __init__(self): + self._white_black_list = {} + self._license_translation = {} + + def _load_license_dict(self, filename): + """ + read the dict from license file + """ + result = {} + if not os.path.isfile(filename): + logger.warning("not found the config file: %s", os.path.basename(filename)) + return result + with open(filename, "r") as f: + for line in f: + if line.startswith("#"): + continue + k, v = line.rsplit(",", 1) + k = PkgLicense._auto_decode_str(k.strip()) + v = PkgLicense._auto_decode_str(v.strip()) + result[k] = v + return result + + def load_config(self): + """ + Load the license white list and translation list into dict + """ + self._white_black_list = self._load_license_dict(self.WHITE_LIST_PATH) + self._license_translation = self._load_license_dict(self.LICENSE_TRANS_PATH) + + def check_license_safe(self, licenses): + """ + Check if the license is in the blacklist + """ + result = True + for lic in licenses: + res = self._white_black_list.get(lic, "Need review") + if res == "white": + logger.info("This license: %s is safe", lic) + elif res == "black": + logger.error("This license: %s is not safe", lic) + result = False + else: + logger.warning("This license: %s need to be review", lic) + result = False + return result + + def translate_license(self, licenses): + """ + Convert license to uniform format + """ + result = set() + for lic in licenses: + real_license = self._license_translation.get(lic, lic) + result.add(real_license) + return result + + @staticmethod + def split_license(licenses): + """ + 分割spec license字段的license + """ + license_set = re.split(r'/\s?|\(|\)|\,|[Aa][Nn][Dd]|or|OR|\s?/g', licenses) + for index in range(len(license_set)): # 去除字符串首尾空格 + license_set[index] = license_set[index].strip() + return set(filter(None, license_set)) # 去除list中空字符串 + + # 以下为从license文件中获取license + def scan_licenses_in_license(self, srcdir): + """ + Find LICENSE files and scan. + """ + licenses_in_file = set() + if not os.path.exists(srcdir): + logger.error("%s not exist.", srcdir) + return licenses_in_file + + for root, dirnames, filenames in os.walk(srcdir): + for filename in filenames: + if (filename.lower() in self.LICENSE_FILE_TARGET + or self.LICENSE_TARGET_PAT.search(filename.lower())): + logger.info("scan the license target file: %s", filename) + licenses_in_file.update( + self.scan_licenses( + os.path.join(root, filename))) + logger.info("all licenses from src: %s", ", ".join([data.encode("utf-8") for data in licenses_in_file])) + return licenses_in_file + + def scan_licenses(self, copying): + """ + Scan licenses from copying file and add to licenses_for_source_files. + if get contents failed or decode data failed, return nothing. + """ + licenses_in_file = set() + + if not os.path.exists(copying): + logger.warning("file: %s not exist", copying) + return licenses_in_file + + for word in self._license_translation: + if word in copying: + licenses_in_file.add(word) + + with open(copying, "rb") as f: + data = f.read() + data = PkgLicense._auto_decode_str(data) + if not data: + return licenses_in_file + for word in self._license_translation: + try: + if word in data: + licenses_in_file.add(word) + except UnicodeDecodeError as e: + logger.exception("decode error: %s", str(e)) + return licenses_in_file + + @staticmethod + def _decode_str(data, charset): + """ + Decode the license string. return the license string or nothing. + """ + if not charset: + return "" + return data.decode(charset) + + @staticmethod + def _auto_decode_str(data): + return PkgLicense._decode_str(data, chardet.detect(data)["encoding"]) + + @staticmethod + def check_licenses_is_same(licenses_for_spec, licenses_for_source_files): + """ + Check if the licenses from SPEC is the same as the licenses from LICENSE file. + if same, return True. if not same return False. + """ + if not licenses_for_source_files: + return False + return licenses_for_spec.issuperset(licenses_for_source_files) \ No newline at end of file diff --git a/src/ac/acl/package_yaml/__init__.py b/src/ac/acl/package_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..054319cd51014650ac251515823879e185c4d0de --- /dev/null +++ b/src/ac/acl/package_yaml/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-17 +# Description: +# ***********************************************************************************/ +""" \ No newline at end of file diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..9541ea3923d2086cb0aa8bf868545481a2ac9c5a --- /dev/null +++ b/src/ac/acl/package_yaml/check_repo.py @@ -0,0 +1,540 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-22 +# Description: check yaml file in software package +# ***********************************************************************************/ +""" +# \ / /| /| /| / +# \ / /_| / | / | / +# / / | / | / | / +# / / | / |/ | /_____ + +import logging +import re +import urlparse +import requests +import json +import subprocess +import tldextract +import abc + +logging.getLogger("ac") + + +class AbsReleaseTags(object): + """ + 获取release tags的抽象类 + """ + + __metaclass__ = abc.ABCMeta + def __init__(self, version_control): + self.version_control = version_control + + @abc.abstractmethod + def url(self, repo): + """ + 抽象方法 + """ + pass + + @abc.abstractmethod + def get_tags(self, repo): + """ + 抽象方法 + """ + pass + + +class DefaultReleaseTags(AbsReleaseTags): + """ + 获取release tags的基类 + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + logging.info("unsupported version control: {}".format(self.version_control)) + return [] + + +class HttpReleaseTagsMixin(object): + """ + 通过web请求形式获取release tags + """ + DEFAULT_REQ_HEADER = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)' + } + + def get_redirect_resp(self, url, response): + """ + 获取重定向的url和cookie + return: bool, str, list + """ + cookie = set() + href = "" + need_redirect = False + for line in response.text.splitlines(): + line = line.strip() + if line.startswith("Redirecting"): + logging.debug("Redirecting with document.cookie") + need_redirect = True + search_result = re.search(r"document\.cookie=\"(.*)\";", line) + if search_result: + cookie = cookie | set(search_result.group(1).split(';')) + search_result = re.search(r"document\.location\.href=\"(.*)\";", line) + if search_result: + href = search_result.group(1) + new_url = urlparse.urljoin(url, href) + if "" in cookie: + cookie.remove("") + return need_redirect, new_url, list(cookie) + + def get_request_response(self, url, timeout=30, headers=None): + """ + 获取url请求获取response + return: reponse + """ + headers = self.DEFAULT_REQ_HEADER if headers is None else headers + try: + response = requests.get(url, headers=headers, timeout=timeout) + need_redirect, new_url, cookies = self.get_redirect_resp(url, response) + if tldextract.extract(url).domain != tldextract.extract(new_url).domain: # 判断域名是否一致 预防csrf攻击 + logging.warning("domain of redirection link is different: {}".format(new_url)) + return "" + if need_redirect: + cookie_dict = {} + for cookie in cookies: + key, val = cookie.split("=") + cookie_dict[key] = val + url = new_url + response = requests.get(url, headers=headers, cookies=cookie_dict, timeout=timeout) + except requests.exceptions.SSLError as e: + logging.warning("requests {} ssl exception, {}".format(url, e)) + return "" + except requests.exceptions.Timeout as e: + logging.warning("requests timeout") + return "" + except requests.exceptions.RequestException as e: + logging.warning("requests exception, {}".format(e)) + return "" + return response + + +class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取hg上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin(repo + "/", "json-tags") if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + if not response: + logging.warning("unable to get response:") + return [] + try: + tags_json = json.loads(response.text) + temp_tags = tags_json.get("tags") + temp_tags.sort(reverse=True, key=lambda x: x["date"][0]) + release_tags = [tag["tag"] for tag in temp_tags] + except Exception as e: + logging.error("exception, {}".format(e)) + return [] + return release_tags + + +class HgRawReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取hg raw上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin(repo + "/", "raw-tags") if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + release_tags = [] + for line in response.text.splitlines(): + release_tags.append(line.split()[0]) + return release_tags + + +class MetacpanReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取metacpan上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://metacpan.org/release/", repo) if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + resp_lines = response.text.splitlines() + release_tags = [] + tag_condition = "value=\"/release" + for index in range(len(resp_lines) - 1): + if tag_condition in resp_lines[index]: + tag = resp_lines[index + 1] + index += 1 + if "DEV" in tag: + continue + tag = tag.strip() + release_tags.append(tag) + return release_tags + + +class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取pypi上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://pypi.org/pypi/", repo + "/json") if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + try: + tags_json = response.json() + release_tags = [tag for tag in tags_json.get("releases")] + except Exception as e: + logging.error("exception, {}".format(e)) + return [] + return release_tags + + +class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取rubygem上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://rubygems.org/api/v1/versions/", repo + ".json") if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + try: + tags_json = response.json() + release_tags = [] + for element in tags_json: + if element.get("number"): + release_tags.append(element.get("number")) + except Exception as e: + logging.error("exception, {}".format(e)) + return [] + return release_tags + + +class GnuftpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取gnu-ftp上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://ftp.gnu.org/gnu/", repo) if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + pattern = re.compile("href=\"(.*)\">(.*)") + release_tags = [] + for line in response.text.splitlines(): + search_result = pattern.search(line) + if search_result: + release_tags.append(search_result.group(1)) # python2用法 python3不同 + return release_tags + + +class FtpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): + """ + 获取ftp上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin('ftp', repo + "/") if repo else "" + + def get_tags(self, repo): + """ + 通过url获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_request_response(url) + pattern = re.compile("href=\"(.*)\">(.*)") + release_tags = [] + for line in response.text.splitlines(): + search_result = pattern.search(line) + if search_result: + release_tags.append(search_result.group(1)) # python2用法 python3不同 + return release_tags + + +class CmdReleaseTagsMixin(object): + """ + 通过shell命令获取上游社区的release tags + """ + def get_cmd_response(self, cmd_list): + """ + 获取shell命令的reponse + return: reponse + """ + sub_proc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) + response = sub_proc.stdout.read().decode("utf-8") + if sub_proc.wait(): + logging.warning("{cmd} > encount errors".format(cmd=" ".join(cmd_list))) + return response + + +class SvnReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): + """ + 通过shell svn命令获取上游社区的release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin(repo + "/", "tags") if repo else "" + + def get_response(self, url): + """ + 生成svn命令并获取reponse + return: response + """ + cmd_list = ["/usr/bin/svn", "ls", "-v", url] + return self.get_cmd_response(cmd_list) + + def get_tags(self, repo): + """ + 通过shell cmd访问远端获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get svn tags".format(repo=url)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_response(url) + release_tags = [] + for line in response.splitlines(): + for item in line.split(): + if item and item[-1] == "/": + release_tags.append(item[:-1]) + break + return release_tags + + +class GitReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): + """ + 通过shell git命令获取上游社区的release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return repo + + def get_response(self, url): + """ + 生成git命令并获取reponse + return: response + """ + cmd_list = ["git", "ls-remote", "--tags", url] + return self.get_cmd_response(cmd_list) + + def trans_reponse_tags(self, reponse): + """ + 解析git命令返回值为纯数字形式的tag + return: list + """ + release_tags = [] + pattern = re.compile(r"^([^ \t]*)[ \t]*refs\/tags\/([^ \t]*)") + for line in reponse.splitlines(): + match_result = pattern.match(line) + if match_result: + tag = match_result.group(2) + if not tag.endswith("^{}"): + release_tags.append(tag) + return release_tags + + def get_tags(self, repo): + """ + 通过shell cmd访问远端获取上游社区的release tags + return: list + """ + url = self.url(repo) + logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + if not url: + logging.warning("illegal url: \"\"") + return [] + response = self.get_response(url) + return self.trans_reponse_tags(response) + + +class GithubReleaseTags(GitReleaseTags): + """ + 获取github上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://github.com/", repo + ".git") if repo else "" + + +class GiteeReleaseTags(GitReleaseTags): + """ + 获取gitee上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + return urlparse.urljoin("https://gitee.com/", repo) if repo else "" + + +class GitlabReleaseTags(GitReleaseTags): + """ + 获取gitlab.gnome上游社区release tags + """ + def url(self, repo): + """ + 通过src_repo生成url + return: str + """ + if not repo: + return "" + src_repos = repo.split("/") + if len(src_repos) == 1: + return urlparse.urljoin("https://gitlab.gnome.org/GNOME/", repo + ".git") + else: + return urlparse.urljoin("https://gitlab.gnome.org/", repo + ".git") + + +class ReleaseTagsFactory(object): + """ + ReleaseTags及其子类的工厂类 + """ + VERSION_CTRL_GETTER_MAPPING = { + "hg": HgReleaseTags, + "hg-raw": HgRawReleaseTags, + "github": GithubReleaseTags, + "git": GitReleaseTags, + "gitlab.gnome": GitlabReleaseTags, + "svn": SvnReleaseTags, + "metacpan": MetacpanReleaseTags, + "pypi": PypiReleaseTags, + "rubygem": RubygemReleaseTags, + "gitee": GiteeReleaseTags, + "gnu-ftp": GnuftpReleaseTags, + "ftp": FtpReleaseTags + } + + @staticmethod + def get_release_tags(version_control): + """ + 通过version control返回对应的ReleaseTags的子类 + return: class + """ + release_tags = ReleaseTagsFactory.VERSION_CTRL_GETTER_MAPPING.get(version_control, DefaultReleaseTags) + return release_tags(version_control) \ No newline at end of file diff --git a/src/ac/acl/package_yaml/check_yaml.py b/src/ac/acl/package_yaml/check_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..62f5d0b01d244514b43d9090ec9ad93d40c4a3d9 --- /dev/null +++ b/src/ac/acl/package_yaml/check_yaml.py @@ -0,0 +1,209 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-17 +# Description: check yaml file in software package +# ***********************************************************************************/ +""" + +import logging +import os +import yaml + +from src.proxy.git_proxy import GitProxy +from src.proxy.requests_proxy import do_requests +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.acl.package_yaml.check_repo import ReleaseTagsFactory +from src.ac.common.rpm_spec_adapter import RPMSpecAdapter + +logger = logging.getLogger("ac") + +class CheckPackageYaml(BaseCheck): + """ + check yaml file + """ + + NOT_FOUND = "NA" + PACKAGE_YAML_NEEDED_KEY = [ + "version_control", + "src_repo", + "tag_prefix", + "separator"] + + VERSION_CTRL_TRANS = { + "gitlab.gnome": "gnome", + "pypi": "pythonhosted" + } + + def __init__(self, workspace, repo, conf): + super(CheckPackageYaml, self).__init__(workspace, repo, conf) + + self._gp = GitProxy(self._work_dir) + self._gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress + if self._gr.spec_file: + self._spec = RPMSpecAdapter(os.path.join(self._work_dir, self._gr.spec_file)) + else: + self._spec = None + self._yaml_content = None + self._yaml_changed = True + self._is_standard = True + + def is_change_package_yaml(self, base="HEAD~1", head="HEAD~0"): + """ + 如果本次提交变更了yaml，则对yaml进行检查 + :param: + base:作为对比的提交点 + head:此次提交点 + :return: boolean + """ + diff_files = self._gp.diff_files_between_commits(base, head) + package_yaml = "{}.yaml".format(self._repo) # package yaml file name + + for change_file in diff_files: + if change_file == package_yaml: + logger.debug("diff files: {}".format(diff_files)) + return True + return False + + def check_fields(self): + """ + 检查yaml规定字段的完整性 + :return: + """ + if not self._yaml_changed: + return SUCCESS + yaml_path = self._gr.yaml_file + if yaml_path is None: + self._is_standard = False + logger.warning("yaml file missing") + return WARNING + try: + with open(os.path.join(self._work_dir, yaml_path), 'r') as yaml_data: # load yaml data + self._yaml_content = yaml.safe_load(yaml_data) + except IOError as e: + logging.warning("package yaml not exist. {}".format(str(e))) + return WARNING + except yaml.YAMLError as exc: + logging.warning("Error parsering YAML: {}".format(str(exc))) + return WARNING + + result = SUCCESS + for keyword in self.PACKAGE_YAML_NEEDED_KEY: + if keyword not in self._yaml_content: + logger.error("yaml field {} missing".format(keyword)) + self._is_standard = True + result = WARNING + return result + + def check_repo(self): + """ + 检查yaml的有效性,能否从上游社区获取版本信息 + :return: + """ + if not self._yaml_changed: + return SUCCESS + if not self._is_standard: + logger.warning("yaml does not comply with the rule") + return SUCCESS + # get value by key from yaml data + vc = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[0]] # value of version_control + sr = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[1]] # value of src_repo + + if vc == self.NOT_FOUND or sr == self.NOT_FOUND: + logger.warning("no info for upsteam") + return WARNING + + release_tags = ReleaseTagsFactory.get_release_tags(vc) + tags = release_tags.get_tags(sr) + + if not tags: + logger.warning("failed to get version by yaml, version_control: {t1}, src_repo: {t2}".format(t1=vc, t2=sr)) + return WARNING + return SUCCESS + + def check_repo_domain(self): + """ + 检查spec中source0域名是否包含yaml的version_control,仅做日志告警只返回SUCCESS(autoconf为特例) + :return: + """ + if not self._yaml_changed: + return SUCCESS + if not self._is_standard: + logger.warning("yaml does not comply with the rule") + return SUCCESS + if not self._spec: + logger.warning("spec does not exist") + return SUCCESS + + vc = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[0]] + if vc == self.NOT_FOUND: + return SUCCESS + src_url = self._spec.get_source("Source0") + if not src_url: + src_url = self._spec.get_source("Source") + vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换 + logger.debug("version control: {vctrl} source url: {url}".format(vctrl=vc, url=src_url)) + if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性 + logger.warning("{vc} is not in url: {url}".format(vc=vc, url=src_url)) + return WARNING + return SUCCESS + + def check_repo_name(self): + """ + 检查spec中是否包含yaml中src_repo字段的软件名,仅做日志告警只返回SUCCESS + :return: + """ + if not self._yaml_changed: + return SUCCESS + if not self._is_standard: + logger.warning("yaml does not comply with the rule") + return SUCCESS + if not self._spec: + logger.warning("spec does not exist") + return SUCCESS + + sr = self._yaml_content[self.PACKAGE_YAML_NEEDED_KEY[1]] + if sr == self.NOT_FOUND: + return SUCCESS + + software_name_list = list(filter(None, sr.split("/"))) + + def guess_real_pkgname(name_list): + """ + 解析yaml中src_repo字段对应的软件包名 + :return: + """ + pkgname = name_list[-1] + if len(name_list) > 1 and name_list[-1] == "svn": + pkgname = name_list[-2] + if pkgname.endswith(".git"): + pkgname = os.path.splitext(pkgname)[0] + return pkgname + + software_name = guess_real_pkgname(software_name_list) + src_url = self._spec.get_source("Source0") + if not src_url: + src_url = self._spec.get_source("Source") + logger.debug("software name: {name} source url: {url}".format(name=software_name, url=src_url)) + if software_name not in src_url: + logger.warning("{name} is not in source0: {url}".format(name=software_name, url=src_url)) + return WARNING + return SUCCESS + + def __call__(self, *args, **kwargs): + logger.info("check {} yaml ...".format(self._repo)) + self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 + return self.start_check_with_order("fields", "repo_domain", "repo_name", "repo") + diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 468b616eb8a30da5be37207de1985279974421ff..6502e91481a280afcc6158b4cad149a9b4d9d786 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -79,7 +79,7 @@ class CheckSpec(BaseCheck): logger.debug("only change package yaml") return SUCCESS - self._gp.checkout_to_commit("HEAD~1") + self._gp.checkout_to_commit_force("HEAD~1") try: gr = GiteeRepo(self._repo, self._work_dir, None) # don't care about decompress fp = self._gp.get_content_of_file_with_commit(gr.spec_file) @@ -88,7 +88,7 @@ class CheckSpec(BaseCheck): return SUCCESS spec_o = RPMSpecAdapter(fp) finally: - self._gp.checkout_to_commit(self._latest_commit) # recover whatever + self._gp.checkout_to_commit_force(self._latest_commit) # recover whatever self._ex_pkgship(spec_o) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 696316caa5f447ef9b3f8b7c81ac142e4821f541..3432d4f579c83e9305b5f8eead33b17c835d510e 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -1,5 +1,6 @@ # -*- encoding=utf-8 -*- -# ********************************************************************************** +""" +# *********************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # [openeuler-jenkins] is licensed under the Mulan PSL v1. # You can use this software according to the terms and conditions of the Mulan PSL v1. @@ -12,13 +13,15 @@ # Author: # Create: 2020-09-23 # Description: Gitee api proxy -# ********************************************************************************** +# ***********************************************************************************/ +""" import os import logging from src.proxy.git_proxy import GitProxy from src.utils.shell_cmd import shell_cmd_live +from src.ac.acl.package_license.package_license import PkgLicense logger = logging.getLogger("ac") @@ -36,6 +39,7 @@ class GiteeRepo(object): self._compress_files = [] self.spec_file = None + self.yaml_file = None self.patch_dir_mapping = {} self.find_file_path() @@ -57,6 +61,9 @@ class GiteeRepo(object): elif self.is_spec_file(filename): logger.debug("find spec file: {}".format(rel_file_path)) spec_files.append(filename) + elif self.is_package_yaml_file(filename): + logger.debug("find yaml file: {}".format(rel_file_path)) + self.yaml_file = rel_file_path def guess_real_spec_file(): """ @@ -162,40 +169,105 @@ class GiteeRepo(object): return 0 if all(rs) else (1 if any(rs) else -1) + def scan_license_in_spec(self, spec): + """ + Find spec file and scan. If no spec file or open file failed, the program will exit with an error. + """ + if not spec: + return set() + licenses = spec.license + licenses_in_spec = PkgLicense.split_license(licenses) + logger.info("all licenses from SPEC: %s", ", ".join(list(licenses_in_spec))) + return licenses_in_spec + @staticmethod def is_py_file(filename): + """ + 功能描述：判断文件是否是python文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".py",)) @staticmethod def is_go_file(filename): + """ + 功能描述：判断文件名是否是go文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".go",)) @staticmethod def is_c_cplusplus_file(filename): + """ + 功能描述：判断文件名是否是c++文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".c", ".cpp", ".cc", ".cxx", ".c++", ".h", ".hpp", "hxx")) @staticmethod def is_code_file(filename): + """ + 功能描述：判断文件名是否是源码文件 + 参数：文件名 + 返回值：bool + """ return GiteeRepo.is_py_file(filename) \ or GiteeRepo.is_go_file(filename) \ or GiteeRepo.is_c_cplusplus_file(filename) @staticmethod def is_patch_file(filename): + """ + 功能描述：判断文件名是否是补丁文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".patch", ".diff")) @staticmethod def is_compress_file(filename): + """ + 功能描述：判断文件名是否是压缩文件 + 参数：文件名 + 返回值：bool + """ return GiteeRepo._is_compress_tar_file(filename) or GiteeRepo._is_compress_zip_file(filename) @staticmethod def _is_compress_zip_file(filename): + """ + 功能描述：判断文件名是否是zip压缩文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".zip",)) @staticmethod def _is_compress_tar_file(filename): + """ + 功能描述：判断文件名是否是tar压缩文件 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".tar.gz", ".tar.bz", ".tar.bz2", ".tar.xz", "tgz")) @staticmethod def is_spec_file(filename): + """ + 功能描述：判断文件名是否以.spec结尾 + 参数：文件名 + 返回值：bool + """ return filename.endswith((".spec",)) + + @staticmethod + def is_package_yaml_file(filename): + """ + 功能描述：判断文件名是否以.yaml结尾 + 参数：文件名 + 返回值：bool + """ + return filename.endswith((".yaml",)) diff --git a/src/ac/common/rpm_spec_adapter.py b/src/ac/common/rpm_spec_adapter.py index 65db3aa241d64c8289463bc475a249fbe94e92f1..023f52f46c96346d8393fc994f3266383d7e63db 100644 --- a/src/ac/common/rpm_spec_adapter.py +++ b/src/ac/common/rpm_spec_adapter.py @@ -45,9 +45,24 @@ class RPMSpecAdapter(object): value = getattr(self._adapter, item) if isinstance(value, list): return [replace_macros(item, self._adapter) for item in value] - return replace_macros(value, self._adapter) if value else "" + def get_source(self, key): + """ + get source url from spec.source_dict by key + :return: + """ + src_url = self._adapter.sources_dict.get(key, "") + return replace_macros(src_url, self._adapter) if src_url else "" + + def get_patch(self, key): + """ + get source url from spec.source_dict by key + :return: + """ + patch = self._adapter.patches_dict.get(key, "") + return replace_macros(patch, self._adapter) if patch else "" + def include_x86_arch(self): """ check include x86-64 diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index f2febf6cf2ff499af96e27bbf91be9e2921c32b9..a65efc8e6c959ac406a36e1c40e262aa3e52b609 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -22,6 +22,7 @@ import logging import json import argparse import importlib +import datetime from yaml.error import YAMLError @@ -30,18 +31,24 @@ class AC(object): """ ac entrypoint """ - def __init__(self, conf): + def __init__(self, conf, community="src-openeuler"): + """ + + :param conf: 配置文件路径 + :param community: src-openeuler or openeuler + :return: + """ self._ac_check_elements = {} # 门禁项 self._ac_check_result = [] # 门禁结果结果 acl_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "../acl")) self._acl_package = "src.ac.acl" # take attention about import module self.load_check_elements_from_acl_directory(acl_path) - self.load_check_elements_from_conf(conf) + self.load_check_elements_from_conf(conf, community) logger.debug("check list: {}".format(self._ac_check_elements)) - def check_all(self, workspace, repo, **kwargs): + def check_all(self, workspace, repo, dataset, **kwargs): """ 门禁检查 :param workspace: @@ -95,7 +102,9 @@ class AC(object): if not hint.startswith("check_"): hint = "check_{}".format(hint) self._ac_check_result.append({"name": hint, "result": result.val}) + dataset.set_attr("access_control.build.acl.{}".format(element), result.hint) + dataset.set_attr("access_control.build.content", self._ac_check_result) logger.debug("ac result: {}".format(self._ac_check_result)) def load_check_elements_from_acl_directory(self, acl_dir): @@ -107,15 +116,16 @@ class AC(object): if os.path.isdir(os.path.join(acl_dir, filename)): self._ac_check_elements[filename] = {} # don't worry, using default when checking - def load_check_elements_from_conf(self, conf_file): + def load_check_elements_from_conf(self, conf_file, community): """ 加载门禁项目，只支持yaml格式 :param conf_file: 配置文件路径 + :param community: src-openeuler or openeuler :return: """ try: with open(conf_file, "r") as f: - elements = yaml.safe_load(f) + content = yaml.safe_load(f) except IOError: logger.exception("ac conf file {} not exist".format(conf_file)) return @@ -123,6 +133,8 @@ class AC(object): logger.exception("illegal conf file format") return + elements = content.get(community, {}) + logger.debug("community \"{}\" conf: {}".format(community, elements)) for name in elements: if name in self._ac_check_elements: if elements[name].get("exclude"): @@ -142,16 +154,34 @@ class AC(object): f.write("ACL={}".format(json.dumps(self._ac_check_result))) -if "__main__" == __name__: +def init_args(): + """ + init args + :return: + """ args = argparse.ArgumentParser() + args.add_argument("-c", type=str, dest="community", default="src-openeuler", help="src-openeuler or openeuler") args.add_argument("-w", type=str, dest="workspace", help="workspace where to find source") args.add_argument("-r", type=str, dest="repo", help="repo name") args.add_argument("-b", type=str, dest="tbranch", help="branch merge to") - args.add_argument("-n", type=str, dest="owner", default="src-openeuler", help="gitee owner") args.add_argument("-o", type=str, dest="output", help="output file to save result") args.add_argument("-p", type=str, dest="pr", help="pull request number") args.add_argument("-t", type=str, dest="token", help="gitee api token") - args = args.parse_args() + args.add_argument("-a", type=str, dest="account", help="gitee account") + + # dataset + args.add_argument("-m", type=str, dest="comment", help="trigger comment") + args.add_argument("-i", type=str, dest="comment_id", help="trigger comment id") + args.add_argument("-e", type=str, dest="committer", help="committer") + args.add_argument("-x", type=str, dest="pr_ctime", help="pr create time") + args.add_argument("-z", type=str, dest="trigger_time", help="job trigger time") + args.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") + + return args.parse_args() + + +if "__main__" == __name__: + args = init_args() # init logging _ = not os.path.exists("log") and os.mkdir("log") @@ -163,11 +193,59 @@ if "__main__" == __name__: # notify gitee from src.proxy.gitee_proxy import GiteeProxy - gp = GiteeProxy(args.owner, args.repo, args.token) + from src.proxy.git_proxy import GitProxy + from src.proxy.es_proxy import ESProxy + from src.utils.dist_dataset import DistDataset + + dd = DistDataset() + dd.set_attr_stime("access_control.job.stime") + + # info from args + dd.set_attr("id", args.comment_id) + dd.set_attr("pull_request.package", args.repo) + dd.set_attr("pull_request.number", args.pr) + dd.set_attr("pull_request.author", args.committer) + dd.set_attr("pull_request.target_branch", args.tbranch) + dd.set_attr("pull_request.ctime", args.pr_ctime) + dd.set_attr("access_control.trigger.link", args.trigger_link) + dd.set_attr("access_control.trigger.reason", args.comment) + ctime = datetime.datetime.strptime(args.trigger_time.split("+")[0], "%Y-%m-%dT%H:%M:%S") + dd.set_attr_ctime("access_control.job.ctime", ctime) + + ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) + + # download repo + dd.set_attr_stime("access_control.scm.stime") + gp = GitProxy.init_repository(args.repo, work_dir=args.workspace) + repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account, args.community, args.repo) + if not gp.fetch_pull_request(repo_url, args.pr, depth=4): + dd.set_attr("access_control.scm.result", "failed") + dd.set_attr_etime("access_control.scm.etime") + + dd.set_attr_etime("access_control.job.etime") + dd.set_attr("access_control.job.result", "successful") + ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) + sys.exit(-1) + else: + gp.checkout_to_commit_force("pull/{}/MERGE".format(args.pr)) + dd.set_attr("access_control.scm.result", "successful") + dd.set_attr_etime("access_control.scm.etime") + + # build start + dd.set_attr_stime("access_control.build.stime") + + # gitee pr tag + gp = GiteeProxy(args.community, args.repo, args.token) gp.delete_tag_of_pr(args.pr, "ci_successful") gp.delete_tag_of_pr(args.pr, "ci_failed") gp.create_tags_of_pr(args.pr, "ci_processing") - ac = AC(os.path.join(os.path.dirname(os.path.realpath(__file__)), "ac.yaml")) - ac.check_all(workspace=args.workspace, repo=args.repo, tbranch=args.tbranch) + # build + ac = AC(os.path.join(os.path.dirname(os.path.realpath(__file__)), "ac.yaml"), args.community) + ac.check_all(workspace=args.workspace, repo=args.repo, dataset=dd, tbranch=args.tbranch) + dd.set_attr_etime("access_control.build.etime") ac.save(args.output) + + dd.set_attr_etime("access_control.job.etime") + dd.set_attr("access_control.job.result", "successful") + ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 1785f2d14b5a012dbc7d769469c1af70ccc71ca5..8a229052ac8c8fadcfa5ccab544b24a7a2dbd136 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -1,11 +1,30 @@ -spec: - hint: check_spec_file - module: spec.check_spec - entry: CheckSpec - ignored: ["homepage"] -code: - hint: check_code_style - module: code.check_code_style - entry: CheckCodeStyle - #exclude: True - ignored: [] +src-openeuler: + spec: + hint: check_spec_file + module: spec.check_spec + entry: CheckSpec + ignored: ["homepage"] + code: + hint: check_code_style + module: code.check_code_style + entry: CheckCodeStyle + #exclude: True + ignored: [] + package_yaml: + hint: check_package_yaml_file + module: package_yaml.check_yaml + entry: CheckPackageYaml + ignored: ["fields"] + package_license: + hint: check_package_license + module: package_license.check_license + entry: CheckLicense +openeuler: + spec: + exclude: True + code: + exclude: True + package_yaml: + exclude: True + package_license: + exclude: True diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 01c09ce5d806db9468b97f31e855f3650ce4143f..edb49b243ebdfd1494d185e4918e51d4e10fbae6 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -52,6 +52,8 @@ class Comment(object): comments = self._comment_build_html_format() gitee_proxy.comment_pr(self._pr, "\n".join(comments)) + return "\n".join(comments) + def comment_at(self, committer, gitee_proxy): """ 通知committer @@ -228,9 +230,14 @@ class Comment(object): return "{} {}{}".format(name, icon, status) -if "__main__" == __name__: +def init_args(): + """ + init args + :return: + """ args = argparse.ArgumentParser() args.add_argument("-p", type=int, dest="pr", help="pull request number") + args.add_argument("-m", type=int, dest="comment_id", help="uniq comment id") args.add_argument("-c", type=str, dest="committer", help="commiter") args.add_argument("-o", type=str, dest="owner", help="gitee owner") args.add_argument("-r", type=str, dest="repo", help="repo name") @@ -244,8 +251,11 @@ if "__main__" == __name__: args.add_argument("--disable", dest="enable", default=True, action="store_false", help="comment to gitee switch") - args = args.parse_args() + return args.parse_args() + +if "__main__" == __name__: + args = init_args() if not args.enable: sys.exit(0) @@ -256,25 +266,49 @@ if "__main__" == __name__: from src.ac.framework.ac_result import ACResult, SUCCESS from src.proxy.gitee_proxy import GiteeProxy + from src.proxy.es_proxy import ESProxy from src.proxy.jenkins_proxy import JenkinsProxy + from src.utils.dist_dataset import DistDataset - # gitee notify + dd = DistDataset() + dd.set_attr_stime("comment.job.stime") + + # gitee pr tag gp = GiteeProxy(args.owner, args.repo, args.gitee_token) gp.delete_tag_of_pr(args.pr, "ci_processing") - jp = JenkinsProxy(args.jenkins_base_url, args.jenkins_user, args.jenkins_api_token) + url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), int(os.environ.get("BUILD_ID"))) + dd.set_attr("comment.job.link", url) + dd.set_attr("comment.trigger.reason", reason) + dd.set_attr_ctime("comment.job.ctime", build_time) - if args.check_abi_comment_files: - comment = Comment(args.pr, jp, *args.check_abi_comment_files) - else: - comment = Comment(args.pr, jp) - logger.info("comment: build result......") - comment.comment_build(gp) + dd.set_attr_stime("comment.build.stime") + comment = Comment(args.pr, jp, *args.check_abi_comment_files) \ + if args.check_abi_comment_files else Comment(args.pr, jp) + logger.info("comment: build result......") + content = comment.comment_build(gp) + dd.set_attr_etime("comment.build.etime") + dd.set_attr("comment.build.content.html", content) + if comment.check_build_result() == SUCCESS: gp.create_tags_of_pr(args.pr, "ci_successful") + dd.set_attr("comment.build.tags", ["ci_successful"]) + dd.set_attr("comment.build.result", "successful") else: gp.create_tags_of_pr(args.pr, "ci_failed") + dd.set_attr("comment.build.tags", ["ci_failed"]) + dd.set_attr("comment.build.result", "failed") + logger.info("comment: at committer......") comment.comment_at(args.committer, gp) + + dd.set_attr_etime("comment.job.etime") + dd.set_attr("comment.job.result", "successful") + + # upload to es + ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) + query = {"term": {"id": args.comment_id}} + script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} + ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index de36b28b71714ac60187870c125d77e0e18f09c7..fa024df31e5547758b4cbdb3a8a29c471e70ff1e 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -28,12 +28,13 @@ class SinglePackageBuild(object): """ GITEE_BRANCH_PROJECT_MAPPING = { - "master": ["bringInRely", "openEuler:Extras", "openEuler:Factory", "openEuler:Mainline"], + "master": ["bringInRely", "openEuler:Extras", "openEuler:Factory", "openEuler:Mainline", "openEuler:Epol"], "openEuler-20.03-LTS": ["openEuler:20.03:LTS"], - "openEuler-20.03-LTS-Next": ["openEuler:20.03:LTS:Next"], + "openEuler-20.03-LTS-Next": ["openEuler:20.03:LTS:Next", "openEuler:20.03:LTS:Next:Epol"], "openEuler-EPOL-LTS": ["bringInRely"], - "openEuler-20.09": ["openEuler:20.09"], - "mkopeneuler-20.03": ["openEuler:Extras"] + "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol"], + "mkopeneuler-20.03": ["openEuler:Extras"], + "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] @@ -137,6 +138,7 @@ class SinglePackageBuild(object): param.text = "tar_local" elif param.get("name") == "url": if "openEuler_kernel" in param.text or "LTS_kernel" in param.text \ + or "openEuler-kernel" in param.text \ or "openEuler-20.09_kernel" in param.text: param.text = "{}/{}".format(code_path, "code") # kernel special logical else: @@ -180,11 +182,11 @@ class SinglePackageBuild(object): """ if self._branch in self.BUILD_IGNORED_GITEE_BRANCH: logger.error("branch \"{}\" ignored".format(self._branch)) - sys.exit(0) + return 0 if self._branch not in self.GITEE_BRANCH_PROJECT_MAPPING: logger.error("branch \"{}\" not support yet".format(self._branch)) - sys.exit(1) + return 1 for project in self.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): logger.debug("start build project {}".format(project)) @@ -199,12 +201,18 @@ class SinglePackageBuild(object): if ret > 0: logger.debug("build run return {}".format(ret)) logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) - sys.exit(1) # finish if any error + return 1 # finish if any error else: logger.info("build {} {} {} ... {}".format(project, self._package, self._arch, "ok")) + return 0 + -if "__main__" == __name__: +def init_args(): + """ + init args + :return: + """ args = argparse.ArgumentParser() args.add_argument("-p", type=str, dest="package", help="obs package") @@ -213,15 +221,66 @@ if "__main__" == __name__: args.add_argument("-c", type=str, dest="code", help="code dir path") args.add_argument("-w", type=str, dest="workspace", default=os.getcwd(), help="obs workspace dir path") - args = args.parse_args() + args.add_argument("-m", type=str, dest="comment_id", help="uniq comment id") + args.add_argument("-r", type=str, dest="repo", help="repo") + args.add_argument("--pr", type=str, dest="pr", help="pull request") + args.add_argument("-t", type=str, dest="account", help="gitee account") + + args.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + + return args.parse_args() + + +if "__main__" == __name__: + args = init_args() _ = not os.path.exists("log") and os.mkdir("log") logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") + from src.utils.dist_dataset import DistDataset + from src.proxy.git_proxy import GitProxy from src.proxy.obs_proxy import OBSProxy + from src.proxy.es_proxy import ESProxy from src.utils.shell_cmd import shell_cmd_live + dd = DistDataset() + dd.set_attr_stime("spb.job.stime") + + ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) + + # download repo + dd.set_attr_stime("spb.scm.stime") + gp = GitProxy.init_repository(args.repo, work_dir=args.workspace) + repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account, args.owner, args.repo) + if not gp.fetch_pull_request(repo_url, args.pr, depth=1): + dd.set_attr("spb.scm.result", "failed") + dd.set_attr_etime("spb.scm.etime") + dd.set_attr_etime("spb.job.etime") + dd.set_attr("spb.job.result", "failed") + + # upload to es + query = {"term": {"id": args.comment_id}} + script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), + "params": dd.to_dict()} + ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) + sys.exit(-1) + else: + gp.checkout_to_commit_force("pull/{}/MERGE".format(args.pr)) + dd.set_attr("spb.scm.result", "successful") + dd.set_attr_etime("spb.scm.etime") + + dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - spb.build(args.workspace, args.code) + rs = spb.build(args.workspace, args.code) + dd.set_attr("spb.job.result", "failed" if rs else "successful") + dd.set_attr_etime("spb.build.etime") + + dd.set_attr_etime("spb.job.etime") + + # upload to es + query = {"term": {"id": args.comment_id}} + script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} + ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) + sys.exit(rs) diff --git a/src/build/related_rpm_package.py b/src/build/related_rpm_package.py index 91aeefd7143b624314a2cffbf7682a0f9f5080f8..627000a9e904804e426a132fee93fb3dca603a84 100755 --- a/src/build/related_rpm_package.py +++ b/src/build/related_rpm_package.py @@ -36,7 +36,8 @@ class RelatedRpms(object): "openEuler-20.03-LTS-Next": ["openEuler:20.03:LTS:Next"], "openEuler-EPOL-LTS": ["bringInRely"], "openEuler-20.09": ["openEuler:20.09"], - "mkopeneuler-20.03": ["openEuler:Extras"] + "mkopeneuler-20.03": ["openEuler:Extras"], + "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1"] } def __init__(self, obs_addr, obs_repo_url, branch_name, package_arch): diff --git a/src/dockerfile/openeuler-base b/src/dockerfile/jenkins-openeuler similarity index 90% rename from src/dockerfile/openeuler-base rename to src/dockerfile/jenkins-openeuler index 88c94fd68c5f57a8e3e295b351c99e24f4498996..98ffa419b8a475cb358da0c1b71288e2162e4a9c 100644 --- a/src/dockerfile/openeuler-base +++ b/src/dockerfile/jenkins-openeuler @@ -1,4 +1,5 @@ -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk-openeuler:11-jdk-stretch +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler-VERSION/openjdk:11-jdk-stretch ARG VERSION=4.3 ARG user=jenkins @@ -7,8 +8,6 @@ ARG uid=1000 ARG gid=1000 ARG AGENT_WORKDIR=/home/${user}/agent -RUN yum install -y shadow git - RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ && chmod 755 /usr/share/jenkins \ && chmod 644 /usr/share/jenkins/agent.jar \ diff --git a/src/dockerfile/openjdk-openeuler b/src/dockerfile/openjdk-openeuler index 1f264e30fd16e103f0dcf13b5dadcfdf50efdf4b..aaf591440970344bdbaf2421b89b9eb8caef92bc 100644 --- a/src/dockerfile/openjdk-openeuler +++ b/src/dockerfile/openjdk-openeuler @@ -1,7 +1,8 @@ -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts-08-20 +# replace VERSION before build +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:VERSION RUN set -eux; \ - yum install -y tar wget + yum install -y tar wget python3 expect osc vim openssh shadow git # Default to UTF-8 file.encoding ENV LANG C.UTF-8 diff --git a/src/jenkinsfile/openeuler_jenkins_test.py b/src/jenkinsfile/openeuler_jenkins_test.py new file mode 100644 index 0000000000000000000000000000000000000000..88d5ee81306b3ceb6b88aa593a2773ca7cecc33e --- /dev/null +++ b/src/jenkinsfile/openeuler_jenkins_test.py @@ -0,0 +1,46 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-21 +# Description: run all test cases in test/ +# ***********************************************************************************/ +""" + +import os +import unittest +import logging.config +import logging +import mock +import shutil + +PATTERN_STR = 'test*.py' # 测试用例文件必须以test_开头 +TEST_DIR = os.path.realpath(os.path.join(os.path.realpath(__file__), '..', '..', '..', 'test')) +LOG_FILE = os.path.realpath(os.path.join(os.path.realpath(__file__), '..', 'test.log')) + +if __name__ == '__main__': + # mock所有logger为测试使用logger,输出为该文件同目录下的test.log + wordk_dir = os.getcwd() + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + + # 递归获取test目录下所有module的所有测试用例(需包含__init__.py) + discover_sets = unittest.defaultTestLoader.discover(start_dir=TEST_DIR, pattern=PATTERN_STR, top_level_dir=None) + suite = unittest.TestSuite() + suite.addTest(discover_sets) + # 运行所有测试用例 + test_runner = unittest.TextTestRunner() + test_runner.run(suite) + os.chdir(wordk_dir) + shutil.rmtree("log") \ No newline at end of file diff --git a/src/proxy/es_proxy.py b/src/proxy/es_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..0304dcaabc772696a06bc696878a8cedc193787c --- /dev/null +++ b/src/proxy/es_proxy.py @@ -0,0 +1,101 @@ +# -*- encoding=utf-8 -*- +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-12-30 +# Description: elasticsearch proxy +# ********************************************************************************** + +import logging +from elasticsearch import Elasticsearch, RequestError + +logger = logging.getLogger("common") + + +class ESProxy(object): + """ + es 代理 + """ + def __init__(self, username, password, hosts=None, timeout=30, **kwargs): + """ + + :param username: + :param password: + :param hosts: 参考Elasticsearch.__init__描述 + :param timeout: + """ + + self._timeout = timeout + self._es = Elasticsearch(hosts=hosts, http_auth=(username, password), timeout=self._timeout, **kwargs) + + def insert(self, index, body): + """ + 插入一条数据 + :param index: + :param body: + :return: + """ + try: + rs = self._es.index(index, body=body) + + return rs["result"] == "created" + except RequestError: + logger.exception("elasticsearch insert document exception") + return False + + def search(self, index, body): + """ + 条件搜索 + :param index: + :param body: + :return: + """ + rs = self._es.search(index=index, body=body) + + return rs['hits']['hits'] + + def update_by_query(self, index, query, script): + """ + 更新一条数据，原数据不变 + eg: + query = { + "term": {"id": 567} + } + script = { + "source": "ctx._source.tags = params.tags", + "params": { + "tags": tags + }, + "lang":"painless" + } + :param index: + :param query: + :param script: + :return: + """ + body = {"query": query, "script": script} + try: + rs = self._es.update_by_query(index, body=body) + logger.debug("elasticsearch update {} documents".format(rs["updated"])) + + return True + except RequestError: + logger.exception("elasticsearch update by query exception") + return False + + +host="https://119.8.112.14:9200" +pwd="Cloudfoundry@123" + +es = Elasticsearch(hosts=host, http_auth=("admin", pwd), verify_certs=False) +i = "openeuler_statewall_ac" +from pprint import pprint +pprint(es.search(index=i, body={"query": {"term": {"id": 9999}}})["hits"]["hits"]) diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index 712f45e5684e3e97a6f2a71df89d93db37bec1bc..204ab371499ee528c18f7bfd7743b1f2de32b0bf 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -1,4 +1,5 @@ # -*- encoding=utf-8 -*- +import os import logging from cStringIO import StringIO @@ -17,6 +18,43 @@ class GitProxy(object): """ self._repo_dir = repo_dir + @classmethod + def init_repository(cls, sub_dir, work_dir=None): + """ + 初始化git仓库 + :param sub_dir: 仓库子目录 + :param work_dir: 仓库根目录 + :return: GitProxy() or None + """ + repo_dir = os.path.join(work_dir, sub_dir) if work_dir else sub_dir + + init_cmd = "git init {}".format(repo_dir) + ret, _, _ = shell_cmd_live(init_cmd) + + if ret: + logger.warning("init repository failed, {}".format(ret)) + return None + + return cls(repo_dir) + + def fetch_pull_request(self, url, pull_request, depth=1, progress=False): + """ + fetch pr + :param url: 仓库地址 + :param pull_request: pr编号 + :param depth: 深度 + :param progress: 展示进度 + :return: + """ + fetch_cmd = "cd {}; git fetch {} --depth {} {} +refs/pull/{}/MERGE:refs/pull/{}/MERGE".format( + self._repo_dir, "--progress" if progress else "", depth, url, pull_request, pull_request) + ret, _, _ = shell_cmd_live(fetch_cmd, cap_out=True, cmd_verbose=False) + if ret: + logger.error("git fetch failed, {}".format(ret)) + return False + + return True + def get_content_of_file_with_commit(self, file_path, commit="HEAD~0"): """ 获取单个commit文件内容 @@ -132,6 +170,21 @@ class GitProxy(object): return True + def checkout_to_commit_force(self, commit): + """ + git checkout + :param commit: HEAD~{} or SHA + :return: boolean + """ + checkout_cmd = "cd {}; git checkout -f {}".format(self._repo_dir, commit) + ret, _, _ = shell_cmd_live(checkout_cmd) + + if ret: + logger.warning("checkout failed, {}".format(ret)) + return False + + return True + def get_tree_hashes(self, commit, number=0, with_merges=True): """ 获取tree对象hash值 diff --git a/src/proxy/jenkins_proxy.py b/src/proxy/jenkins_proxy.py index 63fd8efa56a085cf6fd6839647cb4ca387cd61f9..b4b035bd1cc2a9c1c9629217dacc468a1ea7543b 100644 --- a/src/proxy/jenkins_proxy.py +++ b/src/proxy/jenkins_proxy.py @@ -22,6 +22,7 @@ import re # not friendly when job in folders import jenkinsapi.jenkins as jenkins import src.proxy.jenkins_patch +from src.utils.dt_transform import convert_utc_to_naive logger = logging.getLogger("common") @@ -181,3 +182,52 @@ class JenkinsProxy(object): break return upstream_builds + + def get_job_url(self, job): + """ + 获取任务url + :param job: + :return: + """ + j = self._jenkins[job] + + return self._jenkins[job].url + + def get_build_datetime(self, job, build): + """ + get job build tims + :param job: + :param build: + :return: + """ + return convert_utc_to_naive(self._jenkins[job].get_build(build).get_timestamp()) + + def get_build_trigger_reason(self, job, build): + """ + get job build tims + :param job: + :param build: + :return: + """ + causes = self._jenkins[job].get_build(build).get_causes() + if not causes: + return "N/A" + + return causes[0]["shortDescription"] + + def get_job_build_info(self, job, build): + """ + get job and build info + :param job: + :param build: + :return: + """ + job = self._jenkins[job] + build = job.get_build(build) + + build_dt = convert_utc_to_naive(build.get_timestamp()) + + causes = build.get_causes() + trigger_reason = causes[0]["shortDescription"] if causes else "N/A" + + return job.url, build_dt, trigger_reason diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 15a8fedb702ba1436465ae30d1721d0d7935f75a..eee8ad139bcdf070d7565678d533e871052c8cc1 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -132,3 +132,21 @@ class OBSProxy(object): return False return True + + @staticmethod + def build_history(project, package, repo, arch): + """ + 构建历史 + :param project: + :param package: + :param repo: + :param arch: + :return: + """ + cmd = "osc api /build/{}/{}/{}/{}/_history".format(project, repo, arch, package) + ret, out, _ = shell_cmd_live(cmd, cap_out=True) + if ret: + logger.debug("list build history of package error, {}".format(ret)) + return "" + + return "\n".join(out) diff --git a/src/requirements b/src/requirements index 30e813f0c12d1fb314fa1391865566e27fe019df..34e139555ef82db1274bee08b2b88dd6eb2d7706 100644 --- a/src/requirements +++ b/src/requirements @@ -5,3 +5,6 @@ threadpool PyYAML gevent==1.2.2 jsonpath +mock +tldextract +chardet \ No newline at end of file diff --git a/src/tools/obs_package_build_history.py b/src/tools/obs_package_build_history.py new file mode 100644 index 0000000000000000000000000000000000000000..cd0863785e49dae50e791b254884c10227969e8c --- /dev/null +++ b/src/tools/obs_package_build_history.py @@ -0,0 +1,130 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-29 +# Description: report obs package build info +# ********************************************************************************** +""" + +import gevent +from gevent import monkey +monkey.patch_all() +import logging.config +import logging +import os +import argparse +import time +import xml.etree.ElementTree as ET +from xml.etree.ElementTree import ParseError +import json + +logger = logging.getLogger("common") + + +class JobBuildHistory(object): + @staticmethod + def get_package_job_duration(project, package, repo, arch): + """ + 获取任务构建时间信息 + :param project: + :param package: + :param repo: + :param arch: + :return: + """ + history = OBSProxy.build_history(project, package, repo, arch) + try: + root = ET.fromstring(history) + except ParseError: + logger.exception("package: {}, build history: {}".format(package, history)) + return {"package": package, "max": 0, "min": 0, "average": 0, "times": -1} + + duration = [int(ele.get("duration")) for ele in root.findall("entry")] + + if not duration: + return {"package": package, "max": 0, "min": 0, "average": 0, "times": 0, "duration": []} + + return {"package": package, "max": max(duration), "min": min(duration), "duration": duration, + "average": sum(duration) / len(duration), "times": len(duration)} + + @staticmethod + def get_packages_job_duration(project, repo, arch, concurrency, *packages): + """ + 获取多个包任务构建时间信息 + :param project: + :param packages: + :param repo: + :param arch: + :param concurrency: + :return: + """ + batch = (len(packages) + concurrency - 1) / concurrency + + rs = [] + for index in range(batch): + works = [gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) + for package in packages[index * concurrency: (index + 1) * concurrency]] + logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + gevent.joinall(works) + for work in works: + logger.debug("{}: {}".format(work.value["package"], work.value)) + logger.info("{} ...done".format(work.value["package"])) + rs.append(work.value) + + time.sleep(1) + + rs.sort(key=lambda item: item["max"]) + return rs + + @staticmethod + def get_jobs_duration(project, repo, arch, concurrency=50): + """ + 获取项目下所有包构建时间信息 + :param project: + :param repo: + :param arch: + :return: + """ + packages = OBSProxy.list_project(project) + + return JobBuildHistory.get_packages_job_duration(project, repo, arch, concurrency, *packages) + + +if "__main__" == __name__: + args = argparse.ArgumentParser() + + args.add_argument("-p", type=str, dest="project", help="obs project") + args.add_argument("-g", type=str, dest="packages", nargs="+", help="package") + args.add_argument("-r", type=str, dest="repo", help="repo") + args.add_argument("-a", type=str, dest="arch", help="arch") + args.add_argument("-c", type=str, dest="concurrency", help="concurrency for obs") + args.add_argument("-o", type=str, dest="output", help="output file") + + args = args.parse_args() + + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("build") + + from src.proxy.obs_proxy import OBSProxy + + if args.packages: + result = JobBuildHistory.get_packages_job_duration(args.project, args.repo, args.arch, + int(args.concurrency), *args.packages) + else: + result = JobBuildHistory.get_jobs_duration(args.project, args.repo, args.arch, int(args.concurrency)) + + if args.output: + with open(args.output, "w") as f: + json.dump(result, f) diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index 543af26fa27aa2fb4c1a2b68b8253465d3f419c2..c209f1735cdb7eeab4182c37893a7738e6f305d3 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -32,8 +32,14 @@ class ObsPackageBuildReport(object): PROJECT_BRANCH_MAPPING = { "openEuler:Factory": "master", "openEuler:Mainline": "master", + "openEuler:Epol": "master", "openEuler:20.03:LTS": "openEuler-20.03-LTS", - "openEuler:20.09": "openEuler-20.09" + "openEuler:20.09": "openEuler-20.09", + "openEuler:20.09:Epol": "openEuler-20.09", + "openEuler:20.03:LTS:Next": "openEuler-20.03-LTS-Next", + "openEuler:20.03:LTS:Next:Epol": "openEuler-20.03-LTS-Next", + "openEuler:20.03:LTS:SP1": "openEuler-20.03-LTS-SP1", + "openEuler:20.03:LTS:SP1:Epol": "openEuler-20.03-LTS-SP1" } GITEE_OWNER = "src-openeuler" @@ -74,11 +80,12 @@ class ObsPackageBuildReport(object): :param gitee_api_token: 码云api token :return: """ - try: - branch = self.__class__.PROJECT_BRANCH_MAPPING[self._project] - except KeyError: - logger.exception("project {} not support".format(self._project)) - return +# try: +# branch = self.__class__.PROJECT_BRANCH_MAPPING[self._project] +# except KeyError: +# logger.exception("project {} not support".format(self._project)) +# return + branch = "master" # get packages in project of state packages = OBSProxy.list_packages_of_state(self._project, self._state) @@ -122,8 +129,8 @@ if "__main__" == __name__: from src.proxy.obs_proxy import OBSProxy from src.proxy.gitee_proxy import GiteeProxy - for project in args.project: - for state in args.state: - report = ObsPackageBuildReport(project, state, args.real_name_mapping_file) + for prj in args.project: + for states in args.state: + report = ObsPackageBuildReport(prj, states, args.real_name_mapping_file) report.get_last_committer(args.token) report.dump() diff --git a/src/utils/check_conf.py b/src/utils/check_conf.py index 9a181bdde0be6cafe31f4007a639c02d8559d924..3b65c80fafb43c1b0a81b1b59b325a493077abc7 100755 --- a/src/utils/check_conf.py +++ b/src/utils/check_conf.py @@ -96,8 +96,8 @@ class CheckConfig(object): """ Check md5sum """ - old_md5 = subprocess.run(['md5sum', old_rpm], stdout=subprocess.PIPE, encoding='utf-8') - new_md5 = subprocess.run(['md5sum', new_rpm], stdout=subprocess.PIPE, encoding='utf-8') + old_md5 = subprocess.run(['md5sum', old_rpm], stdout=subprocess.PIPE) + new_md5 = subprocess.run(['md5sum', new_rpm], stdout=subprocess.PIPE) return old_md5.stdout.split()[0] == new_md5.stdout.split()[0] def _check_diff(self, old_and_new_path): @@ -105,6 +105,7 @@ class CheckConfig(object): Check diff file """ for name in self._need_check_file: + logging.info("config file:%s", name) name = name.split("/", 1)[-1].split()[0] logging.debug("path:%s", old_and_new_path) if self._md5_check(os.path.join(old_and_new_path[0], name), os.path.join(old_and_new_path[1], name)): @@ -162,6 +163,7 @@ class CheckConfig(object): logging.info("\n---Change infos write at:%s----", self._output_file) else: logging.info("\n---Configs are same----") + os.remove(self._output_file) ofile.close() def _get_rpms(self, rpm_url, dest): @@ -189,8 +191,8 @@ class CheckConfig(object): if self._md5_check(self._old_rpm, self._new_rpm): logging.info("Same RPM") return - old_config = subprocess.run(['rpm', '-qpc', self._old_rpm], stdout=subprocess.PIPE, encoding='utf-8') - new_config = subprocess.run(['rpm', '-qpc', self._new_rpm], stdout=subprocess.PIPE, encoding='utf-8') + old_config = subprocess.run(['rpm', '-qpc', self._old_rpm], stdout=subprocess.PIPE) + new_config = subprocess.run(['rpm', '-qpc', self._new_rpm], stdout=subprocess.PIPE) for line in old_config.stdout.split(): self._remove_file.add(line) for line in new_config.stdout.split(): diff --git a/src/utils/dist_dataset.py b/src/utils/dist_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..64e160338c78cd36eb15923225449e23fa581dda --- /dev/null +++ b/src/utils/dist_dataset.py @@ -0,0 +1,111 @@ +# -*- encoding=utf-8 -*- +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: gitee api proxy +# ********************************************************************************** + +import logging +from datetime import datetime + +from src.utils.dot_json import MutableDotJson + +logger = logging.getLogger("common") + + +class DistDataset(object): + """ + 分布式数据集，用来保存门禁各阶段性结果 + """ + def __init__(self): + """ + 初始化 + """ + self._json = MutableDotJson() + + def to_dict(self): + """ + + :return: + """ + return self._json.to_dict() + + def set_attr(self, attr, value): + """ + 属性赋值 + :param attr: 属性 + :param value: 值 + :return: + """ + self._json[attr] = value + + def set_attr_ctime(self, attr, ctime): + """ + 属性【开始时间】赋值 + :param attr: + :param ctime: jenkins build time + :return: + """ + self._json[attr] = ctime.strftime("%Y-%m-%d %H:%M:%S") + + attrs = attr.split(".") + # set pending + attrs[-1] = "stime" + try: + stime_str = self._json[".".join(attrs)] + stime = datetime.strptime(stime_str, "%Y-%m-%d %H:%M:%S") + pending = (stime - ctime).total_seconds() # seconds + attrs[-1] = "pending" + self._json[".".join(attrs)] = int(pending) + except KeyError: + logger.debug("no correspond stime") + + def set_attr_stime(self, attr): + """ + 属性【开始时间】赋值 + :param attr: + :return: + """ + now = datetime.now() + self._json[attr] = now.strftime("%Y-%m-%d %H:%M:%S") + + def set_attr_etime(self, attr): + """ + 属性【结束时间】赋值 + :param attr: + :return: + """ + now = datetime.now() + self._json[attr] = now.strftime("%Y-%m-%d %H:%M:%S") + + attrs = attr.split(".") + # set elapse + attrs[-1] = "stime" + try: + stime_str = self._json[".".join(attrs)] + stime = datetime.strptime(stime_str, "%Y-%m-%d %H:%M:%S") + elapse = (now - stime).total_seconds() # seconds + attrs[-1] = "elapse" + self._json[".".join(attrs)] = int(elapse) + except KeyError: + logger.debug("no correspond stime") + + # set interval + attrs[-1] = "ctime" + try: + ctime_str = self._json[".".join(attrs)] + ctime = datetime.strptime(ctime_str, "%Y-%m-%d %H:%M:%S") + elapse = (now - ctime).total_seconds() # seconds + attrs[-1] = "interval" + self._json[".".join(attrs)] = int(elapse) + except KeyError: + logger.debug("no correspond ctime") diff --git a/src/utils/dot_json.py b/src/utils/dot_json.py new file mode 100644 index 0000000000000000000000000000000000000000..d51179edbd8cad0d3d35b4e28c16b745db04df57 --- /dev/null +++ b/src/utils/dot_json.py @@ -0,0 +1,128 @@ +# -*- encoding=utf-8 -*- +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-12-22 +# Description: one kind of json that support access value with continue dot attribute +# ********************************************************************************** + +from collections import Mapping, MutableSequence, MutableMapping + + +class DotJson(object): + """ + one kind of json that support access value with continue dot attribute + """ + def __init__(self, data=None): + """ + 初始化 + :param data: Mapping or Sequence + """ + if isinstance(data, Mapping): + self._data = dict(data) + elif isinstance(data, MutableSequence): + self._data = list(data) + elif data is None: + self._data = dict() + else: + raise Exception("init with type {} not support".format(type(data))) + + def to_dict(self): + """ + dict格式 + :return: + """ + return self._data + + def __str__(self): + """ + + :return: + """ + return str(self._data) + + def __getitem__(self, item): + """ + 支持"."符号连接的嵌套属性取值 + :param item: + :return: + """ + if isinstance(item, int): + return self.__class__.build(self._data[item]) + + s = self + for attr in item.split("."): # eg: item = "a.0.b" + s = s.get(attr) + + return s + + def get(self, item): + """ + 取值辅助函数 + :param item: + :return: + """ + if isinstance(self._data, MutableSequence) and item.isdigit(): # 嵌套属性中的数字分量 + item = int(item) + + return self.__class__.build(self._data[item]) + + def __getattr__(self, item): + """ + 获取属性 + :param item: + :return: + """ + try: + return self[item] + except (IndexError, KeyError, TypeError) as e: + raise AttributeError(e) + + @classmethod + def build(cls, obj): + """ + 自举 + :param obj: + :return: + """ + return cls(obj) if isinstance(obj, (Mapping, MutableSequence)) else obj + + +class MutableDotJson(DotJson): + """ + 可变 + """ + def __setitem__(self, key, value): + """ + 赋值 + :param key: + :param value: + :return: + """ + s = self._data + attrs = key.split(".") + for attr in attrs[:-1]: + if isinstance(s, MutableMapping): + if attr not in s: + s[attr] = {} + s = s[attr] + elif isinstance(s, MutableSequence): + if attr.isdigit(): + s = s[int(attr)] + else: + # set Sequence, but index is not digit + raise TypeError("list indices must be integers or slices, not str") + else: + # object is not Sequence or Mapping + raise TypeError("set value for {} not support".format(type(s))) + + last_attr = attrs[-1] + s[last_attr] = value diff --git a/src/utils/dt_transform.py b/src/utils/dt_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..d0801350b57cdb42b35dbd284a39b4297c0f2743 --- /dev/null +++ b/src/utils/dt_transform.py @@ -0,0 +1,30 @@ +# -*- encoding=utf-8 -*- +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: gitee api proxy +# ********************************************************************************** + +from datetime import datetime +import calendar + + +def convert_utc_to_naive(utc_dt): + """ + utc datetime to local datetime + :param utc_dt: utc datetime + :return: + """ + ts = calendar.timegm(utc_dt.timetuple()) + naive_dt = datetime.fromtimestamp(ts) + + return naive_dt.replace(microsecond=utc_dt.microsecond) diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index a6168f7c8b42ac5bda67bb4fe74d78455162d165..d3bc10cc92b5b8cb2678e8bd20326d23724438cd 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -18,7 +18,7 @@ def shell_cmd(cmd, inmsg=None): return p.returncode, out, err -def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False): +def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False, cmd_verbose=True): """ 创建子进程执行命令，实时输出结果 :param cmd: 命令 @@ -28,7 +28,8 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False :param verbose: show cmd output to console, default not :return: """ - logger.debug("exec cmd -- {}".format(cmd)) + if cmd_verbose: + logger.debug("exec cmd -- {}".format(cmd)) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if cap_in: diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5bdad435b94b1475a582e9f6602dd020179de1 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-21 +# Description: +# ***********************************************************************************/ +""" \ No newline at end of file diff --git a/test/ac/__init__.py b/test/ac/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5bdad435b94b1475a582e9f6602dd020179de1 --- /dev/null +++ b/test/ac/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-21 +# Description: +# ***********************************************************************************/ +""" \ No newline at end of file diff --git a/test/ac/acl/__init__.py b/test/ac/acl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5bdad435b94b1475a582e9f6602dd020179de1 --- /dev/null +++ b/test/ac/acl/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-21 +# Description: +# ***********************************************************************************/ +""" \ No newline at end of file diff --git a/test/ac/acl/license/__init__.py b/test/ac/acl/license/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..36e95ff3db7d173ff656de1605ce299b4e77f17a --- /dev/null +++ b/test/ac/acl/license/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-10-16 +# Description: check spec file +# ********************************************************************************** +""" \ No newline at end of file diff --git a/test/ac/acl/license/license_test_sample/no_spec/no_spec b/test/ac/acl/license/license_test_sample/no_spec/no_spec new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/license_test_sample/no_src/no_src b/test/ac/acl/license/license_test_sample/no_src/no_src new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/license_test_sample/pkgship/README.en.md b/test/ac/acl/license/license_test_sample/pkgship/README.en.md new file mode 100644 index 0000000000000000000000000000000000000000..ae5aff0dad8869a2b9b7b973d58a460cbc71015a --- /dev/null +++ b/test/ac/acl/license/license_test_sample/pkgship/README.en.md @@ -0,0 +1,36 @@ +# pkgship + +#### Description +a package depend query tool + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/test/ac/acl/license/license_test_sample/pkgship/README.md b/test/ac/acl/license/license_test_sample/pkgship/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b5a39bf91029b55ab201ba35a6b0957e3239519b --- /dev/null +++ b/test/ac/acl/license/license_test_sample/pkgship/README.md @@ -0,0 +1,176 @@ +[English](./README-en.md) | 简体中文 + +# pkgship + +## 介绍 +pkgship是一款管理OS软件包依赖关系，提供依赖和被依赖关系的完整图谱查询工具，pkgship提供软件包依赖，生命周期，补丁查询等功能。 +1. 软件包依赖：方便社区人员在新引入、软件包更新和删除的时候能方便的了解软件的影响范围。 +2. 生命周期管理：跟踪upstream软件包发布状态，方便维护人员了解当前软件状态，及时升级到合理的版本。 +3. 补丁查询：方便社区人员了解openEuler软件包的补丁情况，方便的提取补丁内容 + + +### 软件架构 +系统采用flask-restful开发，使用SQLAlchemy ORM查询框架，同时支持mysql和sqlite数据库，通过配置文件的形式进行更改 + + +安装教程 +--- +#### 方法一: 可以使用dnf挂载pkgship软件在所在repo源，直接下载安装pkgship及其依赖 + +``` +dnf install pkgship(版本号) +``` + +#### 方法二: 可以直接下载pkgship的rpm包后安装软件包 + +``` +rpm -ivh pkgship.rpm +``` +或者 +``` +dnf install pkgship-(版本号) +``` + +系统配置 +--- +系统的默认配置文件存放在 /etc/pkgship/packge.ini，请根据实际情况进行配置更改 + +``` +vim /etc/pkgship/package.ini +``` +创建初始化数据库的yaml配置文件: +conf.yaml 文件默认存放在 /etc/pkgship/ 路径下，pkgship会通过该配置读取要建立的数据库名称以及需要导入的sqlite文件。conf.yaml 示例如下： + +``` +- dbname:openEuler-20.03-LTS + src_db_file: +- /etc/pkgship/src.sqlite + bin_db_file: +- /etc/pkgship/bin.sqlite + status:enable + priority:1 +``` + +如需更改存放路径，请更改package.ini下的 init_conf_path 选项 + + +服务启动和停止 +--- +pkgship使用uWSGI web服务器 +``` +pkgshipd start + +pkgshipd stop +``` + +使用说明 +--- +#### 1. 数据库初始化 +``` +pkgship init +``` +#### 2. 单包查询 + +查询源码包(sourceName)在所有数据库中的信息 +``` +pkgship single sourceName +``` +查询当前包(sourceName)在指定数据库(dbName)中的信息 +``` +pkgship single sourceName -db dbName +``` +#### 3. 查询所有包 +查询所有数据库下包含的所有包的信息 +``` +pkgship list +``` +查询指定数据库(dbName)下的所有包的信息 +``` +pkgship list -db dbName +``` +#### 4. 安装依赖查询 +查询二进制包(binaryName)的安装依赖，按照默认优先级查询数据库 +``` +pkgship installdep binaryName +``` +在指定数据库(dbName)下查询二进制包(binaryName)的所有安装依赖 +按照先后顺序指定数据库查询的优先级 +``` +pkgship installdep binaryName -dbs dbName1 dbName2... +``` +#### 5. 编译依赖查询 +查询源码包(sourceName)的所有编译依赖，按照默认优先级查询数据库 +``` +pkgship builddep sourceName +``` +在指定数据库(dbName)下查询源码包(sourceName)的所有安装依赖 +按照先后顺序指定数据库查询的优先级 +``` +pkgship builddep sourceName -dbs dbName1 dbName2... +``` +#### 6. 自编译自安装依赖查询 +查询二进制包(binaryName)的安装和编译依赖，按照默认优先级查询数据库 +``` +pkgship selfbuild binaryName +``` +查询源码包(sourceName )的安装和编译依赖，按照默认优先级查询数据库 +``` +pkgship selfbuild sourceName -t source +``` +其他参数: + +-dbs 指定数据库优先级. +``` +示例:pkgship selfbuild binaryName -dbs dbName1 dbName2 +``` +-s 是否查询自编译依赖 +默认为0不查询自编译依赖，可以指定0或1(表示查询自编译) +``` +查询自编译示例:pkgship selfbuild sourceName -t source -s 1 +``` +-w 是否查询对应包的子包.默认为0，不查询对应子包，可以指定 0或1(表示查询对应子包) +``` +查询子包示例:pkgship selfbuild binaryName -w 1 +``` +#### 7. 被依赖查询 +查询源码包(sourceName)在某数据库(dbName)中被哪些包所依赖 +查询结果默认不包含对应二进制包的子包 +``` +pkgship bedepend sourceName -db dbName +``` +使查询结果包含二进制包的子包 加入参数 -w +``` +pkgship bedepend sourceName -db dbName -w 1 +``` +#### 8. 修改包信息记录 +变更数据库中(dbName)源码包(sourceName)的maintainer为Newmaintainer +``` +pkgship updatepkg sourceName db dbName -m Newmaintainer +``` +变更数据库中(dbName)源码包(sourceName)的maintainlevel为Newmaintainlevel，值在1～4之间 +``` +pkgship updatepkg sourceName db dbName -l Newmaintainlevel +``` +同时变更数据库中(dbName)源码包(sourceName)的maintainer 为Newmaintainer和变更maintainlevel为Newmaintainlevel +``` +pkgship updatepkg sourceName db dbName -m Newmaintainer -l Newmaintainlevel +``` +#### 9. 删除数据库 +删除指定数据库(dbName) +``` +pkgship rm db dbName +``` + +参与贡献 +--- +我们非常欢迎新贡献者加入到项目中来，也非常高兴能为新加入贡献者提供指导和帮助。在您贡献代码前，需要先签署[CLA](https://openeuler.org/en/cla.html)。 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request + + +### 会议记录 +1. 2020.5.18：https://etherpad.openeuler.org/p/aHIX4005bTY1OHtOd_Zc + diff --git a/test/ac/acl/license/license_test_sample/pkgship/pkgship-1.1.0.tar.gz b/test/ac/acl/license/license_test_sample/pkgship/pkgship-1.1.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9e45007af4b886171379b56d7a74feb9acce0481 Binary files /dev/null and b/test/ac/acl/license/license_test_sample/pkgship/pkgship-1.1.0.tar.gz differ diff --git a/test/ac/acl/license/license_test_sample/pkgship/pkgship.spec b/test/ac/acl/license/license_test_sample/pkgship/pkgship.spec new file mode 100644 index 0000000000000000000000000000000000000000..0abe1da5bfa7ef80919984e30bd89295a4a86e24 --- /dev/null +++ b/test/ac/acl/license/license_test_sample/pkgship/pkgship.spec @@ -0,0 +1,164 @@ +Name: pkgship +Version: 1.1.0 +Release: 14 +Summary: Pkgship implements rpm package dependence ,maintainer, patch query and so no. +License: Mulan 2.0 +URL: https://gitee.com/openeuler/openEuler-Advisor +Source0: https://gitee.com/openeuler/openEuler-Advisor/pkgship-%{version}.tar.gz + +# Modify the query logic of package information, reduce redundant queries and align dnf query results, +# extract multiplexing functions, add corresponding docString, and clear pylint +Patch0: 0001-solve-installation-dependency-query-error.patch + +# Fix the problem of continuous spaces in message information in log records +Patch1: 0002-fix-the-problem-of-continuous-spaces.patch + +# When initializing logging, modify the incoming class object to an instance of the class, +# ensure the execution of internal functions,and read configuration file content +Patch2: 0003-fix-log_level-configuration-item-not-work.patch + +# Fix the error when executing query commands +Patch3: 0004-fix-the-error-when-executing-query-commands.patch + +# Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, +# and reduce the judgment branch of the old code. +Patch4: 0005-fix-the-error-when-source-package-has-no-sub-packages.patch + +# Solve the problem of data duplication, increase the maximum queue length judgment, +# and avoid occupying too much memory +Patch5: 0006-fix-memory_caused-service-crash-and-data-duplication-issue.patch + +# Fix the problem of function parameters +Patch6: 0007-correct-the-parameter-transfer-method-and-change-the-status-recording-method.patch + +# Fix the selfbuild error message +Patch7: 0008-fix-selfbuild-error-message.patch + +# Optimize-log-records-when-obtaining-issue-content +Patch8: 0009-optimize-log-records-when-obtaining-issue-content.patch +BuildArch: noarch + +BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-sqlalchemy +BuildRequires: python3-prettytable python3-requests python3-flask-session python3-flask-script python3-marshmallow +BuildRequires: python3-Flask-APScheduler python3-pandas python3-retrying python3-xlrd python3-XlsxWriter +BuildRequires: python3-concurrent-log-handler +Requires: python3-pip python3-flask-restful python3-flask python3 python3-pyyaml +Requires: python3-sqlalchemy python3-prettytable python3-requests python3-concurrent-log-handler +Requires: python3-flask-session python3-flask-script python3-marshmallow python3-uWSGI +Requires: python3-pandas python3-dateutil python3-XlsxWriter python3-xlrd python3-Flask-APScheduler python3-retrying + +%description +Pkgship implements rpm package dependence ,maintainer, patch query and so no. + +%prep +%autosetup -n pkgship-%{version} -p1 + +%build +%py3_build + +%install +%py3_install + + +%check +# The apscheduler cannot catch the local time, so a time zone must be assigned before running the test case. +export TZ=Asia/Shanghai +# change log_path to solve default log_path permission denied problem +log_path=`pwd`/tmp/ +sed -i "/\[LOG\]/a\log_path=$log_path" test/common_files/package.ini +%{__python3} -m unittest test/init_test.py +%{__python3} -m unittest test/read_test.py +%{__python3} -m unittest test/write_test.py +rm -rf $log_path + +%post + +%postun + + +%files +%doc README.md +%{python3_sitelib}/* +%attr(0755,root,root) %config %{_sysconfdir}/pkgship/* +%attr(0755,root,root) %{_bindir}/pkgshipd +%attr(0755,root,root) %{_bindir}/pkgship + +%changelog +* Tue Oct 13 2020 ZhangTao 1.1.0-14 +- correct-the-parameter-transfer-method-and-change-the-status-recording-method. + +* Fri Sep 25 2020 Cheng Shaowei 1.1.0-13 +- Optimize-log-records-when-obtaining-issue-content + +* Fri Sep 25 2020 Zhang Tao - 1.1.0-12 +- In the selfbuild scenario, add the error message that the software package cannot be found + +* Fri Sep 25 2020 Zhang Tao - 1.1.0-11 +- Fix the problem of function parameters + +* Thu Sep 24 2020 Yiru Wang - 1.1.0-10 +- rm queue_maxsize param from package.ini and this parameter is not customizable + +* Tue Sep 21 2020 Shenmei Tu - 1.0-0-9 +- Solve the problem of data duplication, increase the maximum queue length judgment, +- and avoid occupying too much memory + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-8 +- Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, +- and reduce the judgment branch of the old code. + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-7 +- fix the error when executing query commands + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-6 +- When initializing logging, modify the incoming class object to an instance of the class, +- ensure the execution of internal functions,and read configuration file content + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-5 +- Fix the problem of continuous spaces in message information in log records + +* Thu Sep 17 2020 Shenmei Tu - 1.0-0-4 +- Modify the query logic of package information, reduce redundant queries and align dnf query results, +- extract multiplexing functions, add corresponding docString, and clear pylint + +* Fri Sep 11 2020 Yiru Wang - 1.1.0-3 +- #I1UCM8, #I1UC8G: Modify some config files' permission issue; +- #I1TIYQ: Add concurrent-log-handler module to fix log resource conflict issue +- #I1TML0: Fix the matching relationship between source_rpm and src_name + +* Tue Sep 1 2020 Zhengtang Gong - 1.1.0-2 +- Delete the packaged form of pyinstaller and change the execution + of the command in the form of a single file as the input + +* Sat Aug 29 2020 Yiru Wang - 1.1.0-1 +- Add package management features: + RPM packages statically displayed in the version repository + RPM packages used time displayed for current version in the version repository + Issue management of packages in a version-management repository + +* Fri Aug 21 2020 Chengqiang Bao < baochengqiang1@huawei.com > - 1.0.0-7 +- Fixed a problem with command line initialization of the Filepath parameter where relative paths are not supported and paths are too long + +* Wed Aug 12 2020 Zhang Tao - 1.0.0-6 +- Fix the test content to adapt to the new data structure, add BuildRequires for running %check + +* Mon Aug 10 2020 Zhengtang Gong - 1.0-5 +- Command line supports calling remote services + +* Wed Aug 5 2020 Yiru Wang - 1.0-4 +- change Requires rpm pakcages' name to latest one + +* Mon Jul 13 2020 Yiru Wang - 1.0-3 +- run test cases while building + +* Sat Jul 4 2020 Yiru Wang - 1.0-2 +- cheange requires python3.7 to python3,add check pyinstaller file. + +* Tue Jun 30 2020 Yiru Wang - 1.0-1 +- add pkgshipd file + +* Thu Jun 11 2020 Feng Hu - 1.0-0 +- add macro to build cli bin when rpm install + +* Sat Jun 6 2020 Feng Hu - 1.0-0 +- init package diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/2.6.4.tar.gz b/test/ac/acl/license/license_test_sample/rubygem-mail/2.6.4.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3c7c19f7c87bcf156c34a394d514abe16ba5e9fe Binary files /dev/null and b/test/ac/acl/license/license_test_sample/rubygem-mail/2.6.4.tar.gz differ diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-Fix-deprecated-warnings-in-Ruby-2.4.0.patch b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-Fix-deprecated-warnings-in-Ruby-2.4.0.patch new file mode 100644 index 0000000000000000000000000000000000000000..01d3556a6f7966d99b539d2eb65c25d75f8cfc22 --- /dev/null +++ b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-Fix-deprecated-warnings-in-Ruby-2.4.0.patch @@ -0,0 +1,59 @@ +From e8fde9cf1d77ee7e465c12e809501df8d27e8451 Mon Sep 17 00:00:00 2001 +From: Koichi ITO +Date: Sun, 4 Dec 2016 12:33:06 +0800 +Subject: [PATCH] Fix deprecated warnings in Ruby 2.4.0+ + +--- + lib/mail/attachments_list.rb | 2 +- + lib/mail/multibyte/chars.rb | 4 ++-- + lib/mail/network/retriever_methods/test_retriever.rb | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/lib/mail/attachments_list.rb b/lib/mail/attachments_list.rb +index bb34a85..14fe31c 100644 +--- a/lib/mail/attachments_list.rb ++++ b/lib/mail/attachments_list.rb +@@ -30,7 +30,7 @@ module Mail + # mail.attachments['test.png'].filename #=> 'test.png' + # mail.attachments[1].filename #=> 'test.jpg' + def [](index_value) +- if index_value.is_a?(Fixnum) ++ if index_value.is_a?(Integer) + self.fetch(index_value) + else + self.select { |a| a.filename == index_value }.first +diff --git a/lib/mail/multibyte/chars.rb b/lib/mail/multibyte/chars.rb +index bb39897..2e431ca 100644 +--- a/lib/mail/multibyte/chars.rb ++++ b/lib/mail/multibyte/chars.rb +@@ -269,12 +269,12 @@ module Mail #:nodoc: + @wrapped_string[*args] = replace_by + else + result = Unicode.u_unpack(@wrapped_string) +- if args[0].is_a?(Fixnum) ++ if args[0].is_a?(Integer) + raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length + min = args[0] + max = args[1].nil? ? min : (min + args[1] - 1) + range = Range.new(min, max) +- replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum) ++ replace_by = [replace_by].pack('U') if replace_by.is_a?(Integer) + elsif args.first.is_a?(Range) + raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length + range = args[0] +diff --git a/lib/mail/network/retriever_methods/test_retriever.rb b/lib/mail/network/retriever_methods/test_retriever.rb +index 9bb3e1a..dfbc909 100644 +--- a/lib/mail/network/retriever_methods/test_retriever.rb ++++ b/lib/mail/network/retriever_methods/test_retriever.rb +@@ -25,7 +25,7 @@ module Mail + emails_index.reverse! if options[:what] == :last + emails_index = case count = options[:count] + when :all then emails_index +- when Fixnum then emails_index[0, count] ++ when Integer then emails_index[0, count] + else + raise 'Invalid count option value: ' + count.inspect + end +-- +2.11.0 + diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-fix-new-warning-in-ruby-2.4.patch b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-fix-new-warning-in-ruby-2.4.patch new file mode 100644 index 0000000000000000000000000000000000000000..af455602f552ad3069f29a7d32e64feea9dc5b1d --- /dev/null +++ b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4-fix-new-warning-in-ruby-2.4.patch @@ -0,0 +1,38 @@ +From 48cb6db25b31eebe7bdd330d812c52d3c93aa328 Mon Sep 17 00:00:00 2001 +From: "yuuji.yaginuma" +Date: Tue, 13 Dec 2016 07:50:42 +0900 +Subject: [PATCH] fix new warning in ruby 2.4 + +This fixes the following warning. + +``` +/home/travis/build/mikel/mail/lib/mail/fields/common/address_container.rb:11: warning: parentheses after method name is interpreted as +/home/travis/build/mikel/mail/lib/mail/fields/common/address_container.rb:11: warning: an argument list, not a decomposed argument +``` + +Ref: https://github.com/ruby/ruby/commit/65e27c8b138d6959608658ffce2fa761842b8d24 +--- + lib/mail/fields/common/address_container.rb | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/lib/mail/fields/common/address_container.rb b/lib/mail/fields/common/address_container.rb +index f4a5aec..48c1286 100644 +--- a/lib/mail/fields/common/address_container.rb ++++ b/lib/mail/fields/common/address_container.rb +@@ -8,10 +8,10 @@ module Mail + super(list) + end + +- def << (address) ++ def <<(address) + @field << address + end + + end + +-end +\ No newline at end of file ++end +-- +2.11.0 + diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4.gem b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4.gem new file mode 100644 index 0000000000000000000000000000000000000000..4fa553a1f40c3cccb1614b4a9b980cb907d3b860 Binary files /dev/null and b/test/ac/acl/license/license_test_sample/rubygem-mail/mail-2.6.4.gem differ diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.spec b/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.spec new file mode 100644 index 0000000000000000000000000000000000000000..750602f02adb2acd78c5f8d40fb18b9b457d0575 --- /dev/null +++ b/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.spec @@ -0,0 +1,71 @@ +%global gem_name mail +Name: rubygem-%{gem_name} +Version: 2.6.4 +Release: 2 +Summary: Mail provides a nice Ruby DSL for making, sending and reading emails +License: MIT +URL: https://github.com/mikel/mail +Source0: https://rubygems.org/gems/%{gem_name}-%{version}.gem +Source1: https://github.com/mikel/mail/archive/%{version}.tar.gz +# Fix Ruby 2.4 compatibility. +# https://github.com/mikel/mail/commit/e8fde9cf1d77ee7e465c12e809501df8d27e8451 +Patch0: mail-2.6.4-Fix-deprecated-warnings-in-Ruby-2.4.0.patch +# https://github.com/mikel/mail/commit/48cb6db25b31eebe7bdd330d812c52d3c93aa328 +Patch1: mail-2.6.4-fix-new-warning-in-ruby-2.4.patch +BuildRequires: ruby(release) rubygems-devel ruby rubygem(mime-types) >= 1.16 rubygem(rspec) +BuildArch: noarch +%description +A really Ruby Mail handler. + +%package doc +Summary: Documentation for %{name} +Requires: %{name} = %{version}-%{release} +BuildArch: noarch +%description doc +Documentation for %{name}. + +%prep +%setup -q -c -T +ln -s %{_builddir}/%{gem_name}-%{version}/spec ../spec +%gem_install -n %{SOURCE0} +pushd .%{gem_instdir} +%patch0 -p1 +%patch1 -p1 +popd + +%build + +%install +mkdir -p %{buildroot}%{gem_dir} +cp -a .%{gem_dir}/* \ + %{buildroot}%{gem_dir}/ + +%check +pushd .%{gem_instdir} +tar xzvf %{SOURCE1} +rspec spec +popd + +%files +%dir %{gem_instdir} +%license %{gem_instdir}/MIT-LICENSE +%{gem_libdir} +%exclude %{gem_cache} +%{gem_spec} + +%files doc +%doc %{gem_docdir} +%doc %{gem_instdir}/CHANGELOG.rdoc +%doc %{gem_instdir}/CONTRIBUTING.md +%doc %{gem_instdir}/Dependencies.txt +%{gem_instdir}/Gemfile* +%doc %{gem_instdir}/README.md +%{gem_instdir}/Rakefile +%doc %{gem_instdir}/TODO.rdoc + +%changelog +* Tue Sep 8 2020 geyanan - 2.6.4-2 +- fix build fail + +* Wed Aug 19 2020 geyanan - 2.6.4-1 +- package init diff --git a/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.yaml b/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98f733e147e70f49f88c832934a60cef477b5fdf --- /dev/null +++ b/test/ac/acl/license/license_test_sample/rubygem-mail/rubygem-mail.yaml @@ -0,0 +1,4 @@ +version_control: github +src_repo: mikel/mail +tag_prefix: "" +seperator: "." diff --git a/test/ac/acl/license/license_test_sample/spec_fail/spec_fail.spec b/test/ac/acl/license/license_test_sample/spec_fail/spec_fail.spec new file mode 100644 index 0000000000000000000000000000000000000000..80b9b23d87dd793ead6056a013fdac2a522635a4 --- /dev/null +++ b/test/ac/acl/license/license_test_sample/spec_fail/spec_fail.spec @@ -0,0 +1,37 @@ +Name: spec_fail +Version: 1.1.0 +Release: 1 +Summary: test case for spec license error. +License: Mulan 2.0 and ADSL +URL: https://gitee.com/openeuler/openEuler-Advisor +Source0: https://gitee.com/openeuler/openEuler-Advisor/pkgship-%{version}.tar.gz + +%description +test case for spec license error. + +%prep +%autosetup + +%build +%py3_build + +%install +%py3_install + +%check + +%post + +%postun + + +%files +%doc README.md +%{python3_sitelib}/* +%attr(0755,root,root) %config %{_sysconfdir}/pkgship/* +%attr(0755,root,root) %{_bindir}/pkgshipd +%attr(0755,root,root) %{_bindir}/pkgship + +%changelog +* Mon Oct 19 2020 xxx - 1.0-0 +- init package diff --git a/test/ac/acl/license/license_test_sample/spec_src_diff/spec_src_diff b/test/ac/acl/license/license_test_sample/spec_src_diff/spec_src_diff new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/license_test_sample/spec_src_same/spec_src_same b/test/ac/acl/license/license_test_sample/spec_src_same/spec_src_same new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/license_test_sample/src_fail/src_fail b/test/ac/acl/license/license_test_sample/src_fail/src_fail new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/license_test_sample/src_success/src_success b/test/ac/acl/license/license_test_sample/src_success/src_success new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/license/test_check_license.py b/test/ac/acl/license/test_check_license.py new file mode 100644 index 0000000000000000000000000000000000000000..22431ef7cff5ab238c3b1046dcfd51c85dfdc99f --- /dev/null +++ b/test/ac/acl/license/test_check_license.py @@ -0,0 +1,133 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-10-16 +# Description: check spec file +# ********************************************************************************** +""" + +import unittest +import mock +import sys +import os +import types +import logging.config +import logging +import shutil + +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.acl.package_license.check_license import CheckLicense + +logging.getLogger('test_logger') + +class TestCheckPkgLicense(unittest.TestCase): + DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "license_test_sample") + + TEST_SAMPLE_DIR = { + "no_spec": "no_spec", + "spec_success": "pkgship", + "spec_fail": "spec_fail", + "no_src": "no_src", + "src_success": "rubygem-mail", + "src_fail": "pkgship", + "spec_src_same": "rubygem-mail", + "spec_src_diff": "pkgship" + } + + def bind_func(self, check): + def get_work_tar_dir(self): + return self._work_tar_dir + def load_license_config(self): + self._pkg_license.load_config() + def decompress(self): + self._gr.decompress_all() + check.get_work_tar_dir = types.MethodType(get_work_tar_dir, check, CheckLicense) + check.load_license_config = types.MethodType(load_license_config, check, CheckLicense) + check.decompress = types.MethodType(decompress, check, CheckLicense) + + def _test_check_license_in_spec(self, dir_key, predict): + os.chdir(os.path.join(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key])) + cl = CheckLicense(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key]) + self.bind_func(cl) + cl.load_license_config() + self.assertEqual(cl.check_license_in_spec(), predict) + + def test_check_license_in_spec_none(self): + self._test_check_license_in_spec("no_spec", WARNING) + + def test_check_license_in_spec_succeed(self): + self._test_check_license_in_spec("spec_success", SUCCESS) + + def test_check_license_in_spec_failed(self): + self._test_check_license_in_spec("spec_fail", WARNING) + + def _test_check_license_in_src(self, dir_key, predict): + os.chdir(os.path.join(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key])) + cl = CheckLicense(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key]) + self.bind_func(cl) + _ = not os.path.exists(cl.get_work_tar_dir()) and os.mkdir(cl.get_work_tar_dir()) + try: + cl.decompress() + cl.load_license_config() + self.assertEqual(cl.check_license_in_src(), predict) + finally: + shutil.rmtree(cl.get_work_tar_dir()) + + def test_check_license_none(self): + self._test_check_license_in_src("no_src", WARNING) + + def test_check_license_in_src_succeed(self): + self._test_check_license_in_src("src_success", SUCCESS) + + def test_check_license_in_src_failed(self): + self._test_check_license_in_src("src_fail", WARNING) + + def _test_check_license_same(self, dir_key, predict): + os.chdir(os.path.join(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key])) + cl = CheckLicense(self.DIR_PATH, + self.TEST_SAMPLE_DIR[dir_key]) + self.bind_func(cl) + _ = not os.path.exists(cl.get_work_tar_dir()) and os.mkdir(cl.get_work_tar_dir()) + try: + cl.decompress() + cl.load_license_config() + cl.check_license_in_spec() + cl.check_license_in_src() + self.assertEqual(cl.check_license_is_same(), predict) + finally: + shutil.rmtree(cl.get_work_tar_dir()) + + def test_check_license_same_succeed(self): + self._test_check_license_same("spec_src_same", SUCCESS) + + def test_cehck_license_same_failed(self): + self._test_check_license_same("spec_src_diff", WARNING) + + +if __name__ == "__main__": + work_dir = os.getcwd() + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../../src/conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + # Test Package License + suite = unittest.makeSuite(TestCheckPkgLicense) + unittest.TextTestRunner().run(suite) + os.chdir(work_dir) + shutil.rmtree("log") \ No newline at end of file diff --git a/test/ac/acl/package_yaml/__init__.py b/test/ac/acl/package_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5bdad435b94b1475a582e9f6602dd020179de1 --- /dev/null +++ b/test/ac/acl/package_yaml/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-21 +# Description: +# ***********************************************************************************/ +""" \ No newline at end of file diff --git a/test/ac/acl/package_yaml/fields_test_sample/missing.yaml b/test/ac/acl/package_yaml/fields_test_sample/missing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bc47b74eb3f6e60296bc16169369c7172f90c36 --- /dev/null +++ b/test/ac/acl/package_yaml/fields_test_sample/missing.yaml @@ -0,0 +1,2 @@ +src_repo: a/b +tag_prefix: NA \ No newline at end of file diff --git a/test/ac/acl/package_yaml/fields_test_sample/standard.yaml b/test/ac/acl/package_yaml/fields_test_sample/standard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10fe26eb145880d7fb785e919924d09145839248 --- /dev/null +++ b/test/ac/acl/package_yaml/fields_test_sample/standard.yaml @@ -0,0 +1,4 @@ +version_control: testvc +src_repo: src/repo +tag_prefix: "v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.spec b/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..9aebd3154f4fbaeae9044dd72c2a87ffeb5ce72d --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.spec @@ -0,0 +1,84 @@ +%global __requires_exclude ^perl\\(Autom4te:: +%global __provides_exclude ^perl\\(Autom4te:: + +Name: autoconf +Version: 2.69 +Release: 30 +Summary: An extensible package to automatically configure software source code packages +License: GPLv2+ and GPLv3+ and GFDL +URL: https://www.gnu.org/software/%{name}/ +Source0: http://ftp.gnu.org/gnu/%{name}/%{name}-%{version}.tar.xz +Source1: config.site +Source2: autoconf-el.el + +# four patches backport from upstream to solve test suite failure +Patch1: autoscan-port-to-perl-5.17.patch +Patch2: Port-tests-to-Bash-5.patch +Patch3: tests-avoid-spurious-test-failure-with-libtool-2.4.3.patch +#fix the failure of test 38 autotools and whitespace in file names +Patch4: Fix-test-suite-with-modern-Perl.patch + +Patch9000: skip-one-test-at-line-1616-of-autotest.patch + +BuildArch: noarch + +BuildRequires: m4 emacs perl perl-generators help2man +Requires: m4 emacs-filesystem +Requires(post): info +Requires(preun):info + +%package_help + +%description +Autoconf is an extensible package of M4 macros that produce shell scripts to automatically +configure software source code packages. These scripts can adapt the packages to many kinds +of UNIX-like systems without manual user intervention. Autoconf creates a configuration script +for a package from a template file that lists the operating system features that the package +can use, in the form of M4 macro calls. + +%prep +%autosetup -n %{name}-%{version} -p1 + +%build +export EMACS=%{_bindir}/emacs +%configure --with-lispdir=%{_emacs_sitelispdir}/autoconf +%make_build + +%check +make %{?_smp_mflags} check + +%install +%make_install +install -p -D %{SOURCE1} %{buildroot}%{_datadir} +install -p -D %{SOURCE2} %{buildroot}%{_emacs_sitestartdir}/autoconf-el.el + +%post help +/sbin/install-info %{_infodir}/autoconf.info %{_infodir}/dir || : + +%preun help +if [ "$1" = 0 ]; then + /sbin/install-info --delete %{_infodir}/autoconf.info %{_infodir}/dir || : +fi + +%files +%doc ChangeLog README THANKS +%license COPYING* AUTHORS doc/autoconf.info +%{_bindir}/* +%{_datadir}/autoconf/ +%{_datadir}/config.site +%{_datadir}/emacs/site-lisp/* +%exclude %{_infodir}/standards* + +%files help +%doc NEWS TODO +%{_infodir}/autoconf.info* +%{_mandir}/man1/* +%exclude %{_infodir}/dir + + +%changelog +* Sat Jan 4 2020 openEuler Buildteam - 2.69-30 +- Strengthen sources and patches + +* Fri Oct 11 2019 openEuler Buildteam - 2.69-29 +- Package Init diff --git a/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09b64c0d5f55ee466b0e9cfee1f94cb965cd8785 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/git_test/git_test.yaml @@ -0,0 +1,4 @@ +version_control: git +src_repo: https://git.savannah.gnu.org/git/autoconf.git +tag_prefix: "^v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.spec b/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..1da248dec3fff264c43608acd0bb8db6b77a4670 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.spec @@ -0,0 +1,263 @@ +%global _version 2.0.5 +%global _release 20200923.100811.git275398ce +%global is_systemd 1 + +Name: iSulad +Version: %{_version} +Release: %{_release} +Summary: Lightweight Container Runtime Daemon +License: Mulan PSL v2 +URL: https://gitee.com/openeuler/iSulad +Source: https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz +BuildRoot: {_tmppath}/iSulad-%{version} +ExclusiveArch: x86_64 aarch64 + +Patch6000: 0000-config-remove-unused-config.patch +Patch6001: 0001-fix-modify-quota-log-level-to-warning.patch +Patch6002: 0002-fix-memory-leak.patch +Patch6003: 0003-fix-security-opt-parsing-access-out-of-bounds.patch +Patch6004: 0004-fix-delete-rootfs-dir-when-rootfs-load-failed.patch +Patch6005: 0005-fix-code-review.patch +Patch6006: 0006-fix-pull-failure-caused-by-link-conflict.patch +Patch6007: 0007-image-clear-memory-if-failed.patch +Patch6008: 0008-fix-layer-remain-caused-by-hold-flag-not-clean.patch +Patch6009: 0009-fix-coredump-when-pull-image-with-lock-driver-image-.patch +Patch6010: 0010-fix-bad-formatting-placeholder-in-http-parse-module.patch +Patch6011: 0011-iSulad-fix-memory-leak.patch +Patch6012: 0012-fix-coredump-when-load-image-with-uid.patch + +%ifarch x86_64 aarch64 +Provides: libhttpclient.so()(64bit) +Provides: libisula.so()(64bit) +Provides: libisulad_img.so()(64bit) +%endif + +%if 0%{?is_systemd} +# Systemd 230 and up no longer have libsystemd-journal +BuildRequires: pkgconfig(systemd) +Requires: systemd-units +%else +Requires(post): chkconfig +Requires(preun): chkconfig +# This is for /sbin/service +Requires(preun): initscripts +%endif + +BuildRequires: cmake gcc-c++ lxc lxc-devel lcr-devel yajl-devel clibcni-devel +BuildRequires: grpc grpc-plugins grpc-devel protobuf-devel +BuildRequires: libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel +BuildRequires: http-parser-devel +BuildRequires: libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel +BuildRequires: systemd-devel git + +Requires: lcr lxc clibcni +Requires: grpc protobuf +Requires: libcurl +Requires: sqlite http-parser libseccomp +Requires: libcap libselinux libwebsockets libarchive device-mapper +Requires: systemd + +%description +This is a umbrella project for gRPC-services based Lightweight Container +Runtime Daemon, written by C. + +%prep +%autosetup -n %{name} -Sgit -p1 + +%build +mkdir -p build +cd build +%cmake -DDEBUG=ON -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr ../ +%make_build + +%install +rm -rf %{buildroot} +cd build +install -d $RPM_BUILD_ROOT/%{_libdir} +install -m 0644 ./src/libisula.so %{buildroot}/%{_libdir}/libisula.so +install -m 0644 ./src/utils/http/libhttpclient.so %{buildroot}/%{_libdir}/libhttpclient.so +install -m 0644 ./src/daemon/modules/image/libisulad_img.so %{buildroot}/%{_libdir}/libisulad_img.so +chmod +x %{buildroot}/%{_libdir}/libisula.so +chmod +x %{buildroot}/%{_libdir}/libhttpclient.so +chmod +x %{buildroot}/%{_libdir}/libisulad_img.so + +install -d $RPM_BUILD_ROOT/%{_libdir}/pkgconfig +install -m 0640 ./conf/isulad.pc %{buildroot}/%{_libdir}/pkgconfig/isulad.pc + +install -d $RPM_BUILD_ROOT/%{_bindir} +install -m 0755 ./src/isula %{buildroot}/%{_bindir}/isula +install -m 0755 ./src/isulad-shim %{buildroot}/%{_bindir}/isulad-shim +install -m 0755 ./src/isulad %{buildroot}/%{_bindir}/isulad +chrpath -d ./src/isula +chrpath -d ./src/isulad-shim +chrpath -d ./src/isulad + +install -d $RPM_BUILD_ROOT/%{_includedir}/isulad +install -m 0644 ../src/client/libisula.h %{buildroot}/%{_includedir}/isulad/libisula.h +install -m 0644 ../src/client/connect/isula_connect.h %{buildroot}/%{_includedir}/isulad/isula_connect.h +install -m 0644 ../src/utils/cutils/utils_timestamp.h %{buildroot}/%{_includedir}/isulad/utils_timestamp.h +install -m 0644 ../src/utils/cutils/error.h %{buildroot}/%{_includedir}/isulad/error.h +install -m 0644 ../src/daemon/modules/runtime/engines/engine.h %{buildroot}/%{_includedir}/isulad/engine.h +install -m 0644 ../src/daemon/modules/api/image_api.h %{buildroot}/%{_includedir}/isulad/image_api.h + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/isulad +install -m 0640 ../src/contrib/config/daemon.json %{buildroot}/%{_sysconfdir}/isulad/daemon.json +install -m 0640 ../src/contrib/config/seccomp_default.json %{buildroot}/%{_sysconfdir}/isulad/seccomp_default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad +install -m 0640 ../src/contrib/config/config.json %{buildroot}/%{_sysconfdir}/default/isulad/config.json +install -m 0640 ../src/contrib/config/systemcontainer_config.json %{buildroot}/%{_sysconfdir}/default/isulad/systemcontainer_config.json +install -m 0550 ../src/contrib/sysmonitor/isulad-check.sh %{buildroot}/%{_sysconfdir}/default/isulad/isulad-check.sh + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/sysmonitor/process +cp ../src/contrib/sysmonitor/isulad-monit $RPM_BUILD_ROOT/etc/sysmonitor/process + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/default/isulad/hooks +install -m 0640 ../src/contrib/config/hooks/default.json %{buildroot}/%{_sysconfdir}/default/isulad/hooks/default.json + +install -d $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig +install -p -m 0640 ../src/contrib/config/iSulad.sysconfig $RPM_BUILD_ROOT/%{_sysconfdir}/sysconfig/iSulad + +%if 0%{?is_systemd} +install -d $RPM_BUILD_ROOT/%{_unitdir} +install -p -m 0640 ../src/contrib/init/isulad.service $RPM_BUILD_ROOT/%{_unitdir}/isulad.service +%else +install -d $RPM_BUILD_ROOT/%{_initddir} +install -p -m 0640 ../src/contrib/init/isulad.init $RPM_BUILD_ROOT/%{_initddir}/isulad.init +%endif + +%clean +rm -rf %{buildroot} + +%pre +# support update from lcrd to isulad, will remove in next version +if [ "$1" = "2" ]; then +%if 0%{?is_systemd} +systemctl stop lcrd +systemctl disable lcrd +if [ -e %{_sysconfdir}/isulad/daemon.json ];then + sed -i 's#/etc/default/lcrd/hooks#/etc/default/isulad/hooks#g' %{_sysconfdir}/isulad/daemon.json +fi +%else +/sbin/chkconfig --del lcrd +%endif +fi + +%post +if ! getent group isulad > /dev/null; then + groupadd --system isulad +fi + +if [ "$1" = "1" ]; then +%if 0%{?is_systemd} +systemctl enable isulad +systemctl start isulad +%else +/sbin/chkconfig --add isulad +%endif +elif [ "$1" = "2" ]; then +%if 0%{?is_systemd} +# support update from lcrd to isulad, will remove in next version +if [ -e %{_unitdir}/lcrd.service.rpmsave ]; then + mv %{_unitdir}/lcrd.service.rpmsave %{_unitdir}/isulad.service + sed -i 's/lcrd/isulad/g' %{_unitdir}/isulad.service +fi +systemctl status isulad | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + systemctl restart isulad +else + systemctl start isulad +fi +%else +/sbin/service isulad status | grep 'Active:' | grep 'running' +if [ $? -eq 0 ]; then + /sbin/service isulad restart +fi +%endif +fi + +if ! getent group isulad > /dev/null; then + groupadd --system isulad +fi + +%preun +%if 0%{?is_systemd} +%systemd_preun isulad +%else +if [ $1 -eq 0 ] ; then + /sbin/service isulad stop >/dev/null 2>&1 + /sbin/chkconfig --del isulad +fi +%endif + +%postun +%if 0%{?is_systemd} +%systemd_postun_with_restart isulad +%else +if [ "$1" -ge "1" ] ; then + /sbin/service isulad condrestart >/dev/null 2>&1 || : +fi +%endif + +%files +%attr(0600,root,root) %{_sysconfdir}/sysmonitor/process/isulad-monit +%attr(0550,root,root) %{_sysconfdir}/default/isulad/isulad-check.sh +%defattr(0640,root,root,0750) +%{_sysconfdir}/isulad +%{_sysconfdir}/isulad/* +%{_sysconfdir}/default/* +%defattr(-,root,root,-) +%if 0%{?is_systemd} +%{_unitdir}/isulad.service +%attr(0640,root,root) %{_unitdir}/isulad.service +%else +%{_initddir}/isulad.init +%attr(0640,root,root) %{_initddir}/isulad.init +%endif +%{_includedir}/isulad/* +%attr(0755,root,root) %{_libdir}/pkgconfig +%attr(0640,root,root) %{_libdir}/pkgconfig/isulad.pc +%defattr(0550,root,root,0750) +%{_bindir}/* +%{_libdir}/* +%attr(0640,root,root) %{_sysconfdir}/sysconfig/iSulad +%attr(0640,root,root) %{_sysconfdir}/isulad/daemon.json + +%config(noreplace,missingok) %{_sysconfdir}/sysconfig/iSulad +%config(noreplace,missingok) %{_sysconfdir}/isulad/daemon.json +%if 0%{?is_systemd} +%config(noreplace,missingok) %{_unitdir}/isulad.service +%else +%config(noreplace,missingok) %{_initddir}/isulad.init +%endif + +%changelog ++* Fri Sep 23 2020 - 2.0.5-20200923.100811.git275398ce ++- Type:bugfix ++- ID:NA ++- SUG:NA ++- DESC: fix some memory bugs + ++* Fri Sep 18 2020 - 2.0.5-20200918.112827.git9aea9b75 ++- Type:bugfix ++- ID:NA ++- SUG:NA ++- DESC: modify log level to warn + ++* Mon Sep 14 2020 - 2.0.5-20200914.172527.gitae86920a ++- Type:bugfix ++- ID:NA ++- SUG:NA ++- DESC: remove unused config + +* Tue Sep 10 2020 - 2.0.5-20200910.144345.git71b1055b +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: add chrpath + +* Fri Sep 04 2020 zhangxiaoyu - 2.0.5-20200904.114315.gitff1761c3 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC: upgrade from v2.0.3 to v2.0.5 diff --git a/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2484d31d5572364b494dab109bf5223977aa591 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gitee_test/gitee_test.yaml @@ -0,0 +1,4 @@ +version_control: gitee +src_repo: openEuler/iSulad +tag_prefix: "v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.spec b/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..e02588d51731ae4608c690a498fbcaa55a5b121a --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.spec @@ -0,0 +1,107 @@ +%global _name pyinotify +%global _description Pyinotify is a Python module for monitoring filesystems changes. \ +Pyinotify relies on a Linux Kernel feature (merged in kernel 2.6.13) called inotify. \ +inotify is an event-driven notifier, its notifications are exported from kernel space \ +to user space through three system calls. pyinotify binds these system calls and provides \ +an implementation on top of them offering a generic and abstract way to manipulate those \ +functionalities. + +Name: python-inotify +Version: 0.9.6 +Release: 16 +Summary: A Python module for monitoring filesystems changes +License: MIT +URL: https://github.com/seb-m/pyinotify +Source0: https://github.com/seb-m/pyinotify/archive/%{version}.tar.gz#/%{_name}-%{version}.tar.gz + +Patch0: pyinotify-0.9.6-epoint.patch +BuildArch: noarch +BuildRequires: gmp-devel + +%description +%{_description} + +%package -n python2-inotify +Summary: python2 edition of %{name} + +BuildRequires: python2-devel + +Provides: python2-inotify-examples + +Obsoletes: python2-inotify-examples + +%{?python_provide:%python_provide python2-inotify} + +%description -n python2-inotify +This package is the python2 edition of %{name}. + +%package -n python3-inotify +Summary: python3 edition of %{name} + +BuildRequires: python3-devel + +%{?python_provide:%python_provide python3-inotify} + +%description -n python3-inotify +This package is the python3 edition of %{name}. + +%package_help + +%prep +%autosetup -n %{_name}-%{version} -p1 +sed -i '1c#! %{__python2}' python2/pyinotify.py +sed -i '1c#! %{__python3}' python3/pyinotify.py +sed -i '1c#! %{__python2}' python2/examples/*py +sed -i '1c#! %{__python3}' python3/examples/*py +cp -a . $RPM_BUILD_DIR/python3-%{_name}-%{version} + +%build +%py2_build +cd $RPM_BUILD_DIR/python3-%{_name}-%{version} +%py3_build + +%install +cd $RPM_BUILD_DIR/python3-%{_name}-%{version} +%py3_install +mv %{buildroot}%{_bindir}/%{_name} %{buildroot}%{_bindir}/python3-%{_name} +cd - +%py2_install + +%files -n python2-inotify +%defattr(-,root,root) +%doc ACKS +%license COPYING +%doc python2/examples/* +%{_bindir}/%{_name} +%{python2_sitelib}/*.py* +%{python2_sitelib}/%{_name}*info/ + +%files -n python3-inotify +%defattr(-,root,root) +%doc ACKS +%license COPYING +%doc python3/examples/* +%{_bindir}/*3-%{_name} +%{python3_sitelib}/*.py* +%{python3_sitelib}/%{_name}*info/ +%{python3_sitelib}/__pycache__/*.pyc + +%files help +%defattr(-,root,root) +%doc README.md PKG-INFO + +%changelog +* Mon Dec 23 2019 openEuler Buildteam - 0.9.6-16 +- Type:NA +- ID:NA +- SUG:NA +- DESC:delete unneeded comments + +* Fri Sep 27 2019 shenyangyang - 0.9.6-15 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:move license file + +* Thu Sep 12 2019 openEuler Buildteam - 0.9.6-14 +- Package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f629d4e13c7f7b0510bee67f35e8a084f9db332 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/github_test/github_test.yaml @@ -0,0 +1,4 @@ +version_control: github +src_repo: seb-m/pyinotify +tag_prefix: "^" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.spec b/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..40078cbda92824ab7d5ca57505930346ef3b7648 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.spec @@ -0,0 +1,58 @@ +Name: gnome-dictionary +Version: 3.26.1 +Release: 4 +Summary: A dictionary application for GNOME +License: GPLv3+ and LGPLv2+ and GFDL +URL: https://wiki.gnome.org/Apps/Dictionary +Source0: https://download.gnome.org/sources/%{name}/3.26/%{name}-%{version}.tar.xz + +BuildRequires: desktop-file-utils docbook-style-xsl gettext itstool libappstream-glib libxslt +BuildRequires: meson pkgconfig(gobject-introspection-1.0) pkgconfig(gtk+-3.0) +Obsoletes: gnome-utils <= 1:3.3 gnome-utils-libs <= 1:3.3 gnome-utils-devel <= 1:3.3 +Obsoletes: gnome-dictionary-devel < 3.26.0 gnome-dictionary-libs < 3.26.0 + +%description +GNOME Dictionary is a simple, clean, elegant application to look up words in +online dictionaries using the DICT protocol. + +%package help +Summary: Help package for gnome-dictionary + +%description help +This package contains some man help files for gnome-dictionary. + +%prep +%autosetup -n %{name}-%{version} -p1 + +%build +%meson +%meson_build + +%install +%meson_install +%find_lang %{name} --with-gnome + +%check +appstream-util validate-relax --nonet %{buildroot}/%{_datadir}/appdata/*.appdata.xml +desktop-file-validate %{buildroot}/%{_datadir}/applications/*.desktop + +%postun +if [ $1 -eq 0 ]; then + glib-compile-schemas %{_datadir}/glib-2.0/schemas &>/dev/null || : +fi + +%posttrans +glib-compile-schemas %{_datadir}/glib-2.0/schemas &>/dev/null || : + +%files -f %{name}.lang +%doc COPYING* NEWS README.md +%{_bindir}/gnome-dictionary +%{_datadir}/* +%exclude %{_datadir}/man* + +%files help +%{_mandir}/man1/gnome-dictionary.1* + +%changelog +* Wed Dec 11 2019 lingsheng - 3.26.1-4 +- Package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70537d024759da09f4eed7fc480ce859d4cf7c55 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.yaml @@ -0,0 +1,4 @@ +version_control: gitlab.gnome +src_repo: gnome-dictionary +tag_prefix: "^v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.spec b/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..80f2f0c41f29a5ec346725d521021e85cdf5dab2 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.spec @@ -0,0 +1,60 @@ +%bcond_with nls + +Name: help2man +Summary: Create simple man pages from --help output +Version: 1.47.11 +Release: 0 +License: GPLv3+ +URL: http://www.gnu.org/software/help2man +Source: ftp://ftp.gnu.org/gnu/help2man/help2man-%{version}.tar.xz + +%{!?with_nls:BuildArch: noarch} +BuildRequires: gcc perl-generators perl(Getopt::Long) perl(POSIX) perl(Text::ParseWords) perl(Text::Tabs) perl(strict) +%{?with_nls:BuildRequires: perl(Locale::gettext) /usr/bin/msgfmt} +%{?with_nls:BuildRequires: perl(Encode)} +%{?with_nls:BuildRequires: perl(I18N::Langinfo)} +Requires(post): /sbin/install-info +Requires(preun): /sbin/install-info + +%description +help2man is a tool for automatically generating simple manual pages from program output. + +%package_help + +%prep +%autosetup -n %{name}-%{version} + +%build +%configure --%{!?with_nls:disable}%{?with_nls:enable}-nls --libdir=%{_libdir}/help2man +%{make_build} + +%install +make install_l10n DESTDIR=$RPM_BUILD_ROOT +make install DESTDIR=$RPM_BUILD_ROOT +%find_lang %name --with-man + +%post +%install_info %{_infodir}/help2man.info + +%preun +if [ $1 -eq 0 ]; then + %install_info_rm %{_infodir}/help2man.info +fi + +%files -f %name.lang +%defattr(-,root,root) +%doc README NEWS THANKS +%license COPYING +%{_bindir}/help2man +%{_infodir}/* +%if %{with nls} + %{_libdir}/help2man +%endif + +%files help +%defattr(-,root,root) +%{_mandir}/man1/* + +%changelog +* Thu Nov 07 2019 openEuler Buildtam - 1.47.11-0 +- Package Init diff --git a/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdfc5e14ca8b6cab0344cc8b4ac42c44d74f715c --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/gnu_ftp_test/gnu_ftp_test.yaml @@ -0,0 +1,4 @@ +version_control: gnu-ftp +src_repo: help2man +tag_prefix: help2man-(.*).tar.xz(.sig)? +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.spec b/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..5158fb535f18a2e48a7ed7aea6a856e268f2d1b2 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.spec @@ -0,0 +1,381 @@ +%global _hardened_build 1 +%global nginx_user nginx + +%undefine _strict_symbol_defs_build + +%bcond_with geoip + +%global with_gperftools 1 + +%global with_mailcap_mimetypes 0 + +%global with_aio 1 + +Name: nginx +Epoch: 1 +Version: 1.18.0 +Release: 2 +Summary: A HTTP server, reverse proxy and mail proxy server +License: BSD +URL: http://nginx.org/ + +Source0: https://nginx.org/download/nginx-%{version}.tar.gz +Source10: nginx.service +Source11: nginx.logrotate +Source12: nginx.conf +Source13: nginx-upgrade +Source100: index.html +Source102: nginx-logo.png +Source103: 404.html +Source104: 50x.html +Source200: README.dynamic +Source210: UPGRADE-NOTES-1.6-to-1.10 + +Patch0: nginx-auto-cc-gcc.patch +Patch2: nginx-1.12.1-logs-perm.patch +BuildRequires: gcc openssl-devel pcre-devel zlib-devel systemd gperftools-devel +Requires: nginx-filesystem = %{epoch}:%{version}-%{release} openssl pcre +Requires: nginx-all-modules = %{epoch}:%{version}-%{release} +%if 0%{?with_mailcap_mimetypes} +Requires: nginx-mimetypes +%endif +Requires(pre): nginx-filesystem +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +Provides: webserver +Recommends: logrotate + +%description +NGINX is a free, open-source, high-performance HTTP server and reverse proxy, +as well as an IMAP/POP3 proxy server. + +%package all-modules +Summary: Nginx modules +BuildArch: noarch + +%if %{with geoip} +Requires: nginx-mod-http-geoip = %{epoch}:%{version}-%{release} +%endif +Requires: nginx-mod-http-image-filter = %{epoch}:%{version}-%{release} +Requires: nginx-mod-http-perl = %{epoch}:%{version}-%{release} +Requires: nginx-mod-http-xslt-filter = %{epoch}:%{version}-%{release} +Requires: nginx-mod-mail = %{epoch}:%{version}-%{release} +Requires: nginx-mod-stream = %{epoch}:%{version}-%{release} + +%description all-modules +NGINX is a free, open-source, high-performance HTTP server and reverse proxy, +as well as an IMAP/POP3 proxy server. +This package is a meta package that installs all available Nginx modules. + +%package filesystem +Summary: Filesystem for the Nginx server +BuildArch: noarch +Requires(pre): shadow-utils + +%description filesystem +NGINX is a free, open-source, high-performance HTTP server and reverse proxy, +as well as an IMAP/POP3 proxy server. +The package contains the basic directory layout for the Nginx server. + +%if %{with geoip} +%package mod-http-geoip +Summary: HTTP geoip module for nginx +BuildRequires: GeoIP-devel +Requires: nginx GeoIP + +%description mod-http-geoip +The package is the Nginx HTTP geoip module. +%endif + +%package mod-http-image-filter +Summary: HTTP image filter module for nginx +BuildRequires: gd-devel +Requires: nginx gd + +%description mod-http-image-filter +Nginx HTTP image filter module. + +%package mod-http-perl +Summary: HTTP perl module for nginx +BuildRequires: perl-devel perl(ExtUtils::Embed) +Requires: nginx perl(constant) +Requires: perl(:MODULE_COMPAT_%(eval "`%{__perl} -V:version`"; echo $version)) + +%description mod-http-perl +Nginx HTTP perl module. + +%package mod-http-xslt-filter +Summary: XSLT module for nginx +BuildRequires: libxslt-devel +Requires: nginx + +%description mod-http-xslt-filter +Nginx XSLT module. + +%package mod-mail +Summary: mail modules for nginx +Requires: nginx + +%description mod-mail +Nginx mail modules + +%package mod-stream +Summary: stream modules for nginx +Requires: nginx + +%description mod-stream +Nginx stream modules. + +%package_help + +%prep +%autosetup -n %{name}-%{version} -p1 +cp %{SOURCE200} %{SOURCE210} %{SOURCE10} %{SOURCE12} . + +%build +export DESTDIR=%{buildroot} +nginx_ldopts="$RPM_LD_FLAGS -Wl,-E" +if ! ./configure \ + --prefix=%{_datadir}/nginx --sbin-path=%{_sbindir}/nginx --modules-path=%{_libdir}/nginx/modules \ + --conf-path=%{_sysconfdir}/nginx/nginx.conf --error-log-path=%{_localstatedir}/log/nginx/error.log \ + --http-log-path=%{_localstatedir}/log/nginx/access.log \ + --http-client-body-temp-path=%{_localstatedir}/lib/nginx/tmp/client_body \ + --http-fastcgi-temp-path=%{_localstatedir}/lib/nginx/tmp/fastcgi \ + --http-proxy-temp-path=%{_localstatedir}/lib/nginx/tmp/proxy \ + --http-scgi-temp-path=%{_localstatedir}/lib/nginx/tmp/scgi \ + --http-uwsgi-temp-path=%{_localstatedir}/lib/nginx/tmp/uwsgi \ + --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx \ + --user=%{nginx_user} --group=%{nginx_user} \ +%if 0%{?with_aio} + --with-file-aio \ +%endif + --with-ipv6 --with-http_ssl_module --with-http_v2_module --with-http_realip_module \ + --with-http_addition_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic \ +%if %{with geoip} + --with-http_geoip_module=dynamic \ +%endif + --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module \ + --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module \ + --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module \ + --with-http_perl_module=dynamic --with-http_auth_request_module \ + --with-mail=dynamic --with-mail_ssl_module --with-pcre --with-pcre-jit --with-stream=dynamic \ + --with-stream_ssl_module --with-google_perftools_module --with-debug \ + --with-cc-opt="%{optflags} $(pcre-config --cflags)" --with-ld-opt="$nginx_ldopts"; then + : configure failed + cat objs/autoconf.err + exit 1 +fi + +%make_build + + +%install +%make_install INSTALLDIRS=vendor + +find %{buildroot} -type f -empty -exec rm -f '{}' \; +find %{buildroot} -type f -name .packlist -exec rm -f '{}' \; +find %{buildroot} -type f -name perllocal.pod -exec rm -f '{}' \; +find %{buildroot} -type f -iname '*.so' -exec chmod 0755 '{}' \; + +pushd %{buildroot} +install -p -D -m 0644 %{_builddir}/nginx-%{version}/nginx.service .%{_unitdir}/nginx.service +install -p -D -m 0644 %{SOURCE11} .%{_sysconfdir}/logrotate.d/nginx +install -p -d -m 0755 .%{_sysconfdir}/systemd/system/nginx.service.d +install -p -d -m 0755 .%{_unitdir}/nginx.service.d +install -p -d -m 0755 .%{_sysconfdir}/nginx/conf.d +install -p -d -m 0755 .%{_sysconfdir}/nginx/default.d +install -p -d -m 0700 .%{_localstatedir}/lib/nginx +install -p -d -m 0700 .%{_localstatedir}/lib/nginx/tmp +install -p -d -m 0700 .%{_localstatedir}/log/nginx +install -p -d -m 0755 .%{_datadir}/nginx/html +install -p -d -m 0755 .%{_datadir}/nginx/modules +install -p -d -m 0755 .%{_libdir}/nginx/modules +install -p -m 0644 %{_builddir}/nginx-%{version}/nginx.conf .%{_sysconfdir}/nginx +install -p -m 0644 %{SOURCE100} .%{_datadir}/nginx/html +install -p -m 0644 %{SOURCE102} .%{_datadir}/nginx/html +install -p -m 0644 %{SOURCE103} %{SOURCE104} .%{_datadir}/nginx/html + +%if 0%{?with_mailcap_mimetypes} +rm -f .%{_sysconfdir}/nginx/mime.types +%endif + +install -p -D -m 0644 %{_builddir}/nginx-%{version}/man/nginx.8 .%{_mandir}/man8/nginx.8 +install -p -D -m 0755 %{SOURCE13} .%{_bindir}/nginx-upgrade +popd + +for i in ftdetect indent syntax; do + install -p -D -m644 contrib/vim/${i}/nginx.vim %{buildroot}%{_datadir}/vim/vimfiles/${i}/nginx.vim +done + +%if %{with geoip} +echo 'load_module "%{_libdir}/nginx/modules/ngx_http_geoip_module.so";' \ + > %{buildroot}%{_datadir}/nginx/modules/mod-http-geoip.conf +%endif + +pushd %{buildroot} +echo 'load_module "%{_libdir}/nginx/modules/ngx_http_image_filter_module.so";' \ + > .%{_datadir}/nginx/modules/mod-http-image-filter.conf +echo 'load_module "%{_libdir}/nginx/modules/ngx_http_perl_module.so";' \ + > .%{_datadir}/nginx/modules/mod-http-perl.conf +echo 'load_module "%{_libdir}/nginx/modules/ngx_http_xslt_filter_module.so";' \ + > .%{_datadir}/nginx/modules/mod-http-xslt-filter.conf +echo 'load_module "%{_libdir}/nginx/modules/ngx_mail_module.so";' \ + > .%{_datadir}/nginx/modules/mod-mail.conf +echo 'load_module "%{_libdir}/nginx/modules/ngx_stream_module.so";' \ + > .%{_datadir}/nginx/modules/mod-stream.conf +popd + +%pre filesystem +getent group %{nginx_user} > /dev/null || groupadd -r %{nginx_user} +getent passwd %{nginx_user} > /dev/null || useradd -r -d %{_localstatedir}/lib/nginx -g %{nginx_user} \ + -s /sbin/nologin -c "Nginx web server" %{nginx_user} +exit 0 + +%post +%systemd_post nginx.service + +%if %{with geoip} +%post mod-http-geoip +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi +%endif + +%post mod-http-image-filter +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi + +%post mod-http-perl +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi + +%post mod-http-xslt-filter +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi + +%post mod-mail +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi + +%post mod-stream +if [ $1 -eq 1 ]; then + systemctl reload nginx.service >/dev/null 2>&1 || : +fi + +%preun +%systemd_preun nginx.service + +%postun +%systemd_postun nginx.service +if [ $1 -ge 1 ]; then + /usr/bin/nginx-upgrade >/dev/null 2>&1 || : +fi + +%files +%defattr(-,root,root) +%license LICENSE +%config(noreplace) %{_sysconfdir}/nginx/* +%config(noreplace) %{_sysconfdir}/logrotate.d/nginx +%exclude %{_sysconfdir}/nginx/conf.d +%exclude %{_sysconfdir}/nginx/default.d +%if 0%{?with_mailcap_mimetypes} +%exclude %{_sysconfdir}/nginx/mime.types +%endif +%{_bindir}/nginx-upgrade +%{_sbindir}/nginx +%dir %{_libdir}/nginx/modules +%attr(770,%{nginx_user},root) %dir %{_localstatedir}/lib/nginx +%attr(770,%{nginx_user},root) %dir %{_localstatedir}/lib/nginx/tmp +%{_unitdir}/nginx.service +%{_datadir}/nginx/html/* +%{_datadir}/vim/vimfiles/ftdetect/nginx.vim +%{_datadir}/vim/vimfiles/syntax/nginx.vim +%{_datadir}/vim/vimfiles/indent/nginx.vim +%attr(770,%{nginx_user},root) %dir %{_localstatedir}/log/nginx + +%files all-modules + +%files filesystem +%dir %{_sysconfdir}/nginx +%dir %{_sysconfdir}/nginx/{conf.d,default.d} +%dir %{_sysconfdir}/systemd/system/nginx.service.d +%dir %{_unitdir}/nginx.service.d +%dir %{_datadir}/nginx +%dir %{_datadir}/nginx/html + +%if %{with geoip} +%files mod-http-geoip +%{_libdir}/nginx/modules/ngx_http_geoip_module.so +%{_datadir}/nginx/modules/mod-http-geoip.conf +%endif + +%files mod-http-image-filter +%{_libdir}/nginx/modules/ngx_http_image_filter_module.so +%{_datadir}/nginx/modules/mod-http-image-filter.conf + +%files mod-http-perl +%{_libdir}/nginx/modules/ngx_http_perl_module.so +%{_datadir}/nginx/modules/mod-http-perl.conf +%dir %{perl_vendorarch}/auto/nginx +%{perl_vendorarch}/nginx.pm +%{perl_vendorarch}/auto/nginx/nginx.so + +%files mod-http-xslt-filter +%{_libdir}/nginx/modules/ngx_http_xslt_filter_module.so +%{_datadir}/nginx/modules/mod-http-xslt-filter.conf + +%files mod-mail +%{_libdir}/nginx/modules/ngx_mail_module.so +%{_datadir}/nginx/modules/mod-mail.conf + +%files mod-stream +%{_libdir}/nginx/modules/ngx_stream_module.so +%{_datadir}/nginx/modules/mod-stream.conf + +%files help +%defattr(-,root,root) +%doc CHANGES README README.dynamic +%{_mandir}/man3/nginx.3pm* +%{_mandir}/man8/nginx.8* + +%changelog +* Thu Sep 3 2020 yanan li - 1:1.18.0-2 +- add mime.types file to nginx packages + +* Thu Jun 4 2020 huanghaitao - 1:1.18.0-1 +- Change source to latest update + +* Fri May 22 2020 wutao - 1:1.16.1-4 +- change and delete html + +* Mon May 11 2020 wutao - 1:1.16.1-3 +- modify patch and html + +* Wed Mar 18 2020 yuxiangyang - 1:1.16.1-2 +- delete http_stub_status_module.This configuration creates a simple + web page with basic status data,but it will affect cpu scale-out because + it use atomic cas. + +* Mon Mar 16 2020 likexin - 1:1.16.1-1 +- update to 1.16.1 + +* Mon Mar 16 2020 openEuler Buildteam - 1:1.12.1-17 +- Type:bugfix +- ID:NA +- SUG:restart +- DESC: fix CVE-2019-20372 + +* Sat Dec 28 2019 openEuler Buildteam - 1:1.12.1-16 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC: add the with_mailcap_mimetypes + +* Wed Dec 4 2019 openEuler Buildteam - 1:1.12.1-15 +- Package init + diff --git a/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b313a57db912e5e807a88ccb748a09d029c16c3 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/hg_test/hg_test.yaml @@ -0,0 +1,4 @@ +version_control: hg +src_repo: https://hg.nginx.org/nginx +tag_prefix: release- +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.spec b/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..2c119db61ad4230d6c4a6d33633e87d6ee7a897a --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.spec @@ -0,0 +1,54 @@ +Name: perl-HTML-Tagset +Version: 3.20 +Release: 37 +Summary: HTML::Tagset - data tables useful in parsing HTML +License: GPL+ or Artistic +URL: https://metacpan.org/release/HTML-Tagset +Source0: https://cpan.metacpan.org/authors/id/P/PE/PETDANCE/HTML-Tagset-%{version}.tar.gz + +BuildArch: noarch + +BuildRequires: perl-generators perl-interpreter perl(:VERSION) >= 5.4 +BuildRequires: perl(ExtUtils::MakeMaker) >= 6.76 perl(strict) perl(vars) +BuildRequires: perl(Test) perl(Test::More) +BuildRequires: perl(Test::Pod) >= 1.14 + +Requires: perl(:MODULE_COMPAT_%(eval "`perl -V:version`"; echo $version)) + +%description +This module contains data tables useful in dealing with HTML. +It provides no functions or methods. + +%package help +Summary: Documentation for perl-HTML-Tagset + +%description help +Documentation for perl-HTML-Tagset. + +%prep +%autosetup -n HTML-Tagset-%{version} -p1 + +%build +perl Makefile.PL INSTALLDIRS=vendor NO_PACKLIST=1 +%make_build + +%install +make pure_install DESTDIR=%{buildroot} +%{_fixperms} %{buildroot} + +%check +make test + +%files +%{perl_vendorlib}/HTML/ +%exclude %{_libdir}/perl5/perllocal.pod + +%files help +%doc Changes README +%{_mandir}/man3/ + + +%changelog +* Tue Oct 22 2019 Zaiwang Li - 3.20-37 +- Init Package. + diff --git a/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6720c745b66f6bb9a40d4977c6258ac144c418a1 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/metacpan_test/metacpan_test.yaml @@ -0,0 +1,4 @@ +version_control: metacpan +src_repo: HTML-Tagset +tag_prefix: "v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/na_test/na.spec b/test/ac/acl/package_yaml/repo_test_sample/na_test/na.spec new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/ac/acl/package_yaml/repo_test_sample/na_test/na.yaml b/test/ac/acl/package_yaml/repo_test_sample/na_test/na.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa0b9febce760582cbb15012156d1bd5d22f127a --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/na_test/na.yaml @@ -0,0 +1,4 @@ +version_control: NA +src_repo: NA +tag_prefix: NA +separator: NA \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.spec b/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..f73c2a10c9ca82f8acd3c1215deed72d5496e22f --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.spec @@ -0,0 +1,110 @@ +Name: python-idna +Version: 2.10 +Release: 1 +Summary: Internationalized Domain Names in Applications (IDNA) +License: BSD and Python and Unicode +URL: https://github.com/kjd/idna +Source0: https://files.pythonhosted.org/packages/source/i/idna/idna-%{version}.tar.gz + +BuildArch: noarch +%if 0%{?with_python2} +BuildRequires: python2-devel python2-setuptools +%endif +%if 0%{?with_python3} +BuildRequires: python3-devel python3-setuptools +%endif + +%description +A library to support the Internationalised Domain Names in +Applications (IDNA) protocol as specified in RFC 5891 +http://tools.ietf.org/html/rfc5891. This version of the protocol +is often referred to as “IDNA2008” and can produce different +results from the earlier standard from 2003. + +The library is also intended to act as a suitable drop-in replacement +for the “encodings.idna” module that comes with the Python standard +library but currently only supports the older 2003 specification. + +%if 0%{?with_python2} +%package -n python2-idna +Summary: Python2 package for python-idna +Provides: python-idna = %{version}-%{release} +Obsoletes: python-idna < %{version}-%{release} + +%description -n python2-idna +Python2 package for python-idna +%endif + +%if 0%{?with_python3} +%package -n python3-idna +Summary: Python3 package for python-idna +%{?python_provide: %python_provide python3-idna} + +%description -n python3-idna +Python3 package for python-idna +%endif + +%prep +%autosetup -n idna-%{version} -p1 + +%build +%if 0%{?with_python2} +%py2_build +%endif + +%if 0%{?with_python3} +%py3_build +%endif + +%install +%if 0%{?with_python2} +%py2_install +%endif + +%if 0%{?with_python3} +%py3_install +%endif + +%check +%if 0%{?with_python2} +%{__python2} setup.py test +%endif + +%if 0%{?with_python3} +%{__python3} setup.py test +%endif + +%if 0%{?with_python2} +%files -n python2-idna +%defattr(-,root,root) +%doc README.rst HISTORY.rst +%license LICENSE.rst +%{python2_sitelib}/* +%endif + +%if 0%{?with_python3} +%files -n python3-idna +%defattr(-,root,root) +%doc README.rst HISTORY.rst +%license LICENSE.rst +%{python3_sitelib}/* +%endif + +%changelog +* Thu Jul 23 2020 zhouhaibo - 2.10-1 +- Package update + +* Wed Jan 15 2020 openEuler Buildteam - 2.8-3 +- Type:bugfix +- Id:NA +- SUG:NA +- DESC:delete the python-idna + +* Tue Jan 14 2020 openEuler Buildteam - 2.8-2 +- Type:bugfix +- Id:NA +- SUG:NA +- DESC:delete the python provides in python2 + +* Wed Sep 4 2019 openEuler Buildteam - 2.8-1 +- Package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f9dccbb3ddb50ec9d9473d0391edbc611039639 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/pypi_test/pypi_test.yaml @@ -0,0 +1,4 @@ +version_control: pypi +src_repo: idna +tag_prefix: "v" +separator: "." \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.spec b/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..8c21ec669c733fb5f013b97dad5401180fedcc1f --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.spec @@ -0,0 +1,87 @@ +Name: lame +Version: 3.100 +Release: 7 +Summary: Free MP3 audio compressor +License: GPLv2+ +URL: http://lame.sourceforge.net/ +Source0: http://downloads.sourceforge.net/sourceforge/lam/lam-3.100.tar.gz +Patch0001: lame-noexecstack.patch +Patch0002: libmp3lame-symbols.patch + +BuildRequires: ncurses-devel gtk+-devel + +Provides: %{name}-libs = %{version}-%{release} +Obsoletes: %{name}-libs < %{version}-%{release} + +%description +LAME is a high quality MPEG Audio Layer III (MP3) encoder. + +%package devel +Summary: Development files for lame +Requires: %{name} = %{version}-%{release} + +%description devel +The lame-devel package contains development files for lame. + +%package help +Summary: man info for lame + +%description help +The lame-help Package contains man info for lame. + +%package mp3x +Summary: MP3 frame analyzer +Requires: %{name} = %{version}-%{release} + +%description mp3x +The lame-mp3x package contains the mp3x frame analyzer. + + +%prep +%autosetup -p1 + + +%build +sed -i -e 's/^\(\s*hardcode_libdir_flag_spec\s*=\).*/\1/' configure +%configure \ + --disable-dependency-tracking --disable-static --enable-mp3x --enable-mp3rtp +%make_build + + +%install +%make_install +%delete_la +ln -sf lame/lame.h %{buildroot}%{_includedir}/lame.h + + +%check +make test + +%post +/sbin/ldconfig + +%postun +/sbin/ldconfig + + +%files +%exclude %{_docdir}/%{name} +%doc README TODO USAGE doc/html/*.html ChangeLog COPYING LICENSE +%{_bindir}/{lame,mp3rtp} +%{_libdir}/libmp3lame.so.0* + +%files devel +%exclude %{_docdir}/%{name} +%doc API HACKING STYLEGUIDE +%{_libdir}/libmp3lame.so +%{_includedir}/{lame,lame.h} + +%files help +%{_mandir}/man1/lame.1* + +%files mp3x +%{_bindir}/mp3x + +%changelog +* Thu Dec 12 2019 zoushuangshuang - 3.100-7 +- Package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34170dc04114a2389422aca358abe5df6e0d4b49 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/repo_name_test/svn_test.yaml @@ -0,0 +1,4 @@ +version_control: svn +src_repo: https://svn.code.sf.net/p/lame/svn +tag_prefix: "^lame" +separator: _ \ No newline at end of file diff --git a/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.spec b/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..a7b69c26be5130c7f1206d08a6b708d1e325ea10 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.spec @@ -0,0 +1,64 @@ +%global gem_name idn +Name: rubygem-%{gem_name} +Version: 0.0.2 +Release: 1 +Summary: Ruby Bindings for the GNU LibIDN library +License: ASL 2.0 and LGPLv2+ +URL: https://rubygems.org/gems/idn +Source0: https://rubygems.org/gems/%{gem_name}-%{version}.gem +Patch0: rubygem-idn-0.0.2-Fix-for-ruby-1.9.x.patch +Patch1: rubygem-idn-0.0.2-ruby2-encoding-in-tests-fix.patch +BuildRequires: ruby(release) rubygems-devel ruby-devel gcc libidn-devel rubygem(test-unit) +%description +Ruby Bindings for the GNU LibIDN library, an implementation of the Stringprep, +Punycode and IDNA specifications defined by the IETF Internationalized Domain +Names (IDN) working group. + +%package doc +Summary: Documentation for %{name} +Requires: %{name} = %{version}-%{release} +BuildArch: noarch +%description doc +Documentation for %{name}. + +%prep +%setup -q -n %{gem_name}-%{version} +%patch0 -p0 +%patch1 -p1 + +%build +gem build ../%{gem_name}-%{version}.gemspec +%gem_install + +%install +mkdir -p %{buildroot}%{gem_dir} +cp -a .%{gem_dir}/* \ + %{buildroot}%{gem_dir}/ +mkdir -p %{buildroot}%{gem_extdir_mri} +cp -a .%{gem_extdir_mri}/{gem.build_complete,*.so} %{buildroot}%{gem_extdir_mri}/ +rm -rf %{buildroot}%{gem_instdir}/ext/ + +%check +pushd .%{gem_instdir} +ruby -I$(dirs +1)%{gem_extdir_mri} -e 'Dir.glob "./test/tc_*.rb", &method(:require)' +popd + +%files +%dir %{gem_instdir} +%{gem_extdir_mri} +%license %{gem_instdir}/LICENSE +%exclude %{gem_libdir} +%exclude %{gem_cache} +%{gem_spec} + +%files doc +%doc %{gem_docdir} +%doc %{gem_instdir}/CHANGES +%doc %{gem_instdir}/NOTICE +%doc %{gem_instdir}/README +%{gem_instdir}/Rakefile +%{gem_instdir}/test + +%changelog +* Sat Jul 25 2020 wutao - 0.0.2-1 +- package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bfdd1734ef06d2b150d788850edbd7336b256d5 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/rubygem_test/rubygem_test.yaml @@ -0,0 +1,4 @@ +version_control: rubygem +src_repo: idn +tag_prefix: "" +separator: "." diff --git a/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.spec b/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.spec new file mode 100644 index 0000000000000000000000000000000000000000..fcd2463066774b3af9513a39dff239b88c7eaaee --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.spec @@ -0,0 +1,87 @@ +Name: lame +Version: 3.100 +Release: 7 +Summary: Free MP3 audio compressor +License: GPLv2+ +URL: http://lame.sourceforge.net/ +Source0: http://downloads.sourceforge.net/sourceforge/lame/%{name}-%{version}.tar.gz +Patch0001: lame-noexecstack.patch +Patch0002: libmp3lame-symbols.patch + +BuildRequires: ncurses-devel gtk+-devel + +Provides: %{name}-libs = %{version}-%{release} +Obsoletes: %{name}-libs < %{version}-%{release} + +%description +LAME is a high quality MPEG Audio Layer III (MP3) encoder. + +%package devel +Summary: Development files for lame +Requires: %{name} = %{version}-%{release} + +%description devel +The lame-devel package contains development files for lame. + +%package help +Summary: man info for lame + +%description help +The lame-help Package contains man info for lame. + +%package mp3x +Summary: MP3 frame analyzer +Requires: %{name} = %{version}-%{release} + +%description mp3x +The lame-mp3x package contains the mp3x frame analyzer. + + +%prep +%autosetup -p1 + + +%build +sed -i -e 's/^\(\s*hardcode_libdir_flag_spec\s*=\).*/\1/' configure +%configure \ + --disable-dependency-tracking --disable-static --enable-mp3x --enable-mp3rtp +%make_build + + +%install +%make_install +%delete_la +ln -sf lame/lame.h %{buildroot}%{_includedir}/lame.h + + +%check +make test + +%post +/sbin/ldconfig + +%postun +/sbin/ldconfig + + +%files +%exclude %{_docdir}/%{name} +%doc README TODO USAGE doc/html/*.html ChangeLog COPYING LICENSE +%{_bindir}/{lame,mp3rtp} +%{_libdir}/libmp3lame.so.0* + +%files devel +%exclude %{_docdir}/%{name} +%doc API HACKING STYLEGUIDE +%{_libdir}/libmp3lame.so +%{_includedir}/{lame,lame.h} + +%files help +%{_mandir}/man1/lame.1* + +%files mp3x +%{_bindir}/mp3x + +%changelog +* Thu Dec 12 2019 zoushuangshuang - 3.100-7 +- Package init diff --git a/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.yaml b/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34170dc04114a2389422aca358abe5df6e0d4b49 --- /dev/null +++ b/test/ac/acl/package_yaml/repo_test_sample/svn_test/svn_test.yaml @@ -0,0 +1,4 @@ +version_control: svn +src_repo: https://svn.code.sf.net/p/lame/svn +tag_prefix: "^lame" +separator: _ \ No newline at end of file diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py new file mode 100644 index 0000000000000000000000000000000000000000..3de78eba7ff66786d466881b0fa3b14026f17a65 --- /dev/null +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -0,0 +1,117 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-17 +# Description: test for check_repo.py +# ***********************************************************************************/ +""" + +import unittest +import mock +import os +import yaml +import logging.config +import logging +import shutil + +from src.ac.acl.package_yaml.check_repo import ReleaseTagsFactory + +logging.getLogger('test_logger') + +class TestGetReleaseTags(unittest.TestCase): + TEST_YAML_DIR = { + "hg": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/hg_test/hg_test.yaml"), + "github": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/github_test/github_test.yaml"), + "git": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/git_test/git_test.yaml"), + "gitlab.gnome": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.yaml"), + "svn": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/svn_test/svn_test.yaml"), + "metacpan": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/metacpan_test/metacpan_test.yaml"), + "pypi": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/pypi_test/pypi_test.yaml"), + "rubygem": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/rubygem_test/rubygem_test.yaml"), + "gitee": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitee_test/gitee_test.yaml"), + "gnu-ftp": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gnu_ftp_test/gnu_ftp_test.yaml") + } + + def _load_yaml(self, filepath): + result = {} + try: + with open(filepath, 'r') as yaml_data: # load yaml data + result = yaml.safe_load(yaml_data) + except IOError as e: + logging.warning("package yaml not exist. {}".format(str(e))) + except yaml.YAMLError as exc: + logging.warning("Error parsering YAML: {}".format(str(exc))) + finally: + return result + + def _get_test_tags(self, version): + yaml_content = self._load_yaml(self.TEST_YAML_DIR[version]) + vc = yaml_content.get("version_control", "") + sr = yaml_content.get("src_repo", "") + release_tags = ReleaseTagsFactory.get_release_tags(vc) + return release_tags.get_tags(sr) + + def test_get_hg_release_tags(self): + release_tags = self._get_test_tags("hg") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_github_release_tags(self): + release_tags = self._get_test_tags("github") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_git_release_tags(self): + release_tags = self._get_test_tags("git") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_gitlab_gnome_release_tags(self): + release_tags = self._get_test_tags("gitlab.gnome") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_svn_release_tags(self): + release_tags = self._get_test_tags("svn") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_metacpan_release_tags(self): + release_tags = self._get_test_tags("metacpan") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_pypi_release_tags(self): + release_tags = self._get_test_tags("pypi") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_rubygem_release_tags(self): + release_tags = self._get_test_tags("rubygem") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_gitee_release_tags(self): + release_tags = self._get_test_tags("gitee") + self.assertEqual(len(release_tags) > 0, True) + + def test_get_gnu_ftp_release_tags(self): + release_tags = self._get_test_tags("gnu-ftp") + self.assertEqual(len(release_tags) > 0, True) + +if __name__ == '__main__': + work_dir = os.getcwd() + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../../src/conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + # Test get release tags + suite = unittest.makeSuite(TestGetReleaseTags) + unittest.TextTestRunner().run(suite) + os.chdir(work_dir) + shutil.rmtree("log") + + + diff --git a/test/ac/acl/package_yaml/test_check_yaml.py b/test/ac/acl/package_yaml/test_check_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..0a189bae62142e631354e0a3cb9100b3a95658e9 --- /dev/null +++ b/test/ac/acl/package_yaml/test_check_yaml.py @@ -0,0 +1,352 @@ +# -*- encoding=utf-8 -*- +""" +# *********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: DisNight +# Create: 2020-09-17 +# Description: test for check_yaml.py +# ***********************************************************************************/ +""" + +import unittest +import mock +import sys +import os +import types +import logging.config +import logging +import shutil + +from src.ac.common.rpm_spec_adapter import RPMSpecAdapter +from src.proxy.git_proxy import GitProxy +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.acl.package_yaml.check_yaml import CheckPackageYaml +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.acl.package_yaml.check_repo import HgReleaseTags, GithubReleaseTags, GitReleaseTags, \ + GitlabReleaseTags, SvnReleaseTags, MetacpanReleaseTags, \ + PypiReleaseTags, RubygemReleaseTags, GiteeReleaseTags, \ + GnuftpReleaseTags + +logging.getLogger('test_logger') + +class TestCheckYamlField(unittest.TestCase): + + TEST_YAML_DIR = { + "missing": os.path.join(os.path.dirname(os.path.realpath(__file__)), "fields_test_sample/missing.yaml"), + "standard": os.path.join(os.path.dirname(os.path.realpath(__file__)), "fields_test_sample/standard.yaml") + } + + def setUp(self): + self.cy = CheckPackageYaml("", "", "") + def set_yaml(self, file): + self._gr.yaml_file = file + self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + + def test_none_file(self): + self.cy.set_yaml(None) + result = self.cy.check_fields() + self.assertEqual(result, WARNING) + + def test_missing_fields(self): + self.cy.set_yaml(self.TEST_YAML_DIR["missing"]) + result = self.cy.check_fields() + self.assertEqual(result, WARNING) + + def test_standard_fields(self): + self.cy.set_yaml(self.TEST_YAML_DIR["standard"]) + result = self.cy.check_fields() + self.assertEqual(result, SUCCESS) + + +class TestCheckYamlRepo(unittest.TestCase): + + SUCCESS_TAG_LIST = ["0.0.0", "0.0.1"] + FAILED_TAG_LIST = [] + TEST_YAML_DIR = { + "na": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/na_test/na.yaml"), + "hg": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/hg_test/hg_test.yaml"), + "github": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/github_test/github_test.yaml"), + "git": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/git_test/git_test.yaml"), + "gitlab.gnome": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.yaml"), + "svn": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/svn_test/svn_test.yaml"), + "metacpan": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/metacpan_test/metacpan_test.yaml"), + "pypi": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/pypi_test/pypi_test.yaml"), + "rubygem": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/rubygem_test/rubygem_test.yaml"), + "gitee": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitee_test/gitee_test.yaml"), + "gnu-ftp": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gnu_ftp_test/gnu_ftp_test.yaml") + } + + TEST_SPEC_DIR = { + "na": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/na_test/na.spec"), + "hg": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/hg_test/hg_test.spec"), + "github": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/github_test/github_test.spec"), + "git": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/git_test/git_test.spec"), + "gitlab.gnome": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitlab_gnome_test/gitlab_gnome_test.spec"), + "svn": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/svn_test/svn_test.spec"), + "metacpan": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/metacpan_test/metacpan_test.spec"), + "pypi": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/pypi_test/pypi_test.spec"), + "rubygem": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/rubygem_test/rubygem_test.spec"), + "gitee": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gitee_test/gitee_test.spec"), + "gnu-ftp": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/gnu_ftp_test/gnu_ftp_test.spec") + } + + def setUp(self): + self.cy = CheckPackageYaml("", "", "") + def set_yaml(self, file): + self._gr.yaml_file = file + def set_spec(self, file): + self._spec = RPMSpecAdapter(file) + self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + self.cy.set_spec = types.MethodType(set_spec, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + + def test_none_file(self): + self.cy.set_yaml(None) + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + def test_NA_repo(self): + self.cy.set_yaml(self.TEST_YAML_DIR["na"]) + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(HgReleaseTags, "get_tags") + def test_hg_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(HgReleaseTags, "get_tags") + def test_hg_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(GithubReleaseTags, "get_tags") + def test_github_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["github"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(GithubReleaseTags, "get_tags") + def test_github_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["github"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(GitReleaseTags, "get_tags") + def test_git_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["git"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(GitReleaseTags, "get_tags") + def test_git_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["git"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(GitlabReleaseTags, "get_tags") + def test_gitlab_gnome_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(GitlabReleaseTags, "get_tags") + def test_gitlab_gnome_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(SvnReleaseTags, "get_tags") + def test_svn_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(SvnReleaseTags, "get_tags") + def test_svn_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(MetacpanReleaseTags, "get_tags") + def test_metacpan_repoo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(MetacpanReleaseTags, "get_tags") + def test_metacpan_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(PypiReleaseTags, "get_tags") + def test_pypi_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(PypiReleaseTags, "get_tags") + def test_pypi_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(RubygemReleaseTags, "get_tags") + def test_rubygem_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(RubygemReleaseTags, "get_tags") + def test_rubygem_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(GiteeReleaseTags, "get_tags") + def test_gitee_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(GiteeReleaseTags, "get_tags") + def test_gitee_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + @mock.patch.object(GnuftpReleaseTags, "get_tags") + def test_gnu_ftp_repo_success(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"]) + mock_get_tags.return_value = self.SUCCESS_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, SUCCESS) + + @mock.patch.object(GnuftpReleaseTags, "get_tags") + def test_gnu_ftp_repo_failed(self, mock_get_tags): + self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"]) + mock_get_tags.return_value = self.FAILED_TAG_LIST + self.cy.check_fields() + result = self.cy.check_repo() + self.assertEqual(result, WARNING) + + +class TestCheckConsistency(unittest.TestCase): + DIR_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "repo_test_sample") + TEST_SAMPLE_DIR = { + "na": "na_test", + "hg": "hg_test", + "github": "github_test", + "git": "git_test", + "gitlab.gnome": "gitlab_gnome_test", + "svn": "svn_test", + "metacpan": "metacpan_test", + "pypi": "pypi_test", + "rubygem": "rubygem_test", + "gitee": "gitee_test", + "gnu-ftp": "gnu_ftp_test", + "reponame": "repo_name_test" + } + + def _test_repo_domain(self, dir_key, predict): + self.cy = CheckPackageYaml(TestCheckConsistency.DIR_PATH, + TestCheckConsistency.TEST_SAMPLE_DIR[dir_key], + None) + self.cy.check_fields() + result = self.cy.check_repo_domain() + self.assertEqual(result, predict) + + def test_common_repo_domain_success(self): + self._test_repo_domain("gitee", SUCCESS) + + def test_common_repo_domain_failed(self): + self._test_repo_domain("svn", WARNING) + + def test_gnome_repo_domain(self): + self._test_repo_domain("gitlab.gnome", SUCCESS) + + def test_pypi_repo_domain(self): + self._test_repo_domain("pypi", SUCCESS) + + def _test_repo_name(self, dir_key, predict): + self.cy = CheckPackageYaml(TestCheckConsistency.DIR_PATH, + TestCheckConsistency.TEST_SAMPLE_DIR[dir_key], + None) + self.cy.check_fields() + result = self.cy.check_repo_name() + self.assertEqual(result, predict) + + def test_common_repo_name_success(self): + self._test_repo_name("gitlab.gnome", SUCCESS) + + def test_common_repo_name_failed(self): + self._test_repo_name("reponame", WARNING) + + def test_svn_repo_name(self): + self._test_repo_name("svn", SUCCESS) + +if __name__ == '__main__': + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../../src/conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + # Test check yaml fields + suite = unittest.makeSuite(TestCheckYamlField) + unittest.TextTestRunner().run(suite) + # Test check yaml repo + suite = unittest.makeSuite(TestCheckYamlRepo) + unittest.TextTestRunner().run(suite) + # Test check repo name and repo domain + suite = unittest.makeSuite(TestCheckConsistency) + unittest.TextTestRunner().run(suite) + shutil.rmtree("log") \ No newline at end of file