diff --git a/tools/install_dependency/src/download/download_config.py b/tools/install_dependency/src/download/download_config.py index 2964c325829f44343c38ade5aaf3c9bd6610ac0a..ac84b37199f1944d097a64d4d189dccb69672da5 100644 --- a/tools/install_dependency/src/download/download_config.py +++ b/tools/install_dependency/src/download/download_config.py @@ -47,78 +47,93 @@ DevKitCLI = { OpenEuler_2003_LTS = { "component_name": "openEuler_2003_LTS", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS/ISO/aarch64/openEuler-20.03-LTS-everything-aarch64-dvd.iso", + "file_size": "11679975424", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS/ISO/aarch64/openEuler-20.03-LTS-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2003_LTS_SP1 = { "component_name": "openEuler_2003_LTS_SP1", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP1/ISO/aarch64/openEuler-20.03-LTS-SP1-everything-aarch64-dvd.iso", + "file_size": "13486159872", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP1/ISO/aarch64/openEuler-20.03-LTS-SP1-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2003_LTS_SP2 = { "component_name": "openEuler_2003_LTS_SP2", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP2/ISO/aarch64/openEuler-20.03-LTS-SP2-everything-aarch64-dvd.iso", + "file_size": "16177231872", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP2/ISO/aarch64/openEuler-20.03-LTS-SP2-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2003_LTS_SP3 = { "component_name": "openEuler_2003_LTS_SP3", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP3/ISO/aarch64/openEuler-20.03-LTS-SP3-everything-aarch64-dvd.iso", + "file_size": "16473085952", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP3/ISO/aarch64/openEuler-20.03-LTS-SP3-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2003_LTS_SP4 = { "component_name": "openEuler_2003_LTS_SP4", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP4/ISO/aarch64/openEuler-20.03-LTS-SP4-everything-aarch64-dvd.iso", + "file_size": "17341517824", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP4/ISO/aarch64/openEuler-20.03-LTS-SP4-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2009 = { "component_name": "openEuler_2009", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.09/ISO/aarch64/openEuler-20.09-everything-aarch64-dvd.iso", + "file_size": "13531158528", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.09/ISO/aarch64/openEuler-20.09-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2103 = { "component_name": "openEuler_2103", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.03/ISO/aarch64/openEuler-21.03-everything-aarch64-dvd.iso", + "file_size": "14570956800", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.03/ISO/aarch64/openEuler-21.03-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2109 = { "component_name": "openEuler_2109", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.09/ISO/aarch64/openEuler-21.09-everything-aarch64-dvd.iso", + "file_size": "16063641600", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.09/ISO/aarch64/openEuler-21.09-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2203_LTS = { "component_name": "openEuler_2203_LTS", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS/ISO/aarch64/openEuler-22.03-LTS-everything-aarch64-dvd.iso", + "file_size": "16383141888", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS/ISO/aarch64/openEuler-22.03-LTS-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2203_LTS_SP1 = { "component_name": "openEuler_2203_LTS_SP1", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP1/ISO/aarch64/openEuler-22.03-LTS-SP1-everything-aarch64-dvd.iso", + "file_size": "17282813952", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP1/ISO/aarch64/openEuler-22.03-LTS-SP1-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2203_LTS_SP2 = { "component_name": "openEuler_2203_LTS_SP2", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP2/ISO/aarch64/openEuler-22.03-LTS-SP2-everything-aarch64-dvd.iso", + "file_size": "17669261312", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP2/ISO/aarch64/openEuler-22.03-LTS-SP2-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2203_LTS_SP3 = { "component_name": "openEuler_2203_LTS_SP3", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/ISO/aarch64/openEuler-22.03-LTS-SP3-everything-aarch64-dvd.iso", + "file_size": "17898815488", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/ISO/aarch64/openEuler-22.03-LTS-SP3-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2209 = { "component_name": "openEuler_2209", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.09/ISO/aarch64/openEuler-22.09-everything-aarch64-dvd.iso", + "file_size": "18168621056", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.09/ISO/aarch64/openEuler-22.09-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2303 = { "component_name": "openEuler_2303", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.03/ISO/aarch64/openEuler-23.03-everything-aarch64-dvd.iso", + "file_size": "19543351296", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.03/ISO/aarch64/openEuler-23.03-everything-aarch64-dvd.iso.sha256sum", } OpenEuler_2309 = { "component_name": "openEuler_2309", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.09/ISO/aarch64/openEuler-23.09-everything-aarch64-dvd.iso", + "file_size": "20087941120", "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.09/ISO/aarch64/openEuler-23.09-everything-aarch64-dvd.iso.sha256sum", } diff --git a/tools/install_dependency/src/download/download_utils.py b/tools/install_dependency/src/download/download_utils.py index 42022c0842d95a59a225952d57c5a2e6ca784e94..a1acd84b825e4e9322e5ffb3572a366738b2bfd2 100644 --- a/tools/install_dependency/src/download/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -7,7 +7,7 @@ import warnings import requests import wget from download import download_config -from constant import URL, FILE, SAVE_PATH, SHA256, DEFAULT_PATH +from constant import URL, SAVE_PATH, FILE, SHA256, FILE_SIZE, DEFAULT_PATH warnings.filterwarnings("ignore", message='Unverified HTTPS request') @@ -18,40 +18,48 @@ component_collection_map = { "download file": { URL: f"{download_config.BiShengCompiler.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(FILE).split('/')[-1])}", + FILE_SIZE: "1051195289", }, "download sha256": { URL: f"{download_config.BiShengCompiler.get(SHA256)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(SHA256).split('/')[-1])}", + FILE_SIZE: "107", }, }, "GCCforOpenEuler": { "download file": { URL: f"{download_config.GCCforOpenEuler.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(FILE).split('/')[-1])}", + FILE_SIZE: "274901693", }, "download sha256": { URL: f"{download_config.GCCforOpenEuler.get(SHA256)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(SHA256).split('/')[-1])}", + FILE_SIZE: "106", }, }, "BiShengJDK8": { "download file": { URL: f"{download_config.BiShengJDK8.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(FILE).split('/')[-1])}", + FILE_SIZE: "117055434", }, "download sha256": { URL: f"{download_config.BiShengJDK8.get(SHA256)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(SHA256).split('/')[-1])}", + FILE_SIZE: "105", }, }, "BiShengJDK17": { "download file": { URL: f"{download_config.BiShengJDK17.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(FILE).split('/')[-1])}", + FILE_SIZE: "196772672", }, "download sha256": { URL: f"{download_config.BiShengJDK17.get(SHA256)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(SHA256).split('/')[-1])}", + FILE_SIZE: "107", }, }, @@ -59,18 +67,22 @@ component_collection_map = { "download file": { URL: f"{download_config.LkpTests.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get(FILE).split('/')[-1])}", + FILE_SIZE: "29333270", }, "GemDependency": { URL: f"{download_config.LkpTests.get('GemDependency')}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('GemDependency').split('/')[-1])}", + FILE_SIZE: "4206309", }, "CompatibilityTesting": { URL: f"{download_config.LkpTests.get('CompatibilityTesting')}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('CompatibilityTesting').split('/')[-1])}", + FILE_SIZE: "76645477", }, "DevkitDistribute": { URL: f"{download_config.LkpTests.get('DevkitDistribute')}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('DevkitDistribute').split('/')[-1])}", + FILE_SIZE: "13349673", } }, @@ -78,13 +90,15 @@ component_collection_map = { "download file": { URL: f"{download_config.A_FOT.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.A_FOT.get(FILE).split('/')[-1])}", + FILE_SIZE: "15740", } }, "DevKitWeb": { "download file": { URL: f"{download_config.DevKitWeb.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.DevKitWeb.get(FILE).split('/')[-1])}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'DevKit-All-23.0.T30-Linux-Kunpeng.tar.gz')}", + FILE_SIZE: "1143374523", }, }, @@ -92,6 +106,7 @@ component_collection_map = { "download file": { URL: f"{download_config.DevKitCLI.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz')}", + FILE_SIZE: "300000000", } }, } @@ -128,9 +143,10 @@ def download_dependence_file(shell_cmd, shell_dict): url_and_save_path = shell_dict.get(shell_cmd) url_ = url_and_save_path.get("url") save_path = url_and_save_path.get("save_path") + file_size = url_and_save_path.get("file_size") try: print(f"Downloading from {url_}") - download(url_, save_path) + download(url_, save_path, file_size) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError, @@ -156,8 +172,8 @@ def download_dependence_file(shell_cmd, shell_dict): return ret -def download(url, save_path): - if os.path.exists(save_path) and os.path.isfile(save_path): +def download(url, save_path, file_size): + if os.path.exists(save_path) and os.path.isfile(save_path) and str(os.path.getsize(save_path)) == file_size: return if platform.system() == "Windows": wget.download(url, save_path) diff --git a/tools/install_dependency/src/download_main.py b/tools/install_dependency/src/download_main.py index 8100758e54aec0f5d1a17ce22a3673b446045bb2..cb0e0719698f1ac430b08760386836975bff9eb2 100644 --- a/tools/install_dependency/src/download_main.py +++ b/tools/install_dependency/src/download_main.py @@ -12,7 +12,7 @@ from handler.gather_package import GatherPackage from handler.compress_dep import CompressDep from utils import read_yaml_file -from constant import URL, SHA256, SAVE_PATH, FILE, INSTRUCTION +from constant import URL, SAVE_PATH, FILE, SHA256, FILE_SIZE, INSTRUCTION PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] @@ -20,16 +20,11 @@ PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] iso_collection_map = { component.get("component_name"): { - "download file": - { - URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join('./', component.get(FILE).split('/')[-1])}" - }, - "download sha256": - { - URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join('./', component.get(SHA256).split('/')[-1])}" - } + "download file": { + URL: f"{component.get(FILE)}", + SAVE_PATH: f"{os.path.join('./', component.get(FILE).split('/')[-1])}", + FILE_SIZE: f"{component.get(FILE_SIZE)}", + }, } for component in ( download_config.OpenEuler_2003_LTS, download_config.OpenEuler_2003_LTS_SP1,