diff --git a/README.md b/README.md index 578fc04b72fd8cdd684015b53675012e29bf3dda..b42e59290034e8198ec940e3454107144f1b182b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Jenkins华为云插件 +## 工作原理 +![上传](image/jenkins.jpg) + ## 插件使用 #### 下载源码&编译 @@ -59,9 +62,27 @@ ak/sk示例1：OBSPipelineUpload(ak:"...",sk:"...",endpoint:"obs.cn-south-1.myhu ![上传](image/pipeline.jpg) -## 2、CodeArts Check代码检查功能 +## 2、OBS文件下载功能 -#### Pipeline流水线上传 +#### Pipeline流水线下载文件 + +| 参数名称 | 参数说明 | +| ------------------ | ------------------------------------------------------------ | +| ak | Access Key | +| sk | Secret Key | +| endpoint | 终端节点 +| bucket | 需要下载对象所在桶名 | +| remotePrefix | 需要下载对象对象的前缀名 | +| localFolder | 需要下载到的本地路径(在jenkins的安装路径下存放文件) | + +##### 片段生成器生成 +示例：OBSPipelineDownload( ak: '', bucket: '', endpoint: '', localFolder: '', remotePrefix: '', sk: '') + +![上传](image/obsDownload.jpg) + +## 3、CodeArts Check代码检查功能 + +#### Pipeline流水线触发 ##### 参数说明 @@ -76,13 +97,13 @@ ak/sk示例1：OBSPipelineUpload(ak:"...",sk:"...",endpoint:"obs.cn-south-1.myhu ##### 片段生成器生成 -示例：CodeCheckPipeline(ak:'',sk'',region:''',project_id:'',task_id:'', topicUrn:'') +示例：CodeCheckPipeline(ak:'',sk'',region:'',project_id:'',task_id:'', topicUrn:'') ![上传](image/checkpipeline.jpg) -## 3、VSS漏洞扫描功能 +## 4、VSS漏洞扫描功能 -#### Pipeline流水线上传 +#### Pipeline流水线触发 ##### 参数说明 @@ -109,10 +130,31 @@ ak/sk示例1：OBSPipelineUpload(ak:"...",sk:"...",endpoint:"obs.cn-south-1.myhu ##### 片段生成器生成 示例：VssPipeline(ak:'',sk:'',region:'cn-north-4',task_name:'test',url:'') - ![上传](image/vsspipeline.jpg) + +## 5、CodeArts流水线 + +#### Pipeline触发CodeArts流水线 + +##### 参数说明 + +| 参数名称 | 参数说明 | +| ------------------ | ------------------------------------------------------------ | +| ak | Access Key | +| sk | Secret Key | +| region | Region:所在区域，如cn-north-4 +| project_id | codeArts项目id | +| pipeline_id | codeArts流水线id | + +##### 片段生成器生成 + +示例：CodeArtsPipeline(ak:'',sk:'',region:'cn-north-4',project_id:'',pipeline_id:'') + +![上传](image/codeArtsPipeline.jpg) + + ## 构建 1.修改${USER}/.m2/settings.xml中的maven配置文件 @@ -160,3 +202,11 @@ mvn clean package -DskipTests ```bash mvn clean hpi:run ``` + +## jenkins官网 +关于更多huaweicloud-jenkins-plugins的信息和案例可前往华为云官网查看 +https://developer.huaweicloud.com/develop/resource/opensource/jenkins.html + +## 在华为云应用商店快速应用 +已放入华为云应用商店，您只需开通一个华为云账户，即可一键安装部署好，可直接访问使用。 +https://marketplace.huaweicloud.com/contents/5de07bbe-e025-44e4-b065-808c302d9a91#productid=OFFI820561133680414720 \ No newline at end of file diff --git a/image/codeArtsPipeline.jpg b/image/codeArtsPipeline.jpg new file mode 100644 index 0000000000000000000000000000000000000000..318490a79f707fb473efc9250d360d02fa7fbbe6 Binary files /dev/null and b/image/codeArtsPipeline.jpg differ diff --git a/image/jenkins.jpg b/image/jenkins.jpg new file mode 100644 index 0000000000000000000000000000000000000000..98445799e5da2ba5e8623dc47afe35f7447f6075 Binary files /dev/null and b/image/jenkins.jpg differ diff --git a/image/obsDownload.jpg b/image/obsDownload.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d6b1ac7f99950eb4d7662ecdfe4fdf82ee4c8bd5 Binary files /dev/null and b/image/obsDownload.jpg differ diff --git a/src/main/java/com/huawei/jenkins/plugins/storage/CustomInput.java b/src/main/java/com/huawei/jenkins/plugins/storage/CustomInput.java index e0ea07333fa25a8ec24be1b0dea8f71b2b57e09b..39f68a108575713927758b72f07cbdcb315065d8 100644 --- a/src/main/java/com/huawei/jenkins/plugins/storage/CustomInput.java +++ b/src/main/java/com/huawei/jenkins/plugins/storage/CustomInput.java @@ -60,6 +60,12 @@ public class CustomInput implements Serializable { // 重定向 private String redirectLocation; + // 下载对象的前缀名 + private String remotePrefix; + + //下载到的本地路径 + private String localFolder; + public String getAk() { return ak; } @@ -203,4 +209,20 @@ public class CustomInput implements Serializable { public void setRedirectLocation(String redirectLocation) { this.redirectLocation = redirectLocation; } + + public String getRemotePrefix() { + return remotePrefix; + } + + public void setRemotePrefix(String remotePrefix) { + this.remotePrefix = remotePrefix; + } + + public String getLocalFolder() { + return localFolder; + } + + public void setLocalFolder(String localFolder) { + this.localFolder = localFolder; + } } diff --git a/src/main/java/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep.java b/src/main/java/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep.java new file mode 100644 index 0000000000000000000000000000000000000000..723a6a911126918bf1e14b5c1d1eb4b93f97f45b --- /dev/null +++ b/src/main/java/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep.java @@ -0,0 +1,163 @@ +/* + * Copyright 2022. Huawei Technologies Co., Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.huawei.jenkins.plugins.storage; + +import hudson.EnvVars; +import hudson.Extension; +import hudson.FilePath; +import hudson.model.TaskListener; +import org.apache.commons.lang.StringUtils; +import org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl; +import org.jenkinsci.plugins.workflow.steps.*; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * pipeline语法支持obs文件上传 + */ +public class OBSDownloadPipelineStep extends Step { + private String ak; + + private String sk; + + private String bucket; + + private String endpoint; + + private String remotePrefix; + + private String localFolder; + + + public String getAk() { + return ak; + } + + @DataBoundSetter + public void setAk(String ak) { + this.ak = ak; + } + + public String getSk() { + return sk; + } + + @DataBoundSetter + public void setSk(String sk) { + this.sk = sk; + } + + public String getBucket() { + return bucket; + } + + @DataBoundSetter + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getEndpoint() { + return endpoint; + } + + @DataBoundSetter + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getRemotePrefix() { + return remotePrefix; + } + + @DataBoundSetter + public void setRemotePrefix(String remotePrefix) { + this.remotePrefix = remotePrefix; + } + + public String getLocalFolder() { + return localFolder; + } + + @DataBoundSetter + public void setLocalFolder(String localFolder) { + this.localFolder = localFolder; + } + + @DataBoundConstructor + public OBSDownloadPipelineStep(String bucket, String endpoint) { + this.bucket = bucket; + this.endpoint = endpoint; + } + + @Override + public StepExecution start(StepContext context) throws Exception { + return new Execution(this, context); + } + + @Extension + public static class DescriptorImpl extends StepDescriptor { + + @Override + public Set> getRequiredContext() { + return requires(TaskListener.class, EnvVars.class, FilePath.class); + } + + @Override + public String getFunctionName() { + return "OBSPipelineDownload"; + } + + @Override + public String getDisplayName() { + return "华为云OBS文件下载"; + } + } + + public static > Set requires(T... classes) { + return new HashSet<>(Arrays.asList(classes)); + } + + public static class Execution extends SynchronousNonBlockingStepExecution { + + protected static final long serialVersionUID = 1L; + + protected final transient OBSDownloadPipelineStep step; + + public Execution(OBSDownloadPipelineStep step, StepContext context) { + super(context); + this.step = step; + + } + + @Override + public String run() throws Exception { + CustomInput customInput = new CustomInput(); + customInput.setAk(step.getAk()); + customInput.setSk(step.getSk()); + customInput.setBucket(step.getBucket()); + customInput.setEndpoint(step.getEndpoint()); + customInput.setRemotePrefix(step.getRemotePrefix()); + customInput.setLocalFolder(step.getLocalFolder()); + TaskListener listener = Execution.this.getContext().get(TaskListener.class); + return ObsService.download(listener, customInput); + } + } +} diff --git a/src/main/java/com/huawei/jenkins/plugins/storage/ObsService.java b/src/main/java/com/huawei/jenkins/plugins/storage/ObsService.java index 4668e88ddd841719ea0b7f15692399a6d100a43c..1d11e6d36c6d9473b156c46f70c742c0130298a2 100644 --- a/src/main/java/com/huawei/jenkins/plugins/storage/ObsService.java +++ b/src/main/java/com/huawei/jenkins/plugins/storage/ObsService.java @@ -24,6 +24,7 @@ import com.cloudbees.plugins.credentials.impl.BaseStandardCredentials; import com.cloudbees.plugins.credentials.matchers.IdMatcher; import com.google.common.base.Preconditions; import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; import com.obs.services.model.*; import hudson.EnvVars; import hudson.FilePath; @@ -41,6 +42,7 @@ import org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl; import java.io.File; import java.io.IOException; import java.io.Serializable; +import java.nio.file.Paths; import java.util.*; /** @@ -211,6 +213,76 @@ public class ObsService implements Serializable { return AccessControlList.REST_CANNED_PRIVATE; } + + public static String download(TaskListener listener,CustomInput customInput) { + downloadInputValidate(customInput); + ObsClient obsClient = new ObsClient(customInput.getAk(), customInput.getSk(), customInput.getEndpoint()); + List failedList = new ArrayList<>(); + + ListObjectsRequest listRequest = new ListObjectsRequest(); + listRequest.setPrefix(customInput.getRemotePrefix()); + listRequest.setBucketName(customInput.getBucket()); + listRequest.setEncodingType("url"); + ObjectListing objects; + + for (int i = 1; ; i++) { + listener.getLogger().println("Start to Download page " + i); + try { + objects = obsClient.listObjects(listRequest); + } catch (ObsException e) { + listener.getLogger().println("Request Error, RequestID is: " + e.getErrorRequestId()); + throw e; + } + + for (ObsObject object: objects.getObjects()) { + if(object.getObjectKey().endsWith("/")){ + continue; + } + DownloadFileRequest downloadRequest = new DownloadFileRequest(customInput.getBucket(), object.getObjectKey()); + // 将对象名转换为本地路径 + String downloadPath = Paths.get(customInput.getLocalFolder(), + object.getObjectKey().replace("/", File.separator)).toString(); + + downloadRequest.setDownloadFile(downloadPath); + // 设置并发下载数为 10 + downloadRequest.setTaskNum(10); + try { + listener.getLogger().println("Start to download object [" + object.getObjectKey() + + "] to [" + downloadPath + "]"); + obsClient.downloadFile(downloadRequest); + } catch (ObsException e) { + listener.getLogger().println("Failed to download " + object.getObjectKey() + + " RequestID: " + e.getErrorRequestId()); + failedList.add(object.getObjectKey()); + } catch (Exception e) { + failedList.add(object.getObjectKey()); + } + } + if (!objects.isTruncated()) { + break; + } + // 使用上次返回的 next_marker 作为下次列举的 marker + listRequest.setMarker(objects.getNextMarker()); + } + failedList.forEach(item -> listener.getLogger().println("Failed to download " + item + ", please try again")); + return "success"; + } + + private static void downloadInputValidate(CustomInput customInput) { + final String ak = customInput.getAk(); + final String sk = customInput.getSk(); + final String bucketName = customInput.getBucket(); + final String endpoint = customInput.getEndpoint(); + final String localFolder = customInput.getLocalFolder(); + + Preconditions.checkArgument(StringUtils.isNotBlank(localFolder), "local Folder path can not be blank"); + Preconditions.checkArgument(StringUtils.isNotBlank(endpoint), "Obs endpoint can not be blank"); + Preconditions.checkArgument(StringUtils.isNotBlank(bucketName), "Obs bucketName can not be blank"); + Preconditions.checkArgument(StringUtils.isNotBlank(ak) && StringUtils.isNotBlank(sk), "Ak, Sk can not be blank"); + } + + + private static class RemoteUploader extends MasterToSlaveFileCallable { protected static final long serialVersionUID = 1L; diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/config.jelly b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/config.jelly new file mode 100644 index 0000000000000000000000000000000000000000..8a41d37649af5ae41335c5889bebb445f5401e48 --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/config.jelly @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-ak.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-ak.html new file mode 100644 index 0000000000000000000000000000000000000000..7d9c6d8413cb53da24850b648c762d2a1e27b895 --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-ak.html @@ -0,0 +1,19 @@ + +
+ HuaweiCloud Access Key, used only when the credential id is null
+ 去配置 +
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-bucket.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-bucket.html new file mode 100644 index 0000000000000000000000000000000000000000..d3b7d935b0dc9c2ff9f62db8e0d9cc8aeff9d97d --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-bucket.html @@ -0,0 +1,19 @@ + +
+ This is the bucket to use
+ config +
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-endpoint.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-endpoint.html new file mode 100644 index 0000000000000000000000000000000000000000..21584ded2743beafb0ca5046424ced12508357f6 --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-endpoint.html @@ -0,0 +1,18 @@ + +
+ This is the endpoint to use. +
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-localFolder.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-localFolder.html new file mode 100644 index 0000000000000000000000000000000000000000..fe8de5b2cfceddda2d86fbe82fa6b05f1b24bdba --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-localFolder.html @@ -0,0 +1,18 @@ + +
+ Local path to download
+
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-remotePrefix.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-remotePrefix.html new file mode 100644 index 0000000000000000000000000000000000000000..e235e0ccacbc28a7d053f8e831074609434a3d65 --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-remotePrefix.html @@ -0,0 +1,18 @@ + +
+ You need to download the object prefix of the object
+
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-sk.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-sk.html new file mode 100644 index 0000000000000000000000000000000000000000..59444304796d1c9468fe6e0762d5ec7d03ec388f --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help-sk.html @@ -0,0 +1,19 @@ + +
+ HuaweiCloud Secret Key, used only when the credential id is null
+ config +
diff --git a/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help.html b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help.html new file mode 100644 index 0000000000000000000000000000000000000000..a418bd6988f404160d22f2a4f2e674cb7a466447 --- /dev/null +++ b/src/main/resources/com/huawei/jenkins/plugins/storage/OBSDownloadPipelineStep/help.html @@ -0,0 +1,21 @@ + +
+

+ Download a file/folder from the workspace to an obs bucket. + If the RemotePrefix parameter denotes a directory, then the complete directory (including all sub folders) will be download. +

+