开发Maven插件：实现打包后上传tos云服务

开发Maven插件：实现打包后上传tos云服务

契机

⚙ 当前业务项目只有test和uat走的流水线，prod环境都需要手动打包部署，由于生产服务器的特殊性，只能用VPN链接访问，上行带宽不够导致发版绝大部分时间都浪费到上传jar包到linux服务器。当前阶段，我们公司使用的是火山引擎ECS+TOS云存储，在一个地域下ecs可以实现内网访问tos数据。故考虑使用maven写一个tos上传插件，方便环境的CICD

代码实现

直接在idea新建maven项目这里不多赘述

引入依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.bothsavage</groupId>
    <artifactId>jar2tos-plugin</artifactId>
    <version>1.0.0</version>
    <packaging>maven-plugin</packaging>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!--plugin相关依赖-->
        <dependency>
            <groupId>org.apache.maven</groupId>
            <artifactId>maven-plugin-api</artifactId>
            <version>3.8.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.maven.plugin-tools</groupId>
            <artifactId>maven-plugin-annotations</artifactId>
            <version>3.2</version>
            <scope>provided</scope>
        </dependency>
        <!--火山引擎tos相关依赖-->
        <dependency>
            <groupId>com.volcengine</groupId>
            <artifactId>ve-tos-java-sdk</artifactId>
            <version>2.6.6</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-plugin-plugin</artifactId>
                <version>3.6.0</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>descriptor</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>
    
    <!--如果就你一个人开发也可以不用搞这个，install到本地就可以使用了-->
    <!--插件远程deploy仓库-->
    <distributionManagement>
        <repository>
            <id>repo-bothsvage</id>
            <url>https://packages.aliyun.com/x/x/repo-bothsvage</url>
        </repository>
    </distributionManagement>
    
</project>

上传tos代码

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosClientException;
import com.volcengine.tos.TosServerException;
import com.volcengine.tos.comm.event.DataTransferListener;
import com.volcengine.tos.comm.event.DataTransferStatus;
import com.volcengine.tos.comm.event.DataTransferType;
import com.volcengine.tos.model.object.PutObjectInput;
import com.volcengine.tos.model.object.PutObjectOutput;

import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

@Mojo(name = "upload")
public class Upload extends AbstractMojo {

    @Parameter(property = "upload-plugin-endpoint", required = true)
    private String endpoint;

    @Parameter(property = "upload-plugin-region", required = true)
    private String region;

    @Parameter(property = "upload-plugin-bucketName", required = true)
    private String bucketName;

    @Parameter(property = "upload-plugin-accessKey", required = true)
    private String accessKey;

    @Parameter(property = "upload-plugin-secretKey", required = true)
    private String secretKey;

    @Parameter(defaultValue = "${project.basedir}/target", readonly = true)
    private File targetDirectory;

    @Parameter(property = "profiles.nacosNameSpace", required = false)
    private String profileId;

    @Override
    public void execute() throws MojoExecutionException {
        String timestamp = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(new Date());

        getLog().info("上传bucketName：" + bucketName);
        getLog().info("上传endpoint：" + endpoint);
        getLog().info("上传region：" + region);

        File[] jarFiles = findJarFiles(targetDirectory);
        if (jarFiles == null || jarFiles.length == 0) {
            getLog().warn("这个目录没有jar包，忽略");
            return;
        }

        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey);

        for (File jarFile : jarFiles) {
            // 添加条件以仅上传特定的 JAR 文件
            if (!shouldUpload(jarFile.getName())) {
                continue;
            }

            getLog().info("开始上传：" + jarFile.getName());

            try (FileInputStream inputStream = new FileInputStream(jarFile)) {
                String path = "xxxx/" + jarFile.getName();

                PutObjectInput putObjectInput = new PutObjectInput().setBucket(bucketName)
                    .setKey(path)
                    .setContent(inputStream)
                    .setContentLength(jarFile.length());

                DataTransferListener listener = getDataTransferListener();
                putObjectInput.setDataTransferListener(listener);

                PutObjectOutput output = tos.putObject(putObjectInput);

                getLog().info(
                    jarFile.getName().replace(".jar", "") + "：\n\nwget https://" + bucketName + "." + endpoint.replace(
                        "volces", "ivolces") + "/" + path + "\n");
            } catch (IOException e) {
                getLog().error("putObject read file failed for " + jarFile.getName(), e);
            } catch (TosClientException | TosServerException e) {
                getLog().error("putObject failed for " + jarFile.getName(), e);
            } catch (Throwable t) {
                getLog().error("putObject failed for " + jarFile.getName(), t);
            }
        }
    }

    private File[] findJarFiles(File directory) {
        return directory.listFiles((dir, name) -> name.endsWith(".jar"));
    }

    private boolean shouldUpload(String fileName) {
        // 定义允许上传的文件名称列表
        String[] allowedFiles = {
            "xxx-xxxx.jar"
        };

        // 检查文件名是否在允许列表中
        for (String allowedFile : allowedFiles) {
            if (fileName.equals(allowedFile)) {
                return true;
            }
        }
        return false;
    }

    private static DataTransferListener getDataTransferListener() {
        return dataTransferStatus -> {
            long totalBytes = dataTransferStatus.getTotalBytes();
            long consumedBytes = dataTransferStatus.getConsumedBytes();
            double percentage = (totalBytes > 0) ? (double) consumedBytes / totalBytes * 100 : 0;

            if (dataTransferStatus.getType() == DataTransferType.DATA_TRANSFER_STARTED) {
                System.out.println("上传开始。");
            } else if (dataTransferStatus.getType() == DataTransferType.DATA_TRANSFER_RW) {
                System.out.printf("上传中，本次发送 %d 字节，已发送 %d 字节，总计 %d 字节。完成百分比：%.2f%%\n",
                    dataTransferStatus.getRwOnceBytes(), consumedBytes, totalBytes, percentage);
            } else if (dataTransferStatus.getType() == DataTransferType.DATA_TRANSFER_FAILED) {
                System.out.printf("上传失败，已发送 %d 字节，总计 %d 字节。完成百分比：%.2f%%\n", consumedBytes, totalBytes,
                    percentage);
            } else if (dataTransferStatus.getType() == DataTransferType.DATA_TRANSFER_SUCCEED) {
                System.out.printf("上传成功，已发送 %d 字节，总计 %d 字节。完成百分比：%.2f%%\n", consumedBytes, totalBytes,
                    percentage);
            }
        };
    }

}

删除临时文件

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosClientException;
import com.volcengine.tos.TosServerException;
import com.volcengine.tos.model.object.DeleteObjectInput;
import com.volcengine.tos.model.object.DeleteObjectOutput;
import com.volcengine.tos.model.object.ListObjectsType2Input;
import com.volcengine.tos.model.object.ListObjectsType2Output;
import com.volcengine.tos.model.object.PutObjectInput;
import com.volcengine.tos.model.object.PutObjectOutput;

import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

@Mojo(name = "delete")
public class Delete extends AbstractMojo {

    @Parameter(property = "upload-plugin-endpoint", required = true)
    private String endpoint;

    @Parameter(property = "upload-plugin-region", required = true)
    private String region;

    @Parameter(property = "upload-plugin-bucketName", required = true)
    private String bucketName;

    @Parameter(property = "upload-plugin-accessKey", required = true)
    private String accessKey;

    @Parameter(property = "upload-plugin-secretKey", required = true)
    private String secretKey;

    @Override
    public void execute() {

        String prefix = "xxxx/";

        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey);

        try {
            boolean isTruncated = true;
            String continuationToken = null;
            while (isTruncated) {
                ListObjectsType2Input input = new ListObjectsType2Input().setBucket(bucketName)
                    .setPrefix(prefix)
                    .setContinuationToken(continuationToken);
                ListObjectsType2Output output = tos.listObjectsType2(input);
                if (output.getContents() != null) {
                    for (int i = 0; i < output.getContents().size(); i++) {
                        String objectKey = output.getContents().get(i).getKey();
                        DeleteObjectInput deleteInput = new DeleteObjectInput().setBucket(bucketName)
                            .setKey(output.getContents().get(i).getKey());
                        tos.deleteObject(deleteInput);
                        System.out.println("删除成功： " + objectKey);
                    }
                }
                isTruncated = output.isTruncated();
                continuationToken = output.getNextContinuationToken();
            }

        } catch (TosClientException e) {
            // 操作失败，捕获客户端异常，一般情况是请求参数错误，此时请求并未发送
            System.out.println("deleteObject failed");
            System.out.println("Message: " + e.getMessage());
            if (e.getCause() != null) {
                e.getCause().printStackTrace();
            }
        } catch (TosServerException e) {
            // 操作失败，捕获服务端异常，可以获取到从服务端返回的详细错误信息
            System.out.println("deleteObject failed");
            System.out.println("StatusCode: " + e.getStatusCode());
            System.out.println("Code: " + e.getCode());
            System.out.println("Message: " + e.getMessage());
            System.out.println("RequestID: " + e.getRequestID());
        } catch (Throwable t) {
            // 作为兜底捕获其他异常，一般不会执行到这里
            System.out.println("deleteObject failed");
            System.out.println("unexpected exception, message: " + t.getMessage());
        }
    }

}

项目引入

上述插件项目打包后，引入业务项目pom中

	<build>
		<plugins>
        <!--jar包上传tos插件-->
        <plugin>
            <groupId>com.xxxxx</groupId>
            <artifactId>jar2tos-plugin</artifactId>
            <version>1.0.0</version>
            <executions>
                <execution>
                    <goals>
                        <goal>upload</goal>
                        <goal>delete</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
		</plugins>
	</build>

<profile>
	  <id>prod</id>
	  <properties>
	      <profile.name>prod</profile.name>
	      <!--上传插件信息-->
	      <upload-plugin-endpoint>tos-xxx.volces.com</upload-plugin-endpoint>
	      <upload-plugin-region>tos-xxx</upload-plugin-region>
	      <upload-plugin-bucketName>xxxxx</upload-plugin-bucketName>
	      <upload-plugin-accessKey>AKXXXX</upload-plugin-accessKey>
	      <upload-plugin-secretKey>SKXXXX</upload-plugin-secretKey>
	  </properties>
</profile>

<!--如果你的插件在远端，还需要配上这个的-->
<pluginRepositories>
  <pluginRepository>
      <id>repo-xxx</id>
      <url>https://packages.xx.com/xx/maven/repo-xxx</url>
      <releases>
          <enabled>true</enabled>
      </releases>
      <snapshots>
          <enabled>true</enabled>
      </snapshots>
  </pluginRepository>
</pluginRepositories>

使用

#上传完成会返回一个内网下载地址，直接复制到linux中执行
wget https://xxxxx.tos-cn-xxx.ivolces.com/xxxx/oxxe-xxxx.jar

总结

• 因为bucket是公共读，所以直接发版完成需要删除文件
• bucket应该做权限认证访问，我偷懒没做，不要使用业务bucket问题不大
• 如果协同开发，plugin的插件仓库不配置无法拉取

写到最后