deployments/sharedLibs/stages.k8s-prod.groovy

sharedLibsPath = "${env.WORKSPACE}/sharedLibs"
backup_destination = '/var/jenkins_home/backups'
backup_targetFile = 'javaDockerImage.tar'
BACKUP_MAX = 2

generateFullImageUri = {Map DOCKER, String base_branch, String BUILD_NUMBER ->
    return "${DOCKER.registry}/${DOCKER.image}:${base_branch}-${BUILD_NUMBER}"
}

generateImageName = {Map DOCKER, String base_branch, String BUILD_NUMBER ->
    return "${DOCKER.image}:${base_branch}-${BUILD_NUMBER}"
}

def processString(String content) {
    return content
}

def buildJar(Object SERVICE, String path='projdir') {
    
    echo "-----> Building project package ..."
    dir(path) {
        if (SERVICE.module != null && SERVICE.module.trim())
            sh "$MAVEN_HOME/bin/mvn clean package -pl \"${SERVICE.module}\" -am -Dmaven.test.skip=true"
        else
            sh "$MAVEN_HOME/bin/mvn clean package -Dmaven.test.skip=true"
    }
    return path
}

@Deprecated
def backup_jar(Object SERVICE, String srcPath, String workingPath='projdir') {

    echo "-----> Backup project artifact ..."
    def module = SERVICE.module == null ? '' : (SERVICE.module[-1] == '/' ? SERVICE.module : SERVICE.module + '/')
    path = "${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}/target"
    sh """if [ -d ${path} ];
    then
        echo \"${path} already exists\"
    else
        mkdir -p ${path}
    fi
    cp ${workingPath}/${module}${srcPath} ${path}
    """
}

def cleanDockerImage(String dockerImgName) {
    sh "docker rmi ${dockerImgName}"
}

def backupDockerImage(String dockerImgName) {

    echo "-----> Backup artifact(docker image : ${dockerImgName}) ..."
    path = "${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}"
    sh """
    docker save ${dockerImgName} -o ${backup_targetFile}
    if [ -d ${path} ];
    then
        echo \"${path} already exists\"
    else
        mkdir -p ${path}
        mv ${backup_targetFile} ${path}
    fi
    """
}

def backupK8sConfigs(String[] files2backup) {

    path = "${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}/k8s"
    sh """if [ -d ${path} ];
    then
        echo \"${path} already exists!\"
    else
        mkdir -p ${path}
    fi
    """

    for(file in files2backup) {
        sh "cp -r ${file} ${path}"
    }
}

def cleanJar(String path='projdir') {
    echo "-----> Cleaning project package ..."
    dir(path){
        sh '$MAVEN_HOME/bin/mvn clean'
    }
    return path
}

def generateDockerfile(Object SERVICE, String pathOfDockerfile='projdir') {
    
    def module = SERVICE.module == null ? '' : (SERVICE.module[-1] == '/' ? SERVICE.module : SERVICE.module + '/')
    echo "-----> Generating Dockerfile: ${pathOfDockerfile}/Dockerfile ..."
    sh """cat > ${pathOfDockerfile}/Dockerfile<<EOF
FROM openjdk:17-alpine as final
ADD ${module}target/${SERVICE.jar} /app/target/${SERVICE.jar}
EOF
"""
    return pathOfDockerfile
}

def buildDockerImage(Object SERVICE, Object DOCKER, String base_branch, projDir='projdir', String filter2Remove=null) {

    def imageUri = generateFullImageUri(DOCKER, base_branch, BUILD_NUMBER)
    echo "-----> Building docker image: ${imageUri} ..."
    dir(projDir) {
        docker.withRegistry("http://${DOCKER.registry}", "${DOCKER.push_credentialId}") {
            def imageName = generateImageName(DOCKER, base_branch, BUILD_NUMBER)
            def image = docker.build(imageName)
            image.push()
            sh "docker rmi ${imageName}"
            if(filter2Remove != null)
                sh "docker image prune --filter label=${filter2Remove} -f"
        }
    }
    return imageUri
}

def restoreDockerImage(Object DOCKER, String workspace, String base_branch, String rollbackBuildNumber) {

    dir(workspace) {

        imageUri = generateFullImageUri(DOCKER, base_branch, rollbackBuildNumber)
        sh "docker load -i ${backup_targetFile}"
        docker.withRegistry("http://${DOCKER.registry}", "${DOCKER.push_credentialId}") {
            def image = docker.image(imageUri)
            image.push()
        }
    }
    return imageUri
}

// 灰度发布: 部署Canary版本pod
def deployCanaryWithConfigmapEnvOnly(SERVICE, K3S, String base_branch, String[] args, namespace='default') {

        print "loading ${sharedLibsPath}/k3sUtil.groovy ..."
        def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"
        def version = 'canary'

        def configmap_env_name = k3sUtils.applyCanaryConfigMapEnv(SERVICE, K3S, version, "./configmap-env.ini", namespace)
        print "get configmap_env_name: ${configmap_env_name}"
        
        def java_args = '['
            args.each{item -> java_args += "\"$item\", "}
        java_args = java_args[0..-2] + ']'

        def kvs = [:]
        kvs.put('<namespace>', namespace)
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<gray_version>', version)
        kvs.put('<release>', base_branch)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        def canaryDeploy = k3sUtils.generateCanaryDeployment(kvs, version, "./deployment.yaml")
        k3sUtils.applyDeployment(K3S, canaryDeploy)

        if (!fileExists('./service.yaml')) {
            print "skip applying service. './service.yaml' file does not exists" 
            return
        }
        def canaryService = k3sUtils.generateCanaryService(kvs, version, './service.yaml')
        k3sUtils.applyService(K3S, canaryService)
}

def deployWithConfigmapEnvOnly(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
    def java_args = '['
        args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    k3sUtils.generateDeployment(kvs, 'deployment.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment.yaml')

    k3sUtils.generateService(kvs, 'service.yaml')
    k3sUtils.applyService(K3S, 'service.yaml')
}

def deployWithConfigmapEnvOnly_rollback(SERVICE, K3S, String rollbackBuildNumber, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {

        def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        kvs = [:]
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        k3sUtils.generateDeployment(kvs, 'deployment.yaml')
        k3sUtils.applyDeployment(K3S, 'deployment.yaml')

        k3sUtils.generateService(kvs, 'service.yaml')
        k3sUtils.applyService(K3S, 'service.yaml')
    }
}

def deployWithConfigmapEnvOnly2nd(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
    def java_args = '['
        args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    k3sUtils.generateDeployment(kvs, 'deployment2.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment2.yaml')

    k3sUtils.generateService(kvs, 'service2.yaml')
    k3sUtils.applyService(K3S, 'service2.yaml')
}

def deployWithConfigmapEnvOnly2nd_rollback(SERVICE, K3S, String rollbackBuildNumber, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {

        def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        kvs = [:]
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        k3sUtils.generateDeployment(kvs, 'deployment2.yaml')
        k3sUtils.applyDeployment(K3S, 'deployment2.yaml')

        k3sUtils.generateService(kvs, 'service2.yaml')
        k3sUtils.applyService(K3S, 'service2.yaml')
    }
}

def deployWithConfigmaps_rollback(SERVICE, K3S, String rollbackBuildNumber, String[] args, configmapEnv="./configmap-env.ini", configmapConf="./configmap") {
    
    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {
        def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv)
        def configmap_conf_name = k3sUtils.applyConfigMapConfig(SERVICE, K3S, configmapConf)
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        kvs = [:]
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        kvs.put('<configmap_conf_name>', configmap_conf_name)
        k3sUtils.generateDeployment(kvs, 'deployment.yaml')
        k3sUtils.applyDeployment(K3S, 'deployment.yaml')

        k3sUtils.generateService(kvs, 'service.yaml')
        k3sUtils.applyService(K3S, 'service.yaml')
    }
}

// 灰度发布: 部署Canary版本pod
def deployCanaryWithConfigmaps(SERVICE, K3S, String base_branch, String[] args, namespace = "default") {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"
    def version = 'canary'

    def configmap_env_name = k3sUtils.applyCanaryConfigMapEnv(SERVICE, K3S, version, "./configmap-env.ini", namespace)
    def configmap_conf_name = k3sUtils.applyCanaryConfigMapConfig(SERVICE, K3S, version, "./configmap")
    def java_args = '['
        args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    def kvs = [:]
    kvs.put('<namespace>', namespace)
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<gray_version>', version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    kvs.put('<configmap_conf_name>', configmap_conf_name)

    def canaryDeploy = k3sUtils.generateCanaryDeployment(kvs, version, "./deployment.yaml")
    print 'Generated ${canaryDeploy}. skip applying deployment.yaml'
    k3sUtils.applyDeployment(K3S, canaryDeploy)

    if (!fileExists('./service.yaml')) {
        print "skip applying service. './service.yaml' file does not exists" 
        return
    }
    def canaryService = k3sUtils.generateCanaryService(kvs, version, './service.yaml')
    k3sUtils.applyService(K3S, canaryService)
}

def deployWithConfigmaps(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", configmapConf="./configmap") {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv)
    def configmap_conf_name = k3sUtils.applyConfigMapConfig(SERVICE, K3S, configmapConf)
    def java_args = '['
        args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    kvs.put('<configmap_conf_name>', configmap_conf_name)
    k3sUtils.generateDeployment(kvs, 'deployment.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment.yaml')

    k3sUtils.generateService(kvs, 'service.yaml')
    k3sUtils.applyService(K3S, 'service.yaml')
}

def deployWithConfigmaps2nd(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", configmapConf="./configmap") {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv)
    def configmap_conf_name = k3sUtils.applyConfigMapConfig(SERVICE, K3S, configmapConf)
    def java_args = '['
    args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    kvs.put('<configmap_conf_name>', configmap_conf_name)
    k3sUtils.generateDeployment(kvs, 'deployment2.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment2.yaml')

    k3sUtils.generateService(kvs, 'service2.yaml')
    k3sUtils.applyService(K3S, 'service2.yaml')
}

def execute(CONFIG, base_branch, closures=[:]) {                    //配置文件，当前分支，自定义闭包

    stage('source code check out') {
        
        println "loading ${sharedLibsPath}/gitUtil.groovy..."     //打印信息共享库路径，变量路径，存储加载的录像本身地址，脚本名称
        def gitUtils = load "${sharedLibsPath}/gitUtil.groovy"    //load方法加载对象，赋值给gitUtils

        COMMIT_SHA = closures.GITCLONE == null ? gitUtils.clone(CONFIG.git, base_branch) : closures.GITCLONE()  //定义变量，为空则调用git配置中git仓库信息branch当前分支，不空调用自定义闭包
    }

    stage("docker image build") {

        def projDir = 'projdir'                                 //定义变量，项目目录
        if (closures.BUILDJAR != null)                          //检查是否自定义闭包，有就使用，没有就使用默认方法
            projDir = closures.BUILDJAR()
        else
            projDir = buildJar(CONFIG.service, projDir) //default

        if (closures.BACKUP != null)
            closures.BACKUP()

        if (closures.GENERATEDOCKERFILE != null)
            projDir = closures.GENERATEDOCKERFILE()
        else
            projDir = generateDockerfile(CONFIG.service) //default

        if (closures.BUILDIMAGE != null)
            imageUri = closures.BUILDIMAGE()
        else
            imageUri = buildDockerImage(CONFIG.service, CONFIG.docker, base_branch, projDir)

        if (closures.CLEANJAR != null)
            closures.CLEANJAR()
        else
            cleanJar(projDir)
    }
    stage("notify when build") {
        
        //notifywhenbuild(base_branch, CONFIG.service.name)
    }
    stage("apply K8S ConfigMaps && Deployment") {
        
        if (closures.K3SDEPLOY != null) {
            closures.K3SDEPLOY()
            return
        }
    }
}

// 灰度发布: 部署stable版本pod
def deployStableWithConfigmaps(SERVICE, K3S, String BASE_BRANCH, String rollbackBuildNumber, String[] args, namespace='default') {
    
    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"
    def release = 'stable'

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {
        def configmap_env_name = k3sUtils.applyCanaryConfigMapEnv(SERVICE, K3S, release, "./configmap-env.ini", namespace)
        def configmap_conf_name = k3sUtils.applyCanaryConfigMapConfig(SERVICE, K3S, release, "./configmap")
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        def kvs = [:]
        kvs.put('<namespace>', namespace)
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<gray_version>', release)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        kvs.put('<configmap_conf_name>', configmap_conf_name)
        def stableDeploy = k3sUtils.generateCanaryDeployment(kvs, release, './deployment.yaml')
        k3sUtils.applyDeployment(K3S, stableDeploy)

        if (!fileExists('./service.yaml')) {
            print "skip applying service. './service.yaml' file does not exists" 
            return
        }
        def stableService = k3sUtils.generateCanaryService(kvs, release, './service.yaml')
        k3sUtils.applyService(K3S, stableService)
    }
}

// 灰度发布: 部署stable版本pod
def deployStableWithConfigmapEnvOnly(SERVICE, K3S, String base_branch, String[] args, namespace='default') {

        print "loading ${sharedLibsPath}/k3sUtil.groovy ..."
        def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"
        def release = 'stable'

        dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {

            def configmap_env_name = k3sUtils.applyCanaryConfigMapEnv(SERVICE, K3S, release, "./configmap-env.ini", namespace)
            def java_args = '['
                args.each{item -> java_args += "\"$item\", "}
            java_args = java_args[0..-2] + ']'

            def kvs = [:]
            kvs.put('<namespace>', namespace)
            kvs.put('<service_name>', SERVICE.name)
            kvs.put('<service_version>', SERVICE.version)
            kvs.put('<gray_version>', release)
            kvs.put('<release>', base_branch)
            kvs.put('<COMMIT_SHA>', COMMIT_SHA)
            kvs.put('<imagePullSecret>', K3S.pull_secretId)
            kvs.put('<docker_image>', imageUri)
            kvs.put('<java_args>', java_args)
            if (SERVICE.health != null)
                kvs.put('<service_health>', SERVICE.health)
            kvs.put('<configmap_env_name>', configmap_env_name)
            def canaryDeploy = k3sUtils.generateCanaryDeployment(kvs, release, "./deployment.yaml")
            k3sUtils.applyDeployment(K3S, canaryDeploy)

            if (!fileExists(canaryService)) {
                print "skip applying service. './service.yaml' file does not exists" 
                return
            }
            def canaryService = k3sUtils.generateCanaryService(kvs, release, './service.yaml')
            k3sUtils.applyService(K3S, canaryService)
        }
}

// 灰度发布: 升级stable版本
def upgrade_stable(Object CONFIG, String base_branch, String rollbackBuildNumber, closures = [:]) {
    
    // stage('find latest Canary version') {

    //     def basePath = "${backup_destination}/${JOB_NAME}"
    //     def baseDir = new File(basePath)
    //     if (!baseDir.exists() || !baseDir.isDirectory()) {
    //         error "Path does not exist or is not a directory: ${basePath}"
    //         return
    //     }

    //     latestDir = baseDir.listFiles()
    //         ?.findAll { f ->
    //             f.isDirectory() && (f.name ==~ /^\d+$/)
    //         }
    //         ?.max { it.lastModified() }

    //     rollbackBuildNumber = latestDir.name
    // }

    // 2. docker image build
    stage("docker image restore") {

        def envFile = new File("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/env")
        def lines = envFile.readLines()
        COMMIT_SHA = lines[0]
        BASE_BRANCH = lines[1]

        def path = "${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}"
        print("restore docker image from rollbackBuildNo. #${rollbackBuildNumber}")
        // imageUri = restoreDockerImage(CONFIG.docker, path, BASE_BRANCH, rollbackBuildNumber)
        imageUri = generateFullImageUri(CONFIG.docker, BASE_BRANCH, rollbackBuildNumber)
        print "restore docker image: ${imageUri}"
        // cleanDockerImage(imageUri)
    }

    stage("apply Release(Canary:${rollbackBuildNumber}) K8S ConfigMaps && Deployment") {

        if (closures.K3SDEPLOY != null)
            closures.K3SDEPLOY()
    }
    

    stage("Post Execution Script") {

        // print "Clear rollback workspace"
        // 清理rollback工作目录
        // dir("${backup_destination}/${JOB_NAME}") {
        //     sh "rm -rf *@tmp"
        // }
    }
}

// 灰度发布: 升级canary版本
def upgrade(Object CONFIG, String base_branch, String[] k8sBackupFiles, closures=[:]) {
    
    stage('source code check out') {
        
        println "loading ${sharedLibsPath}/gitUtil.groovy..."
        def gitUtils = load "${sharedLibsPath}/gitUtil.groovy"

        COMMIT_SHA = closures.GITCLONE == null ? gitUtils.clone(CONFIG.git, base_branch) : closures.GITCLONE()

        if (closures.POST_GITCLONE != null)
            closures.POST_GITCLONE()
    }

    stage("docker image build") {

        def projDir = 'projdir'
        if (closures.BUILDJAR != null)
            projDir = closures.BUILDJAR()
        else
            projDir = buildJar(CONFIG.service, projDir) //default

        if (closures.GENERATEDOCKERFILE != null)
            projDir = closures.GENERATEDOCKERFILE()
        else
            projDir = generateDockerfile(CONFIG.service) //default

        if (closures.BUILDIMAGE != null)
            imageUri = closures.BUILDIMAGE()
        else
            imageUri = buildDockerImage(CONFIG.service, CONFIG.docker, base_branch, projDir)

        if (closures.CLEANJAR != null)
            closures.CLEANJAR()
        else
            cleanJar(projDir)

        if (k8sBackupFiles != null && k8sBackupFiles.size() > 0) {
            backupDockerImage(imageUri)
            envFile = new File("${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}/env")
            envFile << "${COMMIT_SHA}\n"
            envFile << "${base_branch}\n"
        }

        cleanDockerImage(imageUri)
    }

    stage("apply K8S ConfigMaps && Deployment") {
        
        if (k8sBackupFiles != null && k8sBackupFiles.size() > 0)
            backupK8sConfigs(k8sBackupFiles)

        if (closures.K3SDEPLOY != null) {
            closures.K3SDEPLOY()
            return
        }
    }

    stage("notify after build") {
        if (closures.BUILDINFO != null) {
            def buildInfo = closures.ALLCHANGES()
            if (buildInfo.length() > 800) {
                buildInfo = buildInfo.substring(buildInfo.length() - 800)
            }
            def title = closures.BUILDINFO()+"正在发布"
            def content = "变更："+buildInfo+"\\n"+
                        "分支："+base_branch+"\\n"+
                        "版本："+BUILD_NUMBER+"\\n"
            sendGroupNotification(title, content)
        }
    }

    stage("Post Execution Script") {

        print "Clear older backup version"
        // 清理旧的备份
        dir("${backup_destination}/${JOB_NAME}") {
            sh """
            save_file=`ls -ltr | tail -${BACKUP_MAX} | awk '{print \$NF}'`
            ls | grep -v "\$save_file" | xargs rm -rf
            """
        }
    }
}


def rollback(CONFIG, rollbackBuildNumber, closures=[:]) {
    // 1. restore specified version
    path = "${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}"
    if (fileExists(path)) {

        stage('source code check out') {
            print("souce code check out. Skipped!")
        }


        envFile = new File("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/env")
        def lines = envFile.readLines()
        COMMIT_SHA = lines[0]
        BASE_BRANCH = lines[1]

        // 2. docker image build
        stage("docker image build") {
            print("restore docker image from rollbackNo. #${rollbackBuildNumber}")
            imageUri = restoreDockerImage(CONFIG.docker, path, BASE_BRANCH, rollbackBuildNumber)
            cleanDockerImage(imageUri)
        }

        // 3. k8s configmap && deployment apply
        stage("apply K8S ConfigMaps && Deployment") {
            if (closures.K3SDEPLOY != null)
                closures.K3SDEPLOY()
        }

        stage("Post Execution Script") {

            print "Clear rollback workspace"
            // 清理rollback工作目录
            dir("${backup_destination}/${JOB_NAME}") {
                sh "rm -rf *@tmp"
            }
        }
    }
    
}


def sendGroupNotification(String title, String content) {

    WECHAT_ROBOT_URL = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=92059925-97db-4830-8438-1514a246a143"
    stage('Send Message') {
       
                        // 构建要发送的企业微信消息
                        def message = """
                        {
                            "msgtype": "text",
                            "text": {
                                "content": "【${title}】\n${content}"
                            }
                        }
                        """
                        // 发送消息到企业微信
                        sh """
                        curl -X POST -H 'Content-Type: application/json' -d '${message}' ${WECHAT_ROBOT_URL}
                        """
    }   

}    

        
return this