deployments/sharedLibs/stages.k8s.python.groovy

// -------------
// python on K8S
sharedLibsPath = "${env.WORKSPACE}/sharedLibs"
backup_destination = '/var/jenkins_home/backups'
backup_targetFile = 'dockerImage.tar'
backup_num = 3

// 获取镜像URL
generateFullImageUri = {Map DOCKER, String base_branch, String BUILD_NUMBER ->
    return "${DOCKER.registry}/${DOCKER.image}:${base_branch}-${BUILD_NUMBER}"
}

// 获取镜像名
generateImageName = {Map DOCKER, String base_branch, String BUILD_NUMBER ->
    return "${DOCKER.image}:${base_branch}-${BUILD_NUMBER}"
}

def cleanDockerImage(String dockerImgName) {
    sh "docker rmi ${dockerImgName}"
}

def backupDockerImage(String dockerImgName) {

    echo "-----> Backup artifact(docker image : ${dockerImgName}) ..."
    path = "${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}"

    sh """
    docker save ${dockerImgName} -o ${backup_targetFile}
    if [ -d ${path} ];
    then
        echo \"${path} already exists\"
    else
        mkdir -p ${path}
    fi
    mv ${backup_targetFile} ${path}
    """
}

def backupK8sConfigs(String[] files2backup) {

    path = "${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}/k8s"
    sh """if [ -d ${path} ];
    then
        echo \"${path} already exists!\"
    else
        mkdir -p ${path}
    fi
    """

    for(file in files2backup) {
        sh "cp -r ${file} ${path}"
    }
}

def generateDockerfile(Object SERVICE, String pathOfDockerfile='projdir') {
    
    def module = SERVICE.module == null ? '' : (SERVICE.module[-1] == '/' ? SERVICE.module : SERVICE.module + '/')
    echo "-----> Generating Dockerfile: ${pathOfDockerfile}/Dockerfile ..."
    sh """cat > ${pathOfDockerfile}/Dockerfile<<EOF
FROM python:3.10-slim as final

COPY ./requirements.txt /app
COPY ./spider.py /app
 
RUN pip install --upgrade pip
RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
RUN pip install --no-cache-dir -r requirements.txt
EOF
"""
    return pathOfDockerfile
}

def buildDockerImage(Object SERVICE, Object DOCKER, String base_branch, projDir='projdir', String filter2Remove=null) {

    def imageUri = generateFullImageUri(DOCKER, base_branch, BUILD_NUMBER)
    echo "-----> Building docker image: ${imageUri} ..."
    dir(projDir) {
        docker.withRegistry("http://${DOCKER.registry}", "${DOCKER.push_credentialId}") {
            def imageName = generateImageName(DOCKER, base_branch, BUILD_NUMBER)
            def image = docker.build(imageName)
            image.push()
            sh "docker rmi ${imageName}"
            if(filter2Remove != null)
                sh "docker image prune --filter label=${filter2Remove} -f"
        }
    }
    return imageUri
}

def restoreDockerImage(Object DOCKER, String workspace, String base_branch, String rollbackBuildNumber) {

    dir(workspace) {

        imageUri = generateFullImageUri(DOCKER, base_branch, rollbackBuildNumber)
        // gunzip ${backup_targetFile}.tar.gz | docker load
        sh "docker load -i ${backup_targetFile}"
        // in case, docker image doesn't exist in remote registry
        docker.withRegistry("http://${DOCKER.registry}", "${DOCKER.push_credentialId}") {
            def image = docker.image(imageUri)
            image.push()
        }
    }
    return imageUri
}

def deployWithConfigmapEnvOnly(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
    def python_args = '['
        args.each{item -> python_args += "\"$item\","}
    python_args = python_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<python_args>', python_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    k3sUtils.generateDeployment(kvs, 'deployment.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment.yaml')

    k3sUtils.generateService(kvs, 'service.yaml')
    k3sUtils.applyService(K3S, 'service.yaml')
}

def deployWithConfigmapEnvOnly_rollback(SERVICE, K3S, String rollbackBuildNumber, String[] args, configmapEnv="./configmap-env.ini", namespace='default') {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {

        def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv, namespace)
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        kvs = [:]
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        k3sUtils.generateDeployment(kvs, 'deployment.yaml')
        k3sUtils.applyDeployment(K3S, 'deployment.yaml')

        k3sUtils.generateService(kvs, 'service.yaml')
        k3sUtils.applyService(K3S, 'service.yaml')
    }
}

def deployWithConfigmaps_rollback(SERVICE, K3S, String rollbackBuildNumber, String[] args, configmapEnv="./configmap-env.ini", configmapConf="./configmap") {
    
    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    dir("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/k8s") {
        def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv)
        def configmap_conf_name = k3sUtils.applyConfigMapConfig(SERVICE, K3S, configmapConf)
        def java_args = '['
            args.each{item -> java_args += "\"$item\","}
        java_args = java_args[0..-2] + ']'

        kvs = [:]
        kvs.put('<service_name>', SERVICE.name)
        kvs.put('<service_version>', SERVICE.version)
        kvs.put('<release>', BASE_BRANCH)
        kvs.put('<COMMIT_SHA>', COMMIT_SHA)
        kvs.put('<imagePullSecret>', K3S.pull_secretId)
        kvs.put('<docker_image>', imageUri)
        kvs.put('<java_args>', java_args)
        if (SERVICE.health != null)
            kvs.put('<service_health>', SERVICE.health)
        kvs.put('<configmap_env_name>', configmap_env_name)
        kvs.put('<configmap_conf_name>', configmap_conf_name)
        k3sUtils.generateDeployment(kvs, 'deployment.yaml')
        k3sUtils.applyDeployment(K3S, 'deployment.yaml')

        k3sUtils.generateService(kvs, 'service.yaml')
        k3sUtils.applyService(K3S, 'service.yaml')
    }
}

def deployWithConfigmaps(SERVICE, K3S, String base_branch, String[] args, configmapEnv="./configmap-env.ini", configmapConf="./configmap") {

    print "loading ${sharedLibsPath}/k3sUtil.groovy..."
    def k3sUtils = load "${sharedLibsPath}/k3sUtil.groovy"

    def configmap_env_name = k3sUtils.applyConfigMapEnv(SERVICE, K3S, configmapEnv)
    def configmap_conf_name = k3sUtils.applyConfigMapConfig(SERVICE, K3S, configmapConf)
    def java_args = '['
        args.each{item -> java_args += "\"$item\","}
    java_args = java_args[0..-2] + ']'

    kvs = [:]
    kvs.put('<service_name>', SERVICE.name)
    kvs.put('<service_version>', SERVICE.version)
    kvs.put('<release>', base_branch)
    kvs.put('<COMMIT_SHA>', COMMIT_SHA)
    kvs.put('<imagePullSecret>', K3S.pull_secretId)
    kvs.put('<docker_image>', imageUri)
    kvs.put('<java_args>', java_args)
    if (SERVICE.health != null)
        kvs.put('<service_health>', SERVICE.health)
    kvs.put('<configmap_env_name>', configmap_env_name)
    kvs.put('<configmap_conf_name>', configmap_conf_name)
    k3sUtils.generateDeployment(kvs, 'deployment.yaml')
    k3sUtils.applyDeployment(K3S, 'deployment.yaml')

    k3sUtils.generateService(kvs, 'service.yaml')
    k3sUtils.applyService(K3S, 'service.yaml')
}

// Jenkins pipeline stages
def upgrade(Object CONFIG, String base_branch, String[] k8sBackupFiles, closures=[:]) {
    
    stage('Source code check out') {
        println "loading ${sharedLibsPath}/gitUtil.groovy..."
        def gitUtils = load "${sharedLibsPath}/gitUtil.groovy"

        COMMIT_SHA = closures.GITCLONE == null ? gitUtils.clone(CONFIG.git, base_branch) : closures.GITCLONE()

        if (closures.POST_GITCLONE != null)
            closures.POST_GITCLONE()
    }

    stage("Docker image build") {
        def projDir = 'projdir'

        if (closures.GENERATEDOCKERFILE != null)
            projDir = closures.GENERATEDOCKERFILE()
        else
            projDir = generateDockerfile(CONFIG.service) //default

        if (closures.BUILDIMAGE != null)
            imageUri = closures.BUILDIMAGE()
        else
            imageUri = buildDockerImage(CONFIG.service, CONFIG.docker, base_branch, projDir)

        // stage("Docker Image backup") {
        if (k8sBackupFiles != null && k8sBackupFiles.size() > 0) {
            backupDockerImage(imageUri)
            envFile = new File("${backup_destination}/${JOB_NAME}/${BUILD_NUMBER}/env")
            envFile << "${COMMIT_SHA}\n"
            envFile << "${base_branch}\n"
        }
        // }

        cleanDockerImage(imageUri)
    }

    stage("Apply K8S ConfigMaps && Deployment") {
        // stage("K8s configs backup") {
        if (k8sBackupFiles != null && k8sBackupFiles.size() > 0)
            backupK8sConfigs(k8sBackupFiles)
        // }

        if (closures.K3SDEPLOY != null) {
            closures.K3SDEPLOY()
            return
        }
    }

    stage("Post Execution Script") {
        print "Clear older backup version"
        // 清理旧的备份
        dir("${backup_destination}/${JOB_NAME}") {
            sh """
            save_file=`ls -ltr | tail -${backup_num} | awk '{print \$NF}'`
            ls | grep -v "\$save_file" | xargs rm -rf
            """
        }
    }
}

def rollback(CONFIG, rollbackBuildNumber, closures=[:]) {
    // 1. restore specified version
    path = "${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}"
    if (fileExists(path)) {

        stage('Source code check out') {
            print("souce code check out. Skipped!")
        }

        envFile = new File("${backup_destination}/${JOB_NAME}/${rollbackBuildNumber}/env")
        def lines = envFile.readLines()
        COMMIT_SHA = lines[0]
        BASE_BRANCH = lines[1]

        // 2. docker image build
        stage("Docker image build") {
            print("restore docker image from rollbackNo. #${rollbackBuildNumber}")
            imageUri = restoreDockerImage(CONFIG.docker, path, BASE_BRANCH, rollbackBuildNumber)
            cleanDockerImage(imageUri)
        }

        // 3. k8s configmap && deployment apply
        stage("Apply K8S ConfigMaps && Deployment") {
            if (closures.K3SDEPLOY != null)
                closures.K3SDEPLOY()
        }

        stage("Post Execution Script") {
            print "Clear rollback workspace"
            // 清理rollback工作目录
            dir("${backup_destination}/${JOB_NAME}") {
                sh "rm -rf *@tmp"
            }
        }
    }
}

return this