CI Changes, docker image tag changes and manifests

Jenkinsfile

@@ -8,11 +8,14 @@ pipeline {
     IMAGE            = "nginx-proxy-manager"
     BASE_IMAGE       = "jc21/nginx-proxy-manager-base"
     TEMP_IMAGE       = "nginx-proxy-manager-build_${BUILD_NUMBER}"
-    TEMP_IMAGE_ARM   = "nginx-proxy-manager-arm-build_${BUILD_NUMBER}"
-    TEMP_IMAGE_ARM64 = "nginx-proxy-manager-arm64-build_${BUILD_NUMBER}"
     TAG_VERSION      = getPackageVersion()
     MAJOR_VERSION    = "2"
     BRANCH_LOWER     = "${BRANCH_NAME.toLowerCase()}"
+    // Architectures:
+    AMD64_TAG        = "amd64"
+    ARMV6_TAG        = "armv6l"
+    ARMV7_TAG        = "armv7l"
+    ARM64_TAG        = "aarch64"
   }
   stages {
     stage('Build PR') {
@@ -29,19 +32,19 @@ pipeline {
           sh 'docker run --rm -v $(pwd):/data ${DOCKER_CI_TOOLS} node-prune'
 
           // Docker Build
-          sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE} .'
+          sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${AMD64_TAG} .'
 
           // Dockerhub
-          sh 'docker tag ${TEMP_IMAGE} docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}'
+          sh 'docker tag ${TEMP_IMAGE}-${AMD64_TAG} docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}-${AMD64_TAG}'
           withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
             sh "docker login -u '${duser}' -p '${dpass}'"
-            sh 'docker push docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}'
+            sh 'docker push docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}-${AMD64_TAG}'
           }
 
-          sh 'docker rmi ${TEMP_IMAGE}'
+          sh 'docker rmi ${TEMP_IMAGE}-${AMD64_TAG}'
 
           script {
-            def comment = pullRequest.comment("Docker Image for build ${BUILD_NUMBER} is available on [DockerHub](https://cloud.docker.com/repository/docker/jc21/${IMAGE}) as `jc21/${IMAGE}:github-${BRANCH_LOWER}`")
+            def comment = pullRequest.comment("Docker Image for build ${BUILD_NUMBER} is available on [DockerHub](https://cloud.docker.com/repository/docker/jc21/${IMAGE}) as `jc21/${IMAGE}:github-${BRANCH_LOWER}-${AMD64_TAG}`")
           }
         }
       }
@@ -60,23 +63,16 @@ pipeline {
           sh 'docker run --rm -v $(pwd):/data ${DOCKER_CI_TOOLS} node-prune'
 
           // Docker Build
-          sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE} .'
+          sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${AMD64_TAG} .'
 
           // Dockerhub
-          sh 'docker tag ${TEMP_IMAGE} docker.io/jc21/${IMAGE}:develop'
+          sh 'docker tag ${TEMP_IMAGE}-${AMD64_TAG} docker.io/jc21/${IMAGE}:develop-${AMD64_TAG}'
           withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
             sh "docker login -u '${duser}' -p '${dpass}'"
-            sh 'docker push docker.io/jc21/${IMAGE}:develop'
+            sh 'docker push docker.io/jc21/${IMAGE}:develop-${AMD64_TAG}'
           }
 
-          // Private Registry
+          sh 'docker rmi ${TEMP_IMAGE}-${AMD64_TAG}'
-          sh 'docker tag ${TEMP_IMAGE} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:develop'
-          withCredentials([usernamePassword(credentialsId: 'jc21-private-registry', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-            sh "docker login -u '${duser}' -p '${dpass}' ${DOCKER_PRIVATE_REGISTRY}"
-            sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:develop'
-          }
-
-          sh 'docker rmi ${TEMP_IMAGE}'
         }
       }
     }
@@ -85,7 +81,13 @@ pipeline {
         branch 'master'
       }
       parallel {
-        stage('x86_64') {
+        // ========================
+        // amd64
+        // ========================
+        stage('amd64') {
+          agent {
+            label 'amd64'
+          }
           steps {
             ansiColor('xterm') {
               // Codebase
@@ -96,125 +98,268 @@ pipeline {
               sh 'docker run --rm -v $(pwd):/data ${DOCKER_CI_TOOLS} node-prune'
 
               // Docker Build
-              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE} .'
+              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${AMD64_TAG} .'
 
               // Dockerhub
-              sh 'docker tag ${TEMP_IMAGE} docker.io/jc21/${IMAGE}:${TAG_VERSION}'
+              sh 'docker tag ${TEMP_IMAGE}-${AMD64_TAG} docker.io/jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}'
+              sh 'docker tag ${TEMP_IMAGE}-${AMD64_TAG} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE} docker.io/jc21/${IMAGE}:latest'
+              sh 'docker tag ${TEMP_IMAGE}-${AMD64_TAG} docker.io/jc21/${IMAGE}:latest-${AMD64_TAG}'
 
               withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
                 sh "docker login -u '${duser}' -p '${dpass}'"
-                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}'
+                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG}'
-                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}'
+                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG}'
-                sh 'docker push docker.io/jc21/${IMAGE}:latest'
+                sh 'docker push docker.io/jc21/${IMAGE}:latest-${AMD64_TAG}'
               }
 
-              // Private Registry
+              sh 'docker rmi ${TEMP_IMAGE}-${AMD64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}'
-              sh 'docker tag ${TEMP_IMAGE} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}'
-              sh 'docker tag ${TEMP_IMAGE} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest'
-
-              withCredentials([usernamePassword(credentialsId: 'jc21-private-registry', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-                sh "docker login -u '${duser}' -p '${dpass}' ${DOCKER_PRIVATE_REGISTRY}"
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest'
-              }
-
-              sh 'docker rmi ${TEMP_IMAGE}'
             }
           }
         }
-        stage('armhf') {
+        // ========================
+        // aarch64
+        // ========================
+        stage('aarch64') {
           agent {
-            label 'armhf'
+            label 'aarch64'
           }
           steps {
             ansiColor('xterm') {
               // Codebase
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:armhf yarn install'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install'
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:armhf npm run-script build'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} npm run-script build'
-              sh 'rm -rf node_modules'
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:armhf yarn install --prod'
-
-              // Docker Build
-              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE_ARM} -f Dockerfile.armhf .'
-
-              // Dockerhub
-              sh 'docker tag ${TEMP_IMAGE_ARM} docker.io/jc21/${IMAGE}:${TAG_VERSION}-armhf'
-              sh 'docker tag ${TEMP_IMAGE_ARM} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-armhf'
-              sh 'docker tag ${TEMP_IMAGE_ARM} docker.io/jc21/${IMAGE}:latest-armhf'
-
-              withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-                sh "docker login -u '${duser}' -p '${dpass}'"
-                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-armhf'
-                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-armhf'
-                sh 'docker push docker.io/jc21/${IMAGE}:latest-armhf'
-              }
-
-              // Private Registry
-              sh 'docker tag ${TEMP_IMAGE_ARM} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}-armhf'
-              sh 'docker tag ${TEMP_IMAGE_ARM} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}-armhf'
-              sh 'docker tag ${TEMP_IMAGE_ARM} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest-armhf'
-
-              withCredentials([usernamePassword(credentialsId: 'jc21-private-registry', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-                sh "docker login -u '${duser}' -p '${dpass}' ${DOCKER_PRIVATE_REGISTRY}"
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}-armhf'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}-armhf'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest-armhf'
-              }
-
-              sh 'docker rmi ${TEMP_IMAGE_ARM}'
-            }
-          }
-        }
-        stage('arm64') {
-          agent {
-            label 'arm64'
-          }
-          steps {
-            ansiColor('xterm') {
-              // Codebase
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:arm64 yarn install'
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:arm64 npm run-script build'
               sh 'sudo rm -rf node_modules'
-              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE}:arm64 yarn install --prod'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install --prod'
 
               // Docker Build
-              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE_ARM64} -f Dockerfile.arm64 .'
+              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${ARM64_TAG} -f Dockerfile.${ARM64_TAG} .'
 
               // Dockerhub
-              sh 'docker tag ${TEMP_IMAGE_ARM64} docker.io/jc21/${IMAGE}:${TAG_VERSION}-arm64'
+              sh 'docker tag ${TEMP_IMAGE}-${ARM64_TAG} docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE_ARM64} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-arm64'
+              sh 'docker tag ${TEMP_IMAGE}-${ARM64_TAG} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE_ARM64} docker.io/jc21/${IMAGE}:latest-arm64'
+              sh 'docker tag ${TEMP_IMAGE}-${ARM64_TAG} docker.io/jc21/${IMAGE}:latest-${ARM64_TAG}'
 
               withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
                 sh "docker login -u '${duser}' -p '${dpass}'"
-                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-arm64'
+                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG}'
-                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-arm64'
+                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG}'
-                sh 'docker push docker.io/jc21/${IMAGE}:latest-arm64'
+                sh 'docker push docker.io/jc21/${IMAGE}:latest-${ARM64_TAG}'
               }
 
-              // Private Registry
+              sh 'docker rmi ${TEMP_IMAGE}-${ARM64_TAG}'
-              sh 'docker tag ${TEMP_IMAGE_ARM64} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}-arm64'
-              sh 'docker tag ${TEMP_IMAGE_ARM64} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}-arm64'
-              sh 'docker tag ${TEMP_IMAGE_ARM64} ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest-arm64'
-
-              withCredentials([usernamePassword(credentialsId: 'jc21-private-registry', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
-                sh "docker login -u '${duser}' -p '${dpass}' ${DOCKER_PRIVATE_REGISTRY}"
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${TAG_VERSION}-arm64'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:${MAJOR_VERSION}-arm64'
-                sh 'docker push ${DOCKER_PRIVATE_REGISTRY}/${IMAGE}:latest-arm64'
-              }
-
-              sh 'docker rmi ${TEMP_IMAGE_ARM64}'
 
               // Hack to clean up ec2 instance for next build
               sh 'sudo chown -R ec2-user:ec2-user * || echo "Skipping ec2 ownership"'
             }
           }
         }
+        // ========================
+        // armv7l
+        // ========================
+        stage('armv7l') {
+          agent {
+            label 'armv7l'
+          }
+          steps {
+            ansiColor('xterm') {
+              // Codebase
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} npm run-script build'
+              sh 'rm -rf node_modules'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install --prod'
+
+              // Docker Build
+              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${ARMV7_TAG} -f Dockerfile.${ARMV7_TAG} .'
+
+              // Dockerhub
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV7_TAG} docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG}'
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV7_TAG} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG}'
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV7_TAG} docker.io/jc21/${IMAGE}:latest-${ARMV7_TAG}'
+
+              withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
+                sh "docker login -u '${duser}' -p '${dpass}'"
+                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG}'
+                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG}'
+                sh 'docker push docker.io/jc21/${IMAGE}:latest-${ARMV7_TAG}'
+              }
+
+              sh 'docker rmi ${TEMP_IMAGE}-${ARMV7_TAG}'
+            }
+          }
+        }
+        // ========================
+        // armv6l - Disabled for the time being
+        // ========================
+        /*
+        stage('armv6l') {
+          agent {
+            label 'armv6l'
+          }
+          steps {
+            ansiColor('xterm') {
+              // Codebase
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} npm run-script build'
+              sh 'rm -rf node_modules'
+              sh 'docker run --rm -v $(pwd):/app -w /app ${BASE_IMAGE} yarn install --prod'
+
+              // Docker Build
+              sh 'docker build --pull --no-cache --squash --compress -t ${TEMP_IMAGE}-${ARMV6_TAG} -f Dockerfile.${ARMV6_TAG} .'
+
+              // Dockerhub
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV6_TAG} docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARMV6_TAG}'
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV6_TAG} docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV6_TAG}'
+              sh 'docker tag ${TEMP_IMAGE}-${ARMV6_TAG} docker.io/jc21/${IMAGE}:latest-${ARMV6_TAG}'
+
+              withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
+                sh "docker login -u '${duser}' -p '${dpass}'"
+                sh 'docker push docker.io/jc21/${IMAGE}:${TAG_VERSION}-${ARMV6_TAG}'
+                sh 'docker push docker.io/jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV6_TAG}'
+                sh 'docker push docker.io/jc21/${IMAGE}:latest-${ARMV6_TAG}'
+              }
+
+              sh 'docker rmi ${TEMP_IMAGE}-${ARMV6_TAG}'
+            }
+          }
+        }
+        */
+      }
+    }
+    // ========================
+    // latest manifest
+    // ========================
+    stage('Manifest: latest') {
+      when {
+        branch 'master'
+      }
+      steps {
+        ansiColor('xterm') {
+          sh 'docker pull jc21/${IMAGE}:latest-${AMD64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:latest-${ARM64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:latest-${ARMV7_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:latest-${ARMV6_TAG}'
+
+          sh 'docker manifest push --purge jc21/${IMAGE}:latest || :'
+          sh 'docker manifest create jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${AMD64_TAG} jc21/${IMAGE}:latest-${ARM64_TAG} jc21/${IMAGE}:latest-${ARMV7_TAG}'
+
+          sh 'docker manifest annotate jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${AMD64_TAG} --arch ${AMD64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${ARM64_TAG} --arch ${ARM64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${ARMV7_TAG} --arch arm --variant ${ARMV7_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${ARMV6_TAG} --arch arm --variant ${ARMV6_TAG}'
+          sh 'docker manifest push --purge jc21/${IMAGE}:latest'
+        }
+      }
+    }
+    // ========================
+    // major
+    // ========================
+    stage('Manifest: major') {
+      when {
+        branch 'master'
+      }
+      steps {
+        ansiColor('xterm') {
+          sh 'docker pull jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV6_TAG}'
+
+          sh 'docker manifest push --purge jc21/${IMAGE}:${MAJOR_VERSION} || :'
+          sh 'docker manifest create jc21/${IMAGE}:${MAJOR_VERSION} jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG} jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG} jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG}'
+
+          sh 'docker manifest annotate jc21/${IMAGE}:${MAJOR_VERSION} jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG} --arch ${AMD64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:${MAJOR_VERSION} jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG} --arch ${ARM64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:${MAJOR_VERSION} jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG} --arch arm --variant ${ARMV7_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:${MAJOR_VERSION} jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV6_TAG} --arch arm --variant ${ARMV6_TAG}'
+        }
+      }
+    }
+    // ========================
+    // version
+    // ========================
+    stage('Manifest: version') {
+      when {
+        branch 'master'
+      }
+      steps {
+        ansiColor('xterm') {
+          sh 'docker pull jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG}'
+          sh 'docker pull jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:${TAG_VERSION}-${ARMV6_TAG}'
+
+          sh 'docker manifest push --purge jc21/${IMAGE}:${TAG_VERSION} || :'
+          sh 'docker manifest create jc21/${IMAGE}:${TAG_VERSION} jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG} jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG} jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG}'
+
+          sh 'docker manifest annotate jc21/${IMAGE}:${TAG_VERSION} jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG} --arch ${AMD64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:${TAG_VERSION} jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG} --arch ${ARM64_TAG}'
+          sh 'docker manifest annotate jc21/${IMAGE}:${TAG_VERSION} jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG} --arch arm --variant ${ARMV7_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:${TAG_VERSION} jc21/${IMAGE}:${TAG_VERSION}-${ARMV6_TAG} --arch arm --variant ${ARMV6_TAG}'
+        }
+      }
+    }
+    // ========================
+    // version
+    // ========================
+    stage('Manifest: develop') {
+      when {
+        branch 'develop'
+      }
+      steps {
+        ansiColor('xterm') {
+          sh 'docker pull jc21/${IMAGE}:develop-${AMD64_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:develop-${ARM64_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:develop-${ARMV7_TAG}'
+          //sh 'docker pull jc21/${IMAGE}:${TAG_VERSION}-${ARMV6_TAG}'
+
+          sh 'docker manifest push --purge jc21/${IMAGE}:develop || :'
+          sh 'docker manifest create jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${AMD64_TAG}'
+
+          sh 'docker manifest annotate jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${AMD64_TAG} --arch ${AMD64_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${ARM64_TAG} --arch ${ARM64_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${ARMV7_TAG} --arch arm --variant ${ARMV7_TAG}'
+          //sh 'docker manifest annotate jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${ARMV6_TAG} --arch arm --variant ${ARMV6_TAG}'
+        }
+      }
+    }
+    // ========================
+    // master cleanup
+    // ========================
+    parallel {
+      // ========================
+      // amd64
+      // ========================
+      stage('Latest cleanup') {
+        when {
+          branch 'master'
+        }
+        steps {
+          ansiColor('xterm') {
+            sh 'docker rmi jc21/${IMAGE}:latest jc21/${IMAGE}:latest-${AMD64_TAG} jc21/${IMAGE}:latest-${ARM64_TAG} jc21/${IMAGE}:latest-${ARMV7_TAG}'
+            sh 'docker rmi jc21/${IMAGE}:${MAJOR_VERSION}-${AMD64_TAG} jc21/${IMAGE}:${MAJOR_VERSION}-${ARM64_TAG} jc21/${IMAGE}:${MAJOR_VERSION}-${ARMV7_TAG}'
+            sh 'docker rmi jc21/${IMAGE}:${TAG_VERSION}-${AMD64_TAG} jc21/${IMAGE}:${TAG_VERSION}-${ARM64_TAG} jc21/${IMAGE}:${TAG_VERSION}-${ARMV7_TAG}'
+          }
+        }
+      }
+      stage('Develop cleanup') {
+        when {
+          branch 'develop'
+        }
+        steps {
+          ansiColor('xterm') {
+            sh 'docker rmi jc21/${IMAGE}:develop jc21/${IMAGE}:develop-${AMD64_TAG}'
+          }
+        }
+      }
+      stage('PR cleanup') {
+        when {
+          changeRequest()
+        }
+        steps {
+          ansiColor('xterm') {
+            sh 'docker rmi jc21/${IMAGE}:github-${BRANCH_LOWER}-${AMD64_TAG}'
+          }
+        }
       }
     }
   }
@@ -234,4 +379,3 @@ def getPackageVersion() {
   ver = sh(script: 'docker run --rm -v $(pwd):/data ${DOCKER_CI_TOOLS} bash -c "cat /data/package.json|jq -r \'.version\'"', returnStdout: true)
   return ver.trim()
 }
-