DCD-1253: Add pipelines generator template and convert to the newer CLI-based builder with multi-repo support.

This commit is contained in:
Steve Smith 2021-03-23 15:43:13 +11:00
parent b32ae60c1d
commit 2308c6e5ab
3 changed files with 1958 additions and 949 deletions

File diff suppressed because it is too large Load Diff

164
bitbucket-pipelines.yml.j2 Normal file
View File

@ -0,0 +1,164 @@
---
# NOTE: This file is generated from `bitbucket-pipelines.yml.j2` via `pipelines-generator.py`:
#
# python3 pipelines-generator.py > bitbucket-pipelines.yml
image: atlassian/docker-release-maker:latest
definitions:
services:
docker:
memory: 4096
options:
size: 2x
pipelines:
branches:
######################################################################
# Master branch: Do full release for each image flavour
######################################################################
master:
- step:
name: Check if pipelines config is up-to-date...
script:
- python3 pipelines-generator.py > bitbucket-piplines.yml.expected && diff bitbucket-pipelines.yml bitbucket-piplines.yml.expected
- parallel:
{% for (name, pdata) in images.items() %}
{% for (jdkver, appdata) in pdata.items() %}
{% for offset in range(0, batches) %}
- step:
name: {{ name }} JDK {{ jdkver }} - Batch {{ offset + 1 }}
services:
- docker
script:
- git submodule update --init --recursive
- echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin
- >
python ../docker-release-maker/make-releases.py
--update
--start-version='{{ appdata.start_version }}'
--end-version='{{ appdata.end_version }}'
{% if appdata.default_release %}
--default-release
{% endif %}
--dockerfile-buildargs='BASE_IMAGE={{ appdata.base_image }}'
--dockerfile-version-arg='CONFLUENCE_VERSION'
--mac-product-key='confluence'
--tag-suffixes='{{ tag_suffixes|join(',') }}'
--concurrent-builds='1'
--job-offset='{{ offset }}'
--jobs-total='{{ batches }}'
--docker-repos='{{ appdata.docker_repos|join(',') }}'
--push
{% endfor %}
{% endfor %}
{% endfor %}
- step:
name: Update README
image: python:3.7-alpine3.9
script:
- apk add --no-cache git
- git submodule update --init --recursive
- pip install -q requests
- export DOCKER_REPO='atlassian/confluence-server'
- python shared-components/image/push-readme.py
custom:
######################################################################
# Custom: Do full release for each image flavour
######################################################################
new-releases:
- parallel:
{% for (name, pdata) in images.items() %}
{% for (jdkver, appdata) in pdata.items() %}
{% for offset in range(0, batches) %}
- step:
name: {{ name }} JDK {{ jdkver }} - Batch {{ offset + 1 }}
services:
- docker
script:
- git submodule update --init --recursive
- echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin
- >
python ../docker-release-maker/make-releases.py
--create
--start-version='{{ appdata.start_version }}'
--end-version='{{ appdata.end_version }}'
{% if appdata.default_release %}
--default-release
{% endif %}
--dockerfile-buildargs='BASE_IMAGE={{ appdata.base_image }}'
--dockerfile-version-arg='CONFLUENCE_VERSION'
--mac-product-key='confluence'
--tag-suffixes='{{ tag_suffixes|join(',') }}'
--concurrent-builds='1'
--job-offset='{{ offset }}'
--jobs-total='{{ batches }}'
--docker-repos='{{ appdata.docker_repos|join(',') }}'
--push
{% endfor %}
{% endfor %}
{% endfor %}
custom-release:
- variables:
- name: CONFLUENCE_VERSION
- name: DOCKER_TAG
- step:
services:
- docker
script:
- git submodule update --init --recursive
- echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin
- >
docker build -t atlassian/confluence-server:${DOCKER_TAG}
--build-arg CONFLUENCE_VERSION=${CONFLUENCE_VERSION} .
- snyk auth @SNYK_TOKEN
- snyk container test atlassian/confluence-server:${DOCKER_TAG} --severity-threshold=high
- docker push atlassian/confluence-server:${DOCKER_TAG}
pull-requests:
######################################################################
# All other branches & PRs; run unit tests & functional tests
# against latest app version
######################################################################
'**':
- step:
image: python:3.7-alpine3.9
services:
- docker
script:
- apk add --no-cache git
- git submodule update --init --recursive
- pip install -q -r shared-components/tests/requirements.txt
- export PYTHONPATH=./shared-components/tests:$PYTHONPATH
- export DOCKERFILE='Dockerfile'
- export DOCKERFILE_VERSION_ARG='CONFLUENCE_VERSION'
- export MAC_PRODUCT_KEY='confluence'
- py.test tests/
- py.test shared-components/tests/
- export DOCKERFILE='Dockerfile-alpine'
- py.test tests/
- py.test shared-components/tests/
- step:
services:
- docker
script:
- apk add --no-cache git docker-compose jq curl
- git submodule update --init --recursive
- export CONFLUENCE_VERSION=`curl -s https://marketplace.atlassian.com/rest/2/products/key/confluence/versions/latest | jq -r .name`
- docker build --build-arg CONFLUENCE_VERSION=${CONFLUENCE_VERSION} -t test-image .
- export IS_RELEASE=false
- /usr/src/app/integration_test.sh test-image $IS_RELEASE

55
pipelines-generator.py Executable file
View File

@ -0,0 +1,55 @@
from pathlib import Path
import os
import jinja2 as j2
TEMPLATE_FILE = 'bitbucket-pipelines.yml.j2'
REPOS = ['atlassian/confluence', 'atlassian/confluenec-servert']
images = {
'Confluence Alpine': {
8: {
'start_version': '6',
'end_version': '7',
'default_release': False,
'base_image': 'adoptopenjdk/openjdk8:alpine',
'tag_suffixes': ['alpine','alpine-adoptopenjdk8'],
'dockerfile': 'Dockerfile-alpine',
'docker_repos': REPOS,
},
},
'Confluence Ubuntu': {
8: {
'start_version': '6',
'end_version': '7.1',
'default_release': True,
'base_image': 'adoptopenjdk:8-hotspot',
'tag_suffixes': ['adoptopenjdk8', 'jdk8', 'ubuntu', 'ubuntu-18.04-adoptopenjdk8'],
'dockerfile': 'Dockerfile',
'docker_repos': REPOS,
},
11: {
'start_version': '7.1',
'end_version': '8',
'default_release': True,
'base_image': 'adoptopenjdk:11-hotspot',
'tag_suffixes': ['adoptopenjdk11', 'jdk11', 'ubuntu', 'ubuntu-18.04-adoptopenjdk11'],
'dockerfile': 'Dockerfile',
'docker_repos': REPOS,
}
},
}
def main():
jenv = j2.Environment(
loader=j2.FileSystemLoader('.'),
lstrip_blocks=True,
trim_blocks=True)
template = jenv.get_template(TEMPLATE_FILE)
generated_output = template.render(images=images, batches=12)
print(generated_output)
if __name__ == '__main__':
main()