archive
/
bramw_baserow
mirror of https://gitlab.com/bramw/baserow.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
bramw_baserow/.gitlab/ci_includes/jobs.yml

597 lines
28 KiB
YAML
Raw Permalink Blame History

# ============== "Abstract" ci stages used by real stages =======================
# Builds a dev version of a specific Dockerfile (--target dev) using a previous CI
# image or the latest develop image as a cache to speed up the build. Tags and pushes
# the resulting dev image for later stages in the pipeline to use.
#
# To extend this stage set the DOCKERFILE_PATH and IMAGE_NAME variables.
.build-baserow-image:
image: docker:20.10.18
stage: build
interruptible: true
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to rebuild.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to rebuild.
- tags
services:
- name: $CI_DIND_IMAGE
alias: docker
variables:
DOCKER_BUILDKIT: 1
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
IMAGE_LABELS: >
--label org.opencontainers.image.vendor=$CI_PROJECT_URL
--label org.opencontainers.image.authors=$CI_PROJECT_URL
--label org.opencontainers.image.revision=$CI_COMMIT_SHA
--label org.opencontainers.image.source=$CI_PROJECT_URL
--label org.opencontainers.image.documentation=$CI_PROJECT_URL
--label org.opencontainers.image.licenses=$CI_PROJECT_URL
--label org.opencontainers.image.url=$CI_PROJECT_URL
--label vcs-url=$CI_PROJECT_URL
--label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
--label com.gitlab.ci.email=$GITLAB_USER_EMAIL
--label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
--label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
--label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
--label com.gitlab.ci.cijoburl=$CI_JOB_URL
--label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
--label org.opencontainers.image.ref.name=$CI_IMAGE_REPO:$CI_COMMIT_REF_NAME
script:
- |
echo "$CI_REGISTRY_PASSWORD" | \
docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
if [[ -z "$DOCKERFILE_PATH" ]]; then
echo "Must provide DOCKERFILE_PATH as a job variable" 2>&1
exit 1
fi
if [[ -z "$DEV_IMAGE_NAME" ]]; then
echo "Must provide DEV_IMAGE_NAME as a job variable" 2>&1
exit 1
fi
# Try cache from this branches latest image, if not fall back to the latest
# develop image.
# Ensure we don't go over 128 char docker tag length limit
TRUNCATED_BRANCH_NAME=${CI_COMMIT_REF_NAME:0:100}
# Replace the "/" character with "-" in the branch name.
TRUNCATED_BRANCH_NAME=${TRUNCATED_BRANCH_NAME/\//-}
CI_DEV_LATEST_BRANCH_TAG=$CLEANUP_JOB_CI_TAG_PREFIX$TRUNCATED_BRANCH_NAME
LATEST_BRANCH_CI_IMAGE="$CI_IMAGE_REPO/$DEV_IMAGE_NAME:$CI_DEV_LATEST_BRANCH_TAG"
# ===== 1. Try pull an image we can use to cache the build with =====
CACHE_FROM_BUILD_ARGS=""
EXTRA_BUILD_ARGS=""
if [[ "$TRIGGER_FULL_IMAGE_REBUILD" = "yes" ]]; then
# Set this variable to ensure the image is rebuilt fully and we re-run all
# build steps to ensure we get the latest and most secure base image + the
# the latest upgraded packages. See: https://pythonspeed.com/articles/docker-cache-insecure-images/
echo "Disabling all caching and doing a full image rebuild from scatch to ensure we get all the latest security updates"
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --no-cache --pull"
else
if [[ "$CI_COMMIT_BRANCH" != "$MASTER_BRANCH_NAME" ]]; then
# On non master branches first try cache from the latest ci image built by
# this branch. We don't do this on master changes on master only appear by
# being merged from develop, so by caching master from develop-latest we will
# actually find some layers to use.
if docker pull $LATEST_BRANCH_CI_IMAGE; then
echo "Caching docker build from the latest branch ci image: $LATEST_BRANCH_CI_IMAGE";
CACHE_FROM_BUILD_ARGS="--cache-from $LATEST_BRANCH_CI_IMAGE";
fi
fi
if [[ -z "$CACHE_FROM_BUILD_ARGS" ]]; then
# If that didnt work try the latest ci dev image from develop
DEVELOP_LATEST_BRANCH_CI_IMAGE="$CI_IMAGE_REPO/$DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$DEVELOP_BRANCH_NAME"
if docker pull $DEVELOP_LATEST_BRANCH_CI_IMAGE; then
echo "Caching docker build from the latest develop image: $DEVELOP_LATEST_BRANCH_CI_IMAGE";
CACHE_FROM_BUILD_ARGS="--cache-from $DEVELOP_LATEST_BRANCH_CI_IMAGE";
fi
fi
fi
# This image tag is one that can be used by subsequent build steps, using the
# latest one might introduce race conditions with concurrent pipelines. Instead
# by using a simple name + sha we know we will be getting the right image later on
# and we can easily re-construct this image path also as $CI_COMMIT_SHORT_SHA is
# available in all stages.
CI_IMAGE_PATH=$CI_IMAGE_REPO/$DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
# ===== 2. Build a dev image to be used in subsequent CI stages =====
if [[ -n "$BUILD_FROM_IMAGE" ]]; then
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --build-arg FROM_IMAGE=$BUILD_FROM_IMAGE";
echo "Building from $BUILD_FROM_IMAGE."
fi
docker context create multi-context
docker buildx create --use --name multi --driver docker-container --platform linux/amd64 multi-context
if [[ -n "$BUILD_ARM" || "$CI_COMMIT_BRANCH" == "$BUILD_ARM_ON_BRANCH" ]]; then
which ssh-agent
eval `ssh-agent -s`
echo "${ARM_BUILDER_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "$ARM_BUILDER_SSH_PUBLIC_KEY" >> ~/.ssh/id_rsa.pub
[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
echo "Building multi-arch images for ARM also."
docker buildx create --append --name multi --driver docker-container --platform linux/arm64/v8 "$ARM_TARGET"
DOCKER_BUILD_PLATFORMS=linux/amd64,linux/arm64/v8
else
DOCKER_BUILD_PLATFORMS=linux/amd64
fi
if [[ "$CI_COMMIT_BRANCH" != "$MASTER_BRANCH_NAME" ]]; then
# We don't use latest branch images from master for any caching so no need to
# bother pushing.
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --tag $LATEST_BRANCH_CI_IMAGE";
fi
# * Use `--build-arg BUILDKIT_INLINE_CACHE=1` to ensure this image's intermediate
# layers will be cached so builds caching from this image can use those layers.
# * $CACHE_ARG is a --cache-from if we have an existing image that we can use
# to speed up this build.
# * Target the dev image as we want to run tests and linting in this image.
# * Tag as both the ci image for use in later stages and the latest ci image to
# cache any future ci pipeline runs.
docker buildx build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--platform=$DOCKER_BUILD_PLATFORMS \
$CACHE_FROM_BUILD_ARGS \
$EXTRA_BUILD_ARGS \
$IMAGE_LABELS \
--push \
--target dev \
--tag $CI_IMAGE_PATH \
-f $DOCKERFILE_PATH .;
# Builds a non-dev (no docker build target provided) and fully labelled final image
# and tags and pushes the non-dev and dev images using $TESTED_IMAGE_PREFIX to mark
# them as being successfully tested for the publishing jobs to use.
#
# To extend this stage set the DOCKERFILE_PATH, IMAGE_NAME and optionally the
# DEV_IMAGE_NAME variables. If DEV_IMAGE_NAME is set it will be used to cache the build
# and a tested tag for it will be pushed at the end.
# You can also set EXTRA_BUILD_ARGS to pass any extra args to the docker build call.
.build-final-baserow-image:
image: $CI_UTIL_IMAGE
stage: build-final
interruptible: true
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to rebuild.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to rebuild.
- tags
only:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $BUILD_ALL_IN_ONE == "true"
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
services:
- name: $CI_DIND_IMAGE
alias: docker
variables:
DOCKER_BUILDKIT: 1
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
IMAGE_LABELS: >
--label org.opencontainers.image.vendor=$CI_PROJECT_URL
--label org.opencontainers.image.authors=$CI_PROJECT_URL
--label org.opencontainers.image.revision=$CI_COMMIT_SHA
--label org.opencontainers.image.source=$CI_PROJECT_URL
--label org.opencontainers.image.documentation=$CI_PROJECT_URL
--label org.opencontainers.image.licenses=$CI_PROJECT_URL
--label org.opencontainers.image.url=$CI_PROJECT_URL
--label vcs-url=$CI_PROJECT_URL
--label com.gitlab.ci.user=$CI_SERVER_URL/$GITLAB_USER_LOGIN
--label com.gitlab.ci.email=$GITLAB_USER_EMAIL
--label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
--label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
--label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
--label com.gitlab.ci.cijoburl=$CI_JOB_URL
--label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID
--label org.opencontainers.image.ref.name=$RELEASE_IMAGE_REPO:$CI_COMMIT_REF_NAME
script:
- |
echo "$CI_REGISTRY_PASSWORD" | \
docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
if [[ -z "$DOCKERFILE_PATH" ]]; then
echo "Must provide DOCKERFILE_PATH as a job variable" 2>&1
exit 1
fi
if [[ -z "$IMAGE_NAME" ]]; then
echo "Must provide IMAGE_NAME as a job variable" 2>&1
exit 1
fi
# ===== 1. Setup image metadata labels =====
#Build date for opencontainers
#rfc 3339 date
BUILDDATE="'$(date '+%FT%T%z' | sed -E -n 's/(\+[0-9]{2})([0-9]{2})$/\1:\2/p')'"
IMAGE_LABELS="$IMAGE_LABELS --label org.opencontainers.image.created=$BUILDDATE"
IMAGE_LABELS="$IMAGE_LABELS --label build-date=$BUILDDATE"
# Description for opencontainers
BUILDTITLE="$(echo $CI_PROJECT_TITLE | tr " " "_")_$IMAGE_NAME"
IMAGE_LABELS="$IMAGE_LABELS --label org.opencontainers.image.title=$BUILDTITLE"
IMAGE_LABELS="$IMAGE_LABELS --label org.opencontainers.image.description=$BUILDTITLE"
# ==== 2. Tag, build and push non-dev image ====
CACHE_FROM_BUILD_ARGS=""
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS"
# Cache from the CI dev image to build the non dev image.
if [[ -n "$DEV_IMAGE_NAME" ]]; then
# Always cache from the CI_IMAGE_PATH as even when we are doing a full image
# rebuild it will have been already fully rebuilt once by the first stage so it
# is safe to use as a cache.
CI_IMAGE_PATH=$CI_IMAGE_REPO/$DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
docker pull $CI_IMAGE_PATH
CACHE_FROM_BUILD_ARGS="$CACHE_FROM_BUILD_ARGS --cache-from $CI_IMAGE_PATH";
fi
TRUNCATED_BRANCH_NAME=${CI_COMMIT_REF_NAME:0:100}
TRUNCATED_BRANCH_NAME=${TRUNCATED_BRANCH_NAME/\//-}
NON_DEV_CACHE_IMAGE=$CI_IMAGE_REPO/$IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$TRUNCATED_BRANCH_NAME
TARGET_NON_DEV_IMAGE_PATH=$CI_IMAGE_REPO/$IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TARGET_DEV_IMAGE_PATH=$CI_IMAGE_REPO/$DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
if [[ "$TRIGGER_FULL_IMAGE_REBUILD" != "yes" ]]; then
# Set this variable to ensure the image is rebuilt fully and we re-run all
# build steps to ensure we get the latest and most secure base image + the
# the latest upgraded packages. See: https://pythonspeed.com/articles/docker-cache-insecure-images/
if [[ "$CI_COMMIT_BRANCH" != "$MASTER_BRANCH_NAME" ]]; then
if docker pull $NON_DEV_CACHE_IMAGE ; then
echo "Caching from $NON_DEV_CACHE_IMAGE";
CACHE_FROM_BUILD_ARGS="$CACHE_FROM_BUILD_ARGS --cache-from $NON_DEV_CACHE_IMAGE";
fi
fi
if [[ -z "$CACHE_FROM_BUILD_ARGS" ]]; then
# If that didnt work try the latest ci non-dev image from develop
DEVELOP_LATEST_BRANCH_CI_IMAGE="$CI_IMAGE_REPO/$IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$DEVELOP_BRANCH_NAME"
if docker pull $DEVELOP_LATEST_BRANCH_CI_IMAGE; then
echo "Caching docker build from the latest develop image: $DEVELOP_LATEST_BRANCH_CI_IMAGE";
CACHE_FROM_BUILD_ARGS="$CACHE_FROM_BUILD_ARGS --cache-from $DEVELOP_LATEST_BRANCH_CI_IMAGE";
fi
fi
fi
setup_build_from_image(){
if [[ -z "$1" ]]; then
# If no argument is provided then look for a variable called BUILD_FROM_IMAGE
# and use that.
BUILD_FROM_VARIABLE_NAME="BUILD_FROM_IMAGE"
BUILD_FROM_DOCKER_ARG="FROM_IMAGE"
LABEL_PREFIX="built-from"
else
# If no argument is provided then look for a variable called
# BUILD_FROM_$1_IMAGE and use that.
BUILD_FROM_VARIABLE_NAME="BUILD_FROM_$1_IMAGE"
BUILD_FROM_DOCKER_ARG="FROM_$1_IMAGE"
LOWER_CASE_ARG=$(echo "$1" | tr '[:upper:]' '[:lower:]')
LABEL_PREFIX="built-from-$LOWER_CASE_ARG-"
fi
eval BUILD_FROM_IMAGE=\$${BUILD_FROM_VARIABLE_NAME}
if [[ -n "$BUILD_FROM_IMAGE" ]]; then
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --build-arg $BUILD_FROM_DOCKER_ARG=$BUILD_FROM_IMAGE";
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-image=$BUILD_FROM_IMAGE"
if docker pull "$BUILD_FROM_IMAGE"; then
BUILT_FROM_REVISION=$(docker inspect $BUILD_FROM_IMAGE | jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]')
BUILT_FROM_COMMITURL=$(docker inspect $BUILD_FROM_IMAGE | jq -r '.[0].Config.Labels["com.gitlab.ci.commiturl"]')
BUILT_FROM_CIJOBURL=$(docker inspect $BUILD_FROM_IMAGE | jq -r '.[0].Config.Labels["com.gitlab.ci.cijoburl"]')
BUILT_FROM_MRURL=$(docker inspect $BUILD_FROM_IMAGE | jq -r '.[0].Config.Labels["com.gitlab.ci.mrurl"]')
BUILT_FROM_VCSURL=$(docker inspect $BUILD_FROM_IMAGE | jq -r '.[0].Config.Labels["vcs-url"]')
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-revision=$BUILD_FROM_REVISION"
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-commiturl=$BUILD_FROM_COMMITURL"
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-cijoburl=$BUILD_FROM_CIJOBURL"
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-mrurl=$BUILD_FROM_MRURL"
IMAGE_LABELS="$IMAGE_LABELS --label $LABEL_PREFIX-vcsurl=$BUILD_FROM_VCSURL"
else
echo "Failed to pull build from image $BUILD_FROM_IMAGE, something has gone wrong"
exit 1
fi
fi
}
docker context create multi-context
docker buildx create --use --name multi --driver docker-container --platform linux/amd64 multi-context
if [[ -n "$BUILD_ARM" || "$CI_COMMIT_BRANCH" == "$BUILD_ARM_ON_BRANCH" ]]; then
which ssh-agent
eval `ssh-agent -s`
echo "${ARM_BUILDER_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo "$ARM_BUILDER_SSH_PUBLIC_KEY" >> ~/.ssh/id_rsa.pub
[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
echo "Building multi-arch images for ARM also."
docker buildx create --append --name multi --driver docker-container --platform linux/arm64/v8 "$ARM_TARGET"
DOCKER_BUILD_PLATFORMS=linux/amd64,linux/arm64/v8
else
DOCKER_BUILD_PLATFORMS=linux/amd64
fi
setup_build_from_image ""
setup_build_from_image BACKEND
setup_build_from_image WEBFRONTEND
# Build the normal non-dev image with all the tags and labels.
docker buildx build \
--platform=$DOCKER_BUILD_PLATFORMS \
$CACHE_FROM_BUILD_ARGS \
$EXTRA_BUILD_ARGS \
$FORMATTEDTAGLIST \
$IMAGE_LABELS \
--push \
-t $TARGET_NON_DEV_IMAGE_PATH \
-f $DOCKERFILE_PATH .;
# Build the cache image with layer caching enabled. We don't enable it for the
# image above to reduce its size.
# We don't do this for master as master always caches from develop.
if [[ "$CI_COMMIT_BRANCH" != "$MASTER_BRANCH_NAME" ]]; then
docker buildx build \
--platform=$DOCKER_BUILD_PLATFORMS \
$CACHE_FROM_BUILD_ARGS \
$EXTRA_BUILD_ARGS \
$IMAGE_LABELS \
--push \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-t $NON_DEV_CACHE_IMAGE \
-f $DOCKERFILE_PATH .;
fi
if [[ -n "$DEV_IMAGE_NAME" ]]; then
# Push a tested dev version of the image also.
docker tag $CI_IMAGE_PATH $TARGET_DEV_IMAGE_PATH
docker push $TARGET_DEV_IMAGE_PATH
fi
# A simple docker based test job which does not run for a TAG pipeline and does not
# check out git.
.docker-image-test-stage:
stage: test
image: $CI_UTIL_IMAGE
interruptible: true
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to retest.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to retest.
- tags
services:
- name: $CI_DIND_IMAGE
alias: docker
# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on
# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set
# and so we need to do some extra git work to figure out what branches this commit is
# on.
.skip-if-tag-not-on-branch:
script:
- |
if [[ -n "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
# Query for all the branches that this commit is part of.
curl -s --header "JOB-TOKEN: $CI_JOB_TOKEN" \
"https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$CI_COMMIT_SHA/refs?type=branch" \
-o this_commits_branches.json;
# Extract just the branch names from the json so we can assert it matches.
TAG_BRANCH_NAMES=$(cat this_commits_branches.json | jq -r ".[].name")
NUM_BRANCHES=$(cat this_commits_branches.json | jq length)
# Ensure the commit is only on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches,
# otherwise someone could checkout a master commit as a new branch and tag it to
# cause an image upload.
if [[ "$NUM_BRANCHES" != "1" || "$TAG_BRANCH_NAMES" != "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
echo "Tags should never be applied to non $SKIP_IF_TAG_NOT_ON_BRANCH branches!" 2>&1;
echo "Pipeline is running for tag: $CI_COMMIT_TAG which for a commit that only appears on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches." 2>&1;
echo "Instead this commit appears on $NUM_BRANCHES branches called $TAG_BRANCH_NAMES" 2>&1;
exit 1;
fi
fi
# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not
# for a commit which is the latest on the specified branch name (for example due to
# someone re-running a pipeline for an old commit) this job will be skipped.
.skip-if-not-latest-commit-on-branch:
allow_failure:
# By exiting with this code we can skip this step without failing the build,
# but still fail if something else goes wrong.
exit_codes: 137
script:
- |
if [[ -n "$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH" ]]; then
LATEST_COMMIT_HASH=$(git rev-parse origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH)
HEAD_COMMIT_HASH=$CI_COMMIT_SHA
if [[ "$LATEST_COMMIT_HASH" != "$HEAD_COMMIT_HASH" ]]; then
echo "Pipeline is not running for latest commit on origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH";
echo " which has commit $LATEST_COMMIT_HASH.";
echo "Instead pipeline is running on commit $HEAD_COMMIT_HASH, exitting as configured to do so in this situation...";
exit 137;
fi
fi
# Pushes $SOURCE_IMAGE to $TARGET_IMAGE using the $TARGET_REGISTRY_PASSWORD,
# $TARGET_REGISTRY_USER and $TARGET_REGISTRY credentials.
#
# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on
# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set
# and so we need to do some extra git work to figure out what branches this commit is
# on.
#
# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not
# for a commit which is the latest on the specified branch name (for example due to
# someone re-running a pipeline for an old commit) this job will be skipped.
.publish-baserow-image:
image: $CI_UTIL_IMAGE
stage: publish
services:
- name: $CI_DIND_IMAGE
alias: docker
except:
refs:
- pipelines
variables:
- $ENABLE_RELEASES != "true"
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
allow_failure:
# By exiting with this code we can skip this step without failing the build,
# but still fail if something else goes wrong.
exit_codes: 137
script:
# Import and run the scripts from the jobs above. Separated like this for
# readability and no functional reason.
- !reference [.skip-if-tag-not-on-branch, script]
- !reference [.skip-if-not-latest-commit-on-branch, script]
- |
# If that project registry isn't public then we must login first.
if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
fi
echo "$TARGET_REGISTRY_PASSWORD" | docker login -u "$TARGET_REGISTRY_USER" "$TARGET_REGISTRY" --password-stdin
docker buildx imagetools create -t $TARGET_IMAGE $SOURCE_IMAGE
.skippable-job:
before_script:
- |
CLEAR="\e[0m"
RED="\e[31m"
GREEN="\e[32m"
echo -e "$GREEN =========== JOB SKIPPER =========== $CLEAR"
if [[ -z "$RUN_WHEN_CHANGES_MADE_IN" ]]; then
echo "Must provide RUN_WHEN_CHANGES_MADE_IN as a job variable" 2>&1
exit 1
fi
# These jobs often pull from the projects CI_REGISTRY, but if that project registry isn't public then we must login first.
if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
fi
if [[ -z "$PROJECT_READ_ONLY_API_TOKEN" && "$ENABLE_JOB_SKIPPING" = "true" ]]; then
echo -e "$RED Disabling job skipping as PROJECT_READ_ONLY_API_TOKEN env variable not set. $CLEAR"
ENABLE_JOB_SKIPPING=false
fi
# If we are doing a full rebuild then force run all the linting and tests.
# Always do this on master as we are always doing a full image rebuild and hence
# always need to run the tests.
if [[ "$ENABLE_JOB_SKIPPING" = "true" && "$TRIGGER_FULL_IMAGE_REBUILD" != "yes" ]]; then
exit_with_copied_artifacts_if_successful_job_for_commit(){
COMMIT_HASH=$1
JOB_NAME=$2
URL="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$COMMIT_HASH/statuses?name=$JOB_NAME"
COMMIT_GITLAB_JOBS=$(curl --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" $URL)
if [[ "$COMMIT_GITLAB_JOBS" ]]; then
echo -e "\e[0Ksection_start:`date +%s`:$COMMIT_HASH$JOB_NAME[collapsed=true]\r\e[0KRaw job status download for $JOB_NAME and $COMMIT_HASH"
echo "Got these job statuses: $COMMIT_GITLAB_JOBS"
JOB_ID=$(echo $COMMIT_GITLAB_JOBS| jq "[.[] | select(.status == \"success\")][0].id" || '')
echo -e "\e[0Ksection_end:`date +%s`:$COMMIT_HASH$JOB_NAME\r\e[0K"
# Check if JOB_ID is an integer (POSIX compliant way)
# Check if JOB_ID is an integer using bash magic.
if [ "$JOB_ID" -eq "$JOB_ID" ] 2> /dev/null; then
if [[ -n "$DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP" ]] ; then
exit_code=0
curl --fail --location --output artifacts.zip \
--header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" \
"https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/jobs/$JOB_ID/artifacts" \
|| exit_code=$?;
if [ ${exit_code} -ne 0 ]; then
echo -e "$RED Failed to get artifacts from successful run $JOB_ID $CLEAR"
else
unzip -o artifacts.zip || exit_code=$?
if [ ${exit_code} -ne 0 ]; then
echo -e "$RED Failed to unzip artifacts $CLEAR"
else
# Echo a stdout report if found so gitlab's coverage regex which
# searches stdout to find the overall coverage is correct even for
# skipped jobs.
if [[ -f "reports/stdout.txt" ]]; then
cat reports/stdout.txt;
fi
echo -e "$GREEN Skipping $JOB_NAME as previous successful run for $COMMIT_HASH and it's artifacts were found. $CLEAR"
exit 0;
fi
fi
else
echo -e "$GREEN Skipping $JOB_NAME as previous successful build for $COMMIT_HASH was found. $CLEAR".
exit 0;
fi
else
echo "Failed to find successful run of $JOB_NAME in job statuses from gitlab for commit $COMMIT_HASH."
fi
else
echo -e "$RED Failed to query gitlab for jobs $CLEAR";
fi
}
echo "Checking if we can skip immediately if this commit already has a successful job run..."
exit_with_copied_artifacts_if_successful_job_for_commit $CI_COMMIT_SHA $CI_JOB_NAME
echo "Can't immediately skip as there was no successful previous job for this commit, checking changes..."
CHANGED_FILES=$(git diff --name-only --diff-filter=ADMR @~..@)
found_changes=0
for SEARCH_PATTERN in $RUN_WHEN_CHANGES_MADE_IN; do
if echo $CHANGED_FILES | grep -q $SEARCH_PATTERN; then
echo -e "Found changes matching $GREEN $SEARCH_PATTERN $CLEAR in:"
echo $CHANGED_FILES
echo -e "$GREEN Running job normally without skipping due to the changes. $CLEAR"
found_changes=1
break
fi
done
if [ ${found_changes} -eq 0 ]; then
echo "No git diff changes found matching $RUN_WHEN_CHANGES_MADE_IN."
echo "Checking for previous commits job..."
SECOND_PARENT_COMMIT=$(git rev-list -1 --merges ${CI_COMMIT_SHA}~1..${CI_COMMIT_SHA})
if [[ -z "$SECOND_PARENT_COMMIT" ]] ; then
# If there is no second parent commit then there is only one parent commit
# and so we can safely check to see if that parent commit has a successful
# job run as this commit does not change any relevant files.
PREVIOUS_COMMIT_SHA=$(git rev-parse HEAD~1)
echo "Found single previous commit $PREVIOUS_COMMIT_SHA, checking for job.."
exit_with_copied_artifacts_if_successful_job_for_commit $PREVIOUS_COMMIT_SHA $CI_JOB_NAME
echo -e "$GREEN Running job without skipping as successful run for previous or this commit not found $CLEAR"
else
# There are more than one parent commits meaning we should re-run this job
# as this commit is a merge commit with multiple parents, so we can't safely
# skip this job.
echo -e "$GREEN Running full job as this is a merge commit. $CLEAR"
fi
fi
else
echo -e "$GREEN Force running job regardless of previous runs. $CLEAR"
fi
echo -e "$GREEN ================================ $CLEAR"
Reference in New Issue View Git Blame Copy Permalink