From dfe9fd71023a1cc92b76e540ccc51ebdea94342f Mon Sep 17 00:00:00 2001
From: Nigel Gott <nigel@baserow.io>
Date: Wed, 2 Feb 2022 17:16:02 +0000
Subject: [PATCH] Skip ci jobs where files weren't changed and the
previous/same commit has a successful job run.
---
.gitlab-ci.yml | 55 +++++++++++++-----
.gitlab/ci_includes/jobs.yml | 109 +++++++++++++++++++++++++++++++++++
2 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00b615f04..2a8532026 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,6 +89,9 @@ stages:
- publish
variables:
+ ENABLE_JOB_SKIPPING:
+ value: "true"
+ description: "If set to false then tests and lints will be forced to run and not use previously cached results."
ENABLE_COVERAGE:
value: "true"
description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
@@ -178,24 +181,29 @@ build-backend-image:
DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend lint
+# If pipeline not triggered by tag:
+# - Runs the backend lint if changes to the backend, otherwise skips.
backend-lint:
extends:
- .docker-image-test-stage
+ - .skippable-job
+ variables:
+ RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
script:
- docker run --rm $BACKEND_CI_DEV_IMAGE lint
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend startup check
-# - Generates coverage db's and stores as artifact for later coverage merge and report
+# If pipeline not triggered by tag:
+# - Runs the backend startup check if changes to the backend, otherwise skips.
backend-check-startup:
extends:
- .docker-image-test-stage
+ - .skippable-job
services:
- docker:20.10.12-dind
- name: postgres:11.3
alias: db
+ variables:
+ RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
script:
- DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}')
- ping -w 2 $DB_IP
@@ -208,12 +216,13 @@ backend-check-startup:
--add-host="db:$DB_IP" \
$BACKEND_CI_DEV_IMAGE ci-check-startup;
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend tests (the first 1/3)
+# If pipeline not triggered by tag:
+# - Runs the backend tests (the first 1/3) if changes to the backend, otherwise skips.
# - Generates coverage db's and stores as artifact for later coverage merge and report
backend-test-group-1:
extends:
- .docker-image-test-stage
+ - .skippable-job
services:
- docker:20.10.12-dind
- name: postgres:11.3
@@ -227,6 +236,8 @@ backend-test-group-1:
POSTGRES_PASSWORD: baserow
POSTGRES_DB: baserow
PYTEST_SPLIT_GROUP: 1
+ RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
+ DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
script:
- MJML_IP=$(cat /etc/hosts | awk '{if ($2 == "mjml") print $1;}')
- ping -w 2 $MJML_IP
@@ -244,6 +255,17 @@ backend-test-group-1:
$BACKEND_CI_DEV_IMAGE $TEST_TYPE;
- docker cp baserow_backend_test_container:/baserow/backend/reports .
- docker rm baserow_backend_test_container
+ - |
+ if [[ $PYTEST_SPLIT_GROUP = 1 ]]; then
+ docker run -e DATABASE_USER=baserow \
+ -e DATABASE_NAME=baserow \
+ -e DATABASE_HOST=db \
+ -e DATABASE_PASSWORD=baserow \
+ --rm \
+ --add-host="db:$DB_IP" \
+ --add-host="mjml:$MJML_IP" \
+ $BACKEND_CI_DEV_IMAGE ci-check-startup;
+ fi
artifacts:
paths:
- reports/
@@ -296,6 +318,7 @@ collect-backend-coverage:
reports:
cobertura: coverage.xml
coverage: '/^TOTAL.+?(\d+\%)$/'
+
# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
@@ -316,23 +339,27 @@ build-web-frontend-image:
DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
-# If pipeline not triggered by tag and web-frontend code has changed:
-# - Runs eslint and stylelint
-# - Stores a web-frontend_lint_success file in the cache so future pipelines can skip
-# if no file changes.
+# If pipeline not triggered by tag:
+# - Runs eslint and stylelint if the web-frontend code has changed, otherwise skips.
web-frontend-lint:
extends:
- .docker-image-test-stage
+ - .skippable-job
+ variables:
+ RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
script:
- docker run --rm $WEBFRONTEND_CI_DEV_IMAGE lint
-# If pipeline not triggered by tag and web-frontend code has changed:
-# - Runs the web-frontend tests
+# If pipeline not triggered by tag:
+# - Runs the web-frontend tests if the web-frontend has changed, otherwise skips.
# - Generates coverage and testing reports
-# - Stores the reports in the cache if successful
web-frontend-test:
extends:
- .docker-image-test-stage
+ - .skippable-job
+ variables:
+ RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
+ DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
script:
- mkdir reports/ -p
- TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
diff --git a/.gitlab/ci_includes/jobs.yml b/.gitlab/ci_includes/jobs.yml
index 389ceb2fb..535b08a83 100644
--- a/.gitlab/ci_includes/jobs.yml
+++ b/.gitlab/ci_includes/jobs.yml
@@ -265,3 +265,112 @@
- docker:20.10.12-dind
+.skippable-job:
+ before_script:
+ - |
+ CLEAR="\e[0m"
+ RED="\e[31m"
+ GREEN="\e[32m"
+
+ echo -e "$GREEN =========== JOB SKIPPER =========== $CLEAR"
+ if [[ -z "$RUN_WHEN_CHANGES_MADE_IN" ]]; then
+ echo "Must provide RUN_WHEN_CHANGES_MADE_IN as a job variable" 2>&1
+ exit 1
+ fi
+
+ if [[ "$ENABLE_JOB_SKIPPING" = "true" ]]; then
+
+ exit_with_copied_artifacts_if_successful_job_for_commit(){
+ COMMIT_HASH=$1
+ JOB_NAME=$2
+ URL="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$COMMIT_HASH/statuses?name=$JOB_NAME"
+ COMMIT_GITLAB_JOBS=$(curl --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" $URL)
+
+ if [[ "$COMMIT_GITLAB_JOBS" ]]; then
+ echo -e "\e[0Ksection_start:`date +%s`:$COMMIT_HASH$JOB_NAME[collapsed=true]\r\e[0KRaw job status download for $JOB_NAME and $COMMIT_HASH"
+ echo "Got these job statuses: $COMMIT_GITLAB_JOBS"
+ JOB_ID=$(echo $COMMIT_GITLAB_JOBS| jq "[.[] | select(.status == \"success\")][0].id")
+ echo -e "\e[0Ksection_end:`date +%s`:$COMMIT_HASH$JOB_NAME\r\e[0K"
+ # Check if JOB_ID is an integer (POSIX compliant way)
+
+ # Check if JOB_ID is an integer using bash magic.
+ if [ "$JOB_ID" -eq "$JOB_ID" ] 2> /dev/null; then
+ if [[ -n "$DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP" ]] ; then
+ exit_code=0
+ curl --fail --location --output artifacts.zip \
+ --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" \
+ "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/jobs/$JOB_ID/artifacts" \
+ || exit_code=$?;
+
+ if [ ${exit_code} -ne 0 ]; then
+ echo -e "$RED Failed to get artifacts from successful run $JOB_ID $CLEAR"
+ else
+ unzip artifacts.zip || exit_code=$?
+ if [ ${exit_code} -ne 0 ]; then
+ echo -e "$RED Failed to unzip artifacts $CLEAR"
+ else
+ # Echo a stdout report if found so gitlab's coverage regex which
+ # searches stdout to find the overall coverage is correct even for
+ # skipped jobs.
+ if [[ -f "reports/stdout.txt" ]]; then
+ cat reports/stdout.txt;
+ fi
+ echo -e "$GREEN Skipping $JOB_NAME as previous successful run for $COMMIT_HASH and it's artifacts were found. $CLEAR"
+ exit 0;
+ fi
+ fi
+
+ else
+ echo -e "$GREEN Skipping $JOB_NAME as previous successful build for $COMMIT_HASH was found. $CLEAR".
+ exit 0;
+ fi
+ else
+ echo "Failed to find successful run of $JOB_NAME in job statuses from gitlab for commit $COMMIT_HASH."
+ fi
+ else
+ echo -e "$RED Failed to query gitlab for jobs $CLEAR";
+ fi
+ }
+
+ echo "Checking if we can skip immediately if this commit already has a successful job run..."
+ exit_with_copied_artifacts_if_successful_job_for_commit $CI_COMMIT_SHA $CI_JOB_NAME
+ echo "Can't immediately skip as there was no successful previous job for this commit, checking changes..."
+
+ CHANGED_FILES=$(git diff --name-only --diff-filter=ADMR @~..@)
+ grep_exit_code=0
+ found_changes=0
+ for SEARCH_PATTERN in $RUN_WHEN_CHANGES_MADE_IN; do
+ echo $CHANGED_FILES | grep -q $SEARCH_PATTERN || grep_exit_code=$?;
+ if [ ${grep_exit_code} -eq 0 ]; then
+ echo -e "Found changes matching $GREEN $SEARCH_PATTERN $CLEAR in:"
+ echo $CHANGED_FILES
+ echo -e "$GREEN Running job normally without skipping due to the changes. $CLEAR"
+ found_changes=1
+ break
+ fi
+ done
+
+ if [ ${found_changes} -eq 0 ]; then
+ echo "No git diff changes found matching $RUN_WHEN_CHANGES_MADE_IN."
+ echo "Checking for previous commits job..."
+ SECOND_PARENT_COMMIT=$(git rev-list -1 --merges ${CI_COMMIT_SHA}~1..${CI_COMMIT_SHA})
+ if [[ -z "$SECOND_PARENT_COMMIT" ]] ; then
+ # If there is no second parent commit then there is only one parent commit
+ # and so we can safely check to see if that parent commit has a successful
+ # job run as this commit does not change any relavent files.
+ PREVIOUS_COMMIT_SHA=$(git rev-parse HEAD~1)
+ echo "Found single previous commit $PREVIOUS_COMMIT_SHA, checking for job.."
+ exit_with_copied_artifacts_if_successful_job_for_commit $PREVIOUS_COMMIT_SHA $CI_JOB_NAME
+ echo -e "$GREEN Running job without skipping as successful run for previous or this commit not found $CLEAR"
+ else
+ # There are more than one parent commits meaning we should re-run this job
+ # as this commit is a merge commit with multiple parents, so we can't safely
+ # skip this job.
+ echo -e "$GREEN Running full job as this is a merge commit. $CLEAR"
+ fi
+ fi
+ else
+ echo -e "$GREEN Force running job regardless of previous runs. $CLEAR"
+ fi
+
+ echo -e "$GREEN ================================ $CLEAR"