Skip ci jobs where files weren't changed and the previous/same commit has a successful job run.

.gitlab-ci.yml

@@ -89,6 +89,9 @@ stages:
   - publish
 
 variables:
+  ENABLE_JOB_SKIPPING:
+    value: "true"
+    description: "If set to false then tests and lints will be forced to run and not use previously cached results."
   ENABLE_COVERAGE:
     value: "true"
     description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
@@ -178,24 +181,29 @@ build-backend-image:
     DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
     DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH
 
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend lint
+# If pipeline not triggered by tag:
+# - Runs the backend lint if changes to the backend, otherwise skips.
 backend-lint:
   extends:
     - .docker-image-test-stage
+    - .skippable-job
+  variables:
+    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
   script:
     - docker run --rm $BACKEND_CI_DEV_IMAGE lint
 
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend startup check
-# - Generates coverage db's and stores as artifact for later coverage merge and report
+# If pipeline not triggered by tag:
+# - Runs the backend startup check if changes to the backend, otherwise skips.
 backend-check-startup:
   extends:
     - .docker-image-test-stage
+    - .skippable-job
   services:
     - docker:20.10.12-dind
     - name: postgres:11.3
       alias: db
+  variables:
+    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
   script:
     - DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}')
     - ping -w 2 $DB_IP
@@ -208,12 +216,13 @@ backend-check-startup:
           --add-host="db:$DB_IP" \
           $BACKEND_CI_DEV_IMAGE ci-check-startup;
 
-# If pipeline not triggered by tag and backend code has changed:
-# - Runs the backend tests (the first 1/3)
+# If pipeline not triggered by tag:
+# - Runs the backend tests (the first 1/3) if changes to the backend, otherwise skips.
 # - Generates coverage db's and stores as artifact for later coverage merge and report
 backend-test-group-1:
   extends:
     - .docker-image-test-stage
+    - .skippable-job
   services:
     - docker:20.10.12-dind
     - name: postgres:11.3
@@ -227,6 +236,8 @@ backend-test-group-1:
     POSTGRES_PASSWORD: baserow
     POSTGRES_DB: baserow
     PYTEST_SPLIT_GROUP: 1
+    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
+    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
   script:
     - MJML_IP=$(cat /etc/hosts | awk '{if ($2 == "mjml") print $1;}')
     - ping -w 2 $MJML_IP
@@ -244,6 +255,17 @@ backend-test-group-1:
         $BACKEND_CI_DEV_IMAGE $TEST_TYPE;
     - docker cp baserow_backend_test_container:/baserow/backend/reports .
     - docker rm baserow_backend_test_container
+    - |
+      if [[ $PYTEST_SPLIT_GROUP = 1 ]]; then
+        docker run -e DATABASE_USER=baserow \
+          -e DATABASE_NAME=baserow \
+          -e DATABASE_HOST=db \
+          -e DATABASE_PASSWORD=baserow \
+          --rm \
+          --add-host="db:$DB_IP" \
+          --add-host="mjml:$MJML_IP" \
+          $BACKEND_CI_DEV_IMAGE ci-check-startup;
+      fi
   artifacts:
     paths:
       - reports/
@@ -296,6 +318,7 @@ collect-backend-coverage:
     reports:
       cobertura: coverage.xml
   coverage: '/^TOTAL.+?(\d+\%)$/'
+
 # If pipeline not triggered by tag:
 # - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
 #   those images have passed the tests.
@@ -316,23 +339,27 @@ build-web-frontend-image:
     DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
     DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
 
-# If pipeline not triggered by tag and web-frontend code has changed:
-# - Runs eslint and stylelint
-# - Stores a web-frontend_lint_success file in the cache so future pipelines can skip
-#   if no file changes.
+# If pipeline not triggered by tag:
+# - Runs eslint and stylelint if the web-frontend code has changed, otherwise skips.
 web-frontend-lint:
   extends:
     - .docker-image-test-stage
+    - .skippable-job
+  variables:
+    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
   script:
     - docker run --rm $WEBFRONTEND_CI_DEV_IMAGE lint
 
-# If pipeline not triggered by tag and web-frontend code has changed:
-# - Runs the web-frontend tests
+# If pipeline not triggered by tag:
+# - Runs the web-frontend tests if the web-frontend has changed, otherwise skips.
 # - Generates coverage and testing reports
-# - Stores the reports in the cache if successful
 web-frontend-test:
   extends:
     - .docker-image-test-stage
+    - .skippable-job
+  variables:
+    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
+    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
   script:
     - mkdir reports/ -p
     - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")

.gitlab/ci_includes/jobs.yml

@@ -265,3 +265,112 @@
     - docker:20.10.12-dind
 
 
+.skippable-job:
+  before_script:
+    - |
+      CLEAR="\e[0m"
+      RED="\e[31m"
+      GREEN="\e[32m"
+
+      echo -e "$GREEN =========== JOB SKIPPER =========== $CLEAR"
+      if [[ -z "$RUN_WHEN_CHANGES_MADE_IN" ]]; then
+          echo "Must provide RUN_WHEN_CHANGES_MADE_IN as a job variable" 2>&1
+          exit 1
+      fi
+
+      if [[ "$ENABLE_JOB_SKIPPING" = "true" ]]; then
+
+        exit_with_copied_artifacts_if_successful_job_for_commit(){
+          COMMIT_HASH=$1
+          JOB_NAME=$2
+          URL="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$COMMIT_HASH/statuses?name=$JOB_NAME"
+          COMMIT_GITLAB_JOBS=$(curl --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" $URL)
+
+          if [[ "$COMMIT_GITLAB_JOBS" ]]; then
+            echo -e "\e[0Ksection_start:`date +%s`:$COMMIT_HASH$JOB_NAME[collapsed=true]\r\e[0KRaw job status download for $JOB_NAME and $COMMIT_HASH"
+            echo "Got these job statuses: $COMMIT_GITLAB_JOBS"
+            JOB_ID=$(echo $COMMIT_GITLAB_JOBS| jq "[.[] | select(.status == \"success\")][0].id")
+            echo -e "\e[0Ksection_end:`date +%s`:$COMMIT_HASH$JOB_NAME\r\e[0K"
+            # Check if JOB_ID is an integer (POSIX compliant way)
+
+            # Check if JOB_ID is an integer using bash magic.
+            if [ "$JOB_ID" -eq "$JOB_ID" ] 2> /dev/null;  then
+              if [[ -n "$DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP" ]] ; then
+                exit_code=0
+                curl --fail --location --output artifacts.zip \
+                  --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" \
+                  "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/jobs/$JOB_ID/artifacts" \
+                  || exit_code=$?;
+
+                if [ ${exit_code} -ne 0 ]; then
+                  echo -e "$RED Failed to get artifacts from successful run $JOB_ID $CLEAR"
+                else
+                  unzip artifacts.zip || exit_code=$?
+                  if [ ${exit_code} -ne 0 ]; then
+                    echo -e "$RED Failed to unzip artifacts $CLEAR"
+                  else
+                    # Echo a stdout report if found so gitlab's coverage regex which
+                    # searches stdout to find the overall coverage is correct even for
+                    # skipped jobs.
+                    if [[ -f "reports/stdout.txt" ]]; then
+                        cat reports/stdout.txt;
+                    fi
+                    echo -e "$GREEN Skipping $JOB_NAME as previous successful run for $COMMIT_HASH and it's artifacts were found. $CLEAR"
+                    exit 0;
+                  fi
+                fi
+
+              else
+                echo -e "$GREEN Skipping $JOB_NAME as previous successful build for $COMMIT_HASH was found. $CLEAR".
+                exit 0;
+              fi
+            else
+              echo "Failed to find successful run of $JOB_NAME in job statuses from gitlab for commit $COMMIT_HASH."
+            fi
+          else
+            echo -e "$RED Failed to query gitlab for jobs $CLEAR";
+          fi
+        }
+
+        echo "Checking if we can skip immediately if this commit already has a successful job run..."
+        exit_with_copied_artifacts_if_successful_job_for_commit $CI_COMMIT_SHA $CI_JOB_NAME
+        echo "Can't immediately skip as there was no successful previous job for this commit, checking changes..."
+
+        CHANGED_FILES=$(git diff --name-only --diff-filter=ADMR @~..@)
+        grep_exit_code=0
+        found_changes=0
+        for SEARCH_PATTERN in $RUN_WHEN_CHANGES_MADE_IN; do
+            echo $CHANGED_FILES | grep -q $SEARCH_PATTERN || grep_exit_code=$?;
+            if [ ${grep_exit_code} -eq 0 ]; then
+              echo -e "Found changes matching $GREEN $SEARCH_PATTERN $CLEAR in:"
+              echo $CHANGED_FILES
+              echo -e "$GREEN Running job normally without skipping due to the changes. $CLEAR"
+              found_changes=1
+              break
+            fi
+        done
+
+        if [ ${found_changes} -eq 0 ]; then
+          echo "No git diff changes found matching $RUN_WHEN_CHANGES_MADE_IN."
+          echo "Checking for previous commits job..."
+          SECOND_PARENT_COMMIT=$(git rev-list -1 --merges ${CI_COMMIT_SHA}~1..${CI_COMMIT_SHA})
+          if [[ -z "$SECOND_PARENT_COMMIT" ]] ; then
+            # If there is no second parent commit then there is only one parent commit
+            # and so we can safely check to see if that parent commit has a successful
+            # job run as this commit does not change any relavent files.
+            PREVIOUS_COMMIT_SHA=$(git rev-parse HEAD~1)
+            echo "Found single previous commit $PREVIOUS_COMMIT_SHA, checking for job.."
+            exit_with_copied_artifacts_if_successful_job_for_commit $PREVIOUS_COMMIT_SHA $CI_JOB_NAME
+            echo -e "$GREEN Running job without skipping as successful run for previous or this commit not found $CLEAR"
+          else
+            # There are more than one parent commits meaning we should re-run this job
+            # as this commit is a merge commit with multiple parents, so we can't safely
+            # skip this job.
+            echo -e "$GREEN Running full job as this is a merge commit. $CLEAR"
+          fi
+        fi
+      else
+          echo -e "$GREEN Force running job regardless of previous runs. $CLEAR"
+      fi
+
+      echo -e "$GREEN ================================ $CLEAR"