Push ci built images to release repo from develop and when tagged on master. Trigger pipeline in saas on dev/master commit. Setup job needs for faster build.

2025-04-04 13:15:24 +00:00 · 2022-02-02 17:20:20 +00:00 · 2022-02-02 17:20:20 +00:00 · 1ac1edf540
commit 1ac1edf540
parent dfe9fd7102
2 changed files with 279 additions and 6 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -139,6 +139,8 @@ variables:

 # ==================================== CI UTIL ====================================

+# A simple util image used by the other jobs containing some helper tools like git, jq,
+# coverage etc.
 build-ci-util-image:
  image: docker:20.10.12
  stage: build
@ -191,6 +193,8 @@ backend-lint:
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/"
  script:
    - docker run --rm $BACKEND_CI_DEV_IMAGE lint
+  needs:
+    - job: build-backend-image

 # If pipeline not triggered by tag:
 # - Runs the backend startup check if changes to the backend, otherwise skips.
@ -198,6 +202,8 @@ backend-check-startup:
  extends:
    - .docker-image-test-stage
    - .skippable-job
+  needs:
+    - job: build-backend-image
  services:
    - docker:20.10.12-dind
    - name: postgres:11.3
@ -266,7 +272,10 @@ backend-test-group-1:
          --add-host="mjml:$MJML_IP" \
          $BACKEND_CI_DEV_IMAGE ci-check-startup;
      fi
+  needs:
+    - job: build-backend-image
  artifacts:
+    name: "$CI_JOB_NAME-reports"
    paths:
      - reports/
    reports:
@ -298,13 +307,16 @@ collect-backend-coverage:
      - $ENABLE_COVERAGE == "true"
  # Prevent rebuilds when tagging as all we want to do is tag and push
  except:
-    variables:
-      - $CI_COMMIT_TAG
+    refs:
+      - tags
  # Depend on the `reports` artifacts from the previous jobs
-  dependencies:
-    - backend-test-group-1
-    - backend-test-group-2
-    - backend-test-group-3
+  needs:
+    - job: backend-test-group-1
+      artifacts: true
+    - job: backend-test-group-2
+      artifacts: true
+    - job: backend-test-group-3
+      artifacts: true
  script:
    - . /baserow/venv/bin/activate
    # The reports artifacts will be extracted before the script runs into reports by
@ -324,6 +336,12 @@ collect-backend-coverage:
 #   those images have passed the tests.
 build-final-backend-image:
  extends: .build-final-baserow-image
+  needs:
+    - job: backend-check-startup
+    - job: backend-test-group-1
+    - job: backend-test-group-2
+    - job: backend-test-group-3
+    - job: backend-lint
  variables:
    IMAGE_NAME: $BACKEND_IMAGE_NAME
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
@ -345,6 +363,8 @@ web-frontend-lint:
  extends:
    - .docker-image-test-stage
    - .skippable-job
+  needs:
+    - job: build-web-frontend-image
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
  script:
@ -360,6 +380,8 @@ web-frontend-test:
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/"
    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
+  needs:
+    - job: build-web-frontend-image
  script:
    - mkdir reports/ -p
    - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
@ -381,7 +403,171 @@ web-frontend-test:
 #   those images have passed the tests.
 build-final-web-frontend-image:
  extends: .build-final-baserow-image
+  needs:
+    - job: web-frontend-test
+    - job: web-frontend-lint
  variables:
    IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
+
+
+# ================================== TRIGGER SAAS =====================================
+
+# Triggers a special pipeline in dependant project and passes various variables to it.
+# Only on master and develop.
+trigger-saas-build:
+  stage: publish
+  inherit:
+    variables:
+      - CI_COMMIT_BRANCH
+      - TESTED_BACKEND_CI_IMAGE
+      - TESTED_WEBFRONTEND_CI_IMAGE
+      - CI_COMMIT_SHA
+      - CI_COMMIT_SHORT_SHA
+      - DEVELOP_BRANCH_NAME
+      - MASTER_BRANCH_NAME
+  variables:
+    UPSTREAM_SHA: $CI_COMMIT_SHA
+    UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA
+    UPSTREAM_TESTED_BACKEND_CI_IMAGE: $TESTED_BACKEND_CI_IMAGE
+    UPSTREAM_TESTED_WEBFRONTEND_CI_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
+  only:
+    changes:
+      - web-frontend/**/*
+      - premium/web-frontend/**/*
+      - backend/**/*
+      - premium/backend/**/*
+    variables:
+      - ($CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME || $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME)
+  allow_failure: true
+  trigger:
+    project: bramw/baserow-saas
+    branch: $CI_COMMIT_BRANCH
+
+# ================================== PUSHING BACKEND ==================================
+
+# Push baserow/backend:develop_latest
+publish-backend-develop-latest-image:
+  extends: .publish-baserow-image
+  only:
+    variables:
+      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+
+# Push baserow/backend_dev:develop_latest
+publish-backend-develop-latest-dev-image:
+  extends: .publish-baserow-image
+  only:
+    variables:
+      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_BACKEND_CI_DEV_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# Push baserow/backend:$VERSION_GIT_TAG
+publish-backend-release-tagged-image:
+  extends: .publish-baserow-image
+  only:
+    refs:
+      - tags
+  dependencies: []
+  variables:
+    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# Push baserow/backend:latest
+publish-backend-latest-release-image:
+  extends: .publish-baserow-image
+  only:
+    refs:
+      - tags
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# ================================ PUSHING WEB-FRONTEND ===============================
+
+# Push baserow/web-frontend:develop_latest
+publish-webfrontend-develop-latest-image:
+  extends: .publish-baserow-image
+  only:
+    variables:
+      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# Push baserow/web-frontend_dev:develop_latest
+publish-webfrontend-develop-latest-dev-image:
+  extends: .publish-baserow-image
+  only:
+    variables:
+      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_DEV_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# Push baserow/web-frontend:$VERSION_GIT_TAG
+publish-webfrontend-release-tagged-image:
+  extends: .publish-baserow-image
+  only:
+    refs:
+      - tags
+  dependencies: []
+  variables:
+    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
+
+# Push baserow/web-frontend:latest
+publish-webfrontend-latest-release-image:
+  extends: .publish-baserow-image
+  only:
+    refs:
+      - tags
+  dependencies: []
+  variables:
+    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
+    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
+    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest"
+    TARGET_REGISTRY: $CI_REGISTRY
+    TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
+    TARGET_REGISTRY_USER: $CI_REGISTRY_USER
--- a/.gitlab/ci_includes/jobs.yml
+++ b/.gitlab/ci_includes/jobs.yml
@ -264,6 +264,93 @@
  services:
    - docker:20.10.12-dind

+# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on
+# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set
+# and so we need to do some extra git work to figure out what branches this commit is
+# on.
+.skip-if-tag-not-on-branch:
+  script:
+    - |
+      if [[ -n "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
+        # Query for all the branches that this commit is part of.
+        curl -s --header "JOB-TOKEN: $CI_JOB_TOKEN" \
+          "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$CI_COMMIT_SHA/refs?type=branch" \
+          -o this_commits_branches.json;
+        # Extract just the branch names from the json so we can assert it matches.
+        TAG_BRANCH_NAMES=$(cat this_commits_branches.json | jq -r ".[].name")
+        NUM_BRANCHES=$(cat this_commits_branches.json | jq length)
+        # Ensure the commit is only on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches,
+        # otherwise someone could checkout a master commit as a new branch and tag it to
+        # cause an image upload.
+        if [[ "$NUM_BRANCHES" != "1" || "$TAG_BRANCH_NAMES" != "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
+          echo "Tags should never be applied to non $SKIP_IF_TAG_NOT_ON_BRANCH branches!" 2>&1;
+          echo "Pipeline is running for tag: $CI_COMMIT_TAG which for a commit that only appears on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches." 2>&1;
+          echo "Instead this commit appears on $NUM_BRANCHES branches called $TAG_BRANCH_NAMES" 2>&1;
+          exit 1;
+        fi
+      fi
+
+# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not
+# for a commit which is the latest on the specified branch name (for example due to
+# someone re-running a pipeline for an old commit) this job will be skipped.
+.skip-if-not-latest-commit-on-branch:
+  allow_failure:
+    # By exiting with this code we can skip this step without failing the build,
+    # but still fail if something else goes wrong.
+    exit_codes: 137
+  script:
+    - |
+      if [[ -n "$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH" ]]; then
+        LATEST_COMMIT_HASH=$(git rev-parse origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH)
+        HEAD_COMMIT_HASH=$CI_COMMIT_SHA
+        if [[ "$LATEST_COMMIT_HASH" != "$HEAD_COMMIT_HASH" ]]; then
+            echo "Pipeline is not running for latest commit on origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH";
+            echo " which has commit $LATEST_COMMIT_HASH.";
+            echo "Instead pipeline is running on commit $HEAD_COMMIT_HASH, exitting as configured to do so in this situation...";
+            exit 137;
+        fi
+      fi
+
+# Pushes $SOURCE_IMAGE to $TARGET_IMAGE using the $TARGET_REGISTRY_PASSWORD,
+# $TARGET_REGISTRY_USER and $TARGET_REGISTRY credentials.
+#
+# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on
+# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set
+# and so we need to do some extra git work to figure out what branches this commit is
+# on.
+#
+# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not
+# for a commit which is the latest on the specified branch name (for example due to
+# someone re-running a pipeline for an old commit) this job will be skipped.
+.publish-baserow-image:
+  image: $CI_UTIL_IMAGE
+  stage: publish
+  services:
+    - docker:20.10.12-dind
+  except:
+    refs:
+      - pipelines
+  variables:
+    DOCKER_HOST: tcp://docker:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+  allow_failure:
+    # By exiting with this code we can skip this step without failing the build,
+    # but still fail if something else goes wrong.
+    exit_codes: 137
+  script:
+    # Import and run the scripts from the jobs above. Separated like this for
+    # readability and no functional reason.
+    - !reference [.skip-if-tag-not-on-branch, script]
+    - !reference [.skip-if-not-latest-commit-on-branch, script]
+    - |
+      echo "$TARGET_REGISTRY_PASSWORD" | docker login -u "$TARGET_REGISTRY_USER" "$TARGET_REGISTRY" --password-stdin
+
+      if ! docker pull $SOURCE_IMAGE; then
+        echo "Could not pull $SOURCE_IMAGE, has the build pipeline finished yet?" 2>&1;
+        exit 1
+      fi
+      docker tag $SOURCE_IMAGE $TARGET_IMAGE
+      docker push $TARGET_IMAGE

 .skippable-job:
  before_script: