diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a8532026..eb3637bb5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,6 +139,8 @@ variables: # ==================================== CI UTIL ==================================== +# A simple util image used by the other jobs containing some helper tools like git, jq, +# coverage etc. build-ci-util-image: image: docker:20.10.12 stage: build @@ -191,6 +193,8 @@ backend-lint: RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/" script: - docker run --rm $BACKEND_CI_DEV_IMAGE lint + needs: + - job: build-backend-image # If pipeline not triggered by tag: # - Runs the backend startup check if changes to the backend, otherwise skips. @@ -198,6 +202,8 @@ backend-check-startup: extends: - .docker-image-test-stage - .skippable-job + needs: + - job: build-backend-image services: - docker:20.10.12-dind - name: postgres:11.3 @@ -266,7 +272,10 @@ backend-test-group-1: --add-host="mjml:$MJML_IP" \ $BACKEND_CI_DEV_IMAGE ci-check-startup; fi + needs: + - job: build-backend-image artifacts: + name: "$CI_JOB_NAME-reports" paths: - reports/ reports: @@ -298,13 +307,16 @@ collect-backend-coverage: - $ENABLE_COVERAGE == "true" # Prevent rebuilds when tagging as all we want to do is tag and push except: - variables: - - $CI_COMMIT_TAG + refs: + - tags # Depend on the `reports` artifacts from the previous jobs - dependencies: - - backend-test-group-1 - - backend-test-group-2 - - backend-test-group-3 + needs: + - job: backend-test-group-1 + artifacts: true + - job: backend-test-group-2 + artifacts: true + - job: backend-test-group-3 + artifacts: true script: - . /baserow/venv/bin/activate # The reports artifacts will be extracted before the script runs into reports by @@ -324,6 +336,12 @@ collect-backend-coverage: # those images have passed the tests. build-final-backend-image: extends: .build-final-baserow-image + needs: + - job: backend-check-startup + - job: backend-test-group-1 + - job: backend-test-group-2 + - job: backend-test-group-3 + - job: backend-lint variables: IMAGE_NAME: $BACKEND_IMAGE_NAME DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME @@ -345,6 +363,8 @@ web-frontend-lint: extends: - .docker-image-test-stage - .skippable-job + needs: + - job: build-web-frontend-image variables: RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/" script: @@ -360,6 +380,8 @@ web-frontend-test: variables: RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/" DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true' + needs: + - job: build-web-frontend-image script: - mkdir reports/ -p - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test") @@ -381,7 +403,171 @@ web-frontend-test: # those images have passed the tests. build-final-web-frontend-image: extends: .build-final-baserow-image + needs: + - job: web-frontend-test + - job: web-frontend-lint variables: IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH + + +# ================================== TRIGGER SAAS ===================================== + +# Triggers a special pipeline in dependant project and passes various variables to it. +# Only on master and develop. +trigger-saas-build: + stage: publish + inherit: + variables: + - CI_COMMIT_BRANCH + - TESTED_BACKEND_CI_IMAGE + - TESTED_WEBFRONTEND_CI_IMAGE + - CI_COMMIT_SHA + - CI_COMMIT_SHORT_SHA + - DEVELOP_BRANCH_NAME + - MASTER_BRANCH_NAME + variables: + UPSTREAM_SHA: $CI_COMMIT_SHA + UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA + UPSTREAM_TESTED_BACKEND_CI_IMAGE: $TESTED_BACKEND_CI_IMAGE + UPSTREAM_TESTED_WEBFRONTEND_CI_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE + only: + changes: + - web-frontend/**/* + - premium/web-frontend/**/* + - backend/**/* + - premium/backend/**/* + variables: + - ($CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME || $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME) + allow_failure: true + trigger: + project: bramw/baserow-saas + branch: $CI_COMMIT_BRANCH + +# ================================== PUSHING BACKEND ================================== + +# Push baserow/backend:develop_latest +publish-backend-develop-latest-image: + extends: .publish-baserow-image + only: + variables: + - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME + SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + + +# Push baserow/backend_dev:develop_latest +publish-backend-develop-latest-dev-image: + extends: .publish-baserow-image + only: + variables: + - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME + SOURCE_IMAGE: $TESTED_BACKEND_CI_DEV_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# Push baserow/backend:$VERSION_GIT_TAG +publish-backend-release-tagged-image: + extends: .publish-baserow-image + only: + refs: + - tags + dependencies: [] + variables: + SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME + SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# Push baserow/backend:latest +publish-backend-latest-release-image: + extends: .publish-baserow-image + only: + refs: + - tags + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME + SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME + SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# ================================ PUSHING WEB-FRONTEND =============================== + +# Push baserow/web-frontend:develop_latest +publish-webfrontend-develop-latest-image: + extends: .publish-baserow-image + only: + variables: + - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME + SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# Push baserow/web-frontend_dev:develop_latest +publish-webfrontend-develop-latest-dev-image: + extends: .publish-baserow-image + only: + variables: + - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME + SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_DEV_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# Push baserow/web-frontend:$VERSION_GIT_TAG +publish-webfrontend-release-tagged-image: + extends: .publish-baserow-image + only: + refs: + - tags + dependencies: [] + variables: + SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME + SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER + +# Push baserow/web-frontend:latest +publish-webfrontend-latest-release-image: + extends: .publish-baserow-image + only: + refs: + - tags + dependencies: [] + variables: + SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME + SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME + SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE + TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest" + TARGET_REGISTRY: $CI_REGISTRY + TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + TARGET_REGISTRY_USER: $CI_REGISTRY_USER diff --git a/.gitlab/ci_includes/jobs.yml b/.gitlab/ci_includes/jobs.yml index 535b08a83..9407f2de1 100644 --- a/.gitlab/ci_includes/jobs.yml +++ b/.gitlab/ci_includes/jobs.yml @@ -264,6 +264,93 @@ services: - docker:20.10.12-dind +# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on +# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set +# and so we need to do some extra git work to figure out what branches this commit is +# on. +.skip-if-tag-not-on-branch: + script: + - | + if [[ -n "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then + # Query for all the branches that this commit is part of. + curl -s --header "JOB-TOKEN: $CI_JOB_TOKEN" \ + "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$CI_COMMIT_SHA/refs?type=branch" \ + -o this_commits_branches.json; + # Extract just the branch names from the json so we can assert it matches. + TAG_BRANCH_NAMES=$(cat this_commits_branches.json | jq -r ".[].name") + NUM_BRANCHES=$(cat this_commits_branches.json | jq length) + # Ensure the commit is only on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches, + # otherwise someone could checkout a master commit as a new branch and tag it to + # cause an image upload. + if [[ "$NUM_BRANCHES" != "1" || "$TAG_BRANCH_NAMES" != "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then + echo "Tags should never be applied to non $SKIP_IF_TAG_NOT_ON_BRANCH branches!" 2>&1; + echo "Pipeline is running for tag: $CI_COMMIT_TAG which for a commit that only appears on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches." 2>&1; + echo "Instead this commit appears on $NUM_BRANCHES branches called $TAG_BRANCH_NAMES" 2>&1; + exit 1; + fi + fi + +# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not +# for a commit which is the latest on the specified branch name (for example due to +# someone re-running a pipeline for an old commit) this job will be skipped. +.skip-if-not-latest-commit-on-branch: + allow_failure: + # By exiting with this code we can skip this step without failing the build, + # but still fail if something else goes wrong. + exit_codes: 137 + script: + - | + if [[ -n "$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH" ]]; then + LATEST_COMMIT_HASH=$(git rev-parse origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH) + HEAD_COMMIT_HASH=$CI_COMMIT_SHA + if [[ "$LATEST_COMMIT_HASH" != "$HEAD_COMMIT_HASH" ]]; then + echo "Pipeline is not running for latest commit on origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH"; + echo " which has commit $LATEST_COMMIT_HASH."; + echo "Instead pipeline is running on commit $HEAD_COMMIT_HASH, exitting as configured to do so in this situation..."; + exit 137; + fi + fi + +# Pushes $SOURCE_IMAGE to $TARGET_IMAGE using the $TARGET_REGISTRY_PASSWORD, +# $TARGET_REGISTRY_USER and $TARGET_REGISTRY credentials. +# +# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on +# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set +# and so we need to do some extra git work to figure out what branches this commit is +# on. +# +# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not +# for a commit which is the latest on the specified branch name (for example due to +# someone re-running a pipeline for an old commit) this job will be skipped. +.publish-baserow-image: + image: $CI_UTIL_IMAGE + stage: publish + services: + - docker:20.10.12-dind + except: + refs: + - pipelines + variables: + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + allow_failure: + # By exiting with this code we can skip this step without failing the build, + # but still fail if something else goes wrong. + exit_codes: 137 + script: + # Import and run the scripts from the jobs above. Separated like this for + # readability and no functional reason. + - !reference [.skip-if-tag-not-on-branch, script] + - !reference [.skip-if-not-latest-commit-on-branch, script] + - | + echo "$TARGET_REGISTRY_PASSWORD" | docker login -u "$TARGET_REGISTRY_USER" "$TARGET_REGISTRY" --password-stdin + + if ! docker pull $SOURCE_IMAGE; then + echo "Could not pull $SOURCE_IMAGE, has the build pipeline finished yet?" 2>&1; + exit 1 + fi + docker tag $SOURCE_IMAGE $TARGET_IMAGE + docker push $TARGET_IMAGE .skippable-job: before_script: