archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-01-28 18:09:49 +00:00
Code Issues Projects Releases Wiki Activity
bramw_baserow/.gitlab-ci.yml
Joeri Malmberg 56e293d79e Helm Chart: Fixed extraEnv for migration job
2024-12-13 12:48:16 +00:00

1010 lines
35 KiB
YAML
Raw Permalink Blame History

# Please see `docs/development/ci-cd.md` for more information.
include: '/.gitlab/ci_includes/jobs.yml'
stages:
- build
- test
- build-final
- publish
variables:
# Visit https://gitlab.com/baserow/baserow/-/pipelines/new select your branch
# and click run to be able to run a new pipeline where you can manually control these
# variables.
TRIGGER_FULL_IMAGE_REBUILD:
value: "no"
description: "If set to yes then all images will re-pull their base images and rebuild entirely from scratch with no caching."
ENABLE_JOB_SKIPPING:
value: "false"
description: "If set to true then tests and lints will skipped when safe to do so."
ENABLE_COVERAGE:
value: "true"
description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
ENABLE_RELEASES:
value: "false"
description: "If true then on master and develop release images will be pushed automatically"
BUILD_ALL_IN_ONE:
value: "false"
description: "If true then regardless of what branch we are on the all in one image will be built."
BUILD_ALL_IN_ONE_PG_11:
value: "false"
description: "If true then regardless of what branch we are on the all in one image with embedded PostgreSQL version 11 (legacy) will be built."
BUILD_ALL_IN_ONE_PGAUTOUPGRADE:
value: "false"
description: "If true then regardless of what branch we are on the image which auto upgrades PostgreSQL data directory from version 11 to 15 will be built."
# An image repo which is used for storing and passing images between ci pipeline jobs
# and also speeding up ci builds by caching from the latest ci image when building.
CI_IMAGE_REPO: $CI_REGISTRY_IMAGE/ci
# Any images with tags prefixed with the two variables below will be cleaned up automatically
# by our gitlab cleanup job:
# (https://gitlab.com/baserow/baserow/-/settings/packages_and_registries).
#
# ## Note:
# These cleanup tag prefixes are needed as gitlab only supports cleanup by defining
# a regex that matches tags, so we can't do cleanup differently based on image name
# or repo...
#
# IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
CLEANUP_JOB_CI_TAG_PREFIX: ci-latest-
# IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
TESTED_IMAGE_PREFIX: ci-tested-
BACKEND_IMAGE_NAME: backend
BACKEND_DEV_IMAGE_NAME: backend_dev
WEBFRONTEND_IMAGE_NAME: web-frontend
ALLINONE_IMAGE_NAME: baserow
ALLINONE_PG_11_IMAGE_NAME: baserow-pg11
ALLINONE_PGAUTOUPGRADE_IMAGE_NAME: baserow-pgautoupgrade
CLOUDRON_IMAGE_NAME: cloudron
HEROKU_IMAGE_NAME: heroku
WEBFRONTEND_DEV_IMAGE_NAME: web-frontend_dev
BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
# Once images are tested they will publish under these names to ensure that any
# tag only runs of the pipeline can never publish untested images.
TESTED_BACKEND_CI_IMAGE: $CI_IMAGE_REPO/$BACKEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_WEBFRONTEND_CI_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_ALLINONE_CI_IMAGE: $CI_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_ALLINONE_PG_11_CI_IMAGE: $CI_IMAGE_REPO/$ALLINONE_PG_11_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_ALLINONE_PGAUTOUPGRADE_CI_IMAGE: $CI_IMAGE_REPO/$ALLINONE_PGAUTOUPGRADE_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_CLOUDRON_CI_IMAGE: $CI_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
TESTED_HEROKU_CI_IMAGE: $CI_IMAGE_REPO/$HEROKU_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
# Used to tag the latest images on $DEVELOP_BRANCH_NAME
DEVELOP_LATEST_TAG: develop-latest
# Names of important branches used to decide when to run certain jobs.
MASTER_BRANCH_NAME: master
DEVELOP_BRANCH_NAME: develop
# The locations of the various dockerfiles to build.
BACKEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/backend/Dockerfile
WEBFRONTEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/web-frontend/Dockerfile
ALLINONE_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/all-in-one/Dockerfile
ALLINONE_PG_11_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/all-in-one/pg11.Dockerfile
ALLINONE_PGAUTOUPGRADE_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/all-in-one/pgautoupgrade.Dockerfile
CLOUDRON_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/cloudron/Dockerfile
HEROKU_DOCKERFILE_PATH: $CI_PROJECT_DIR/heroku.Dockerfile
# By default, forks can use the baserow projects ci util image so they don't have
# to run the manual job to build their own. If you want to build and use your own
# then set PROJECT_PATH_FOR_CI_UTIL_IMAGE in your gitlab ci settings as a project
# level variable to whichever project you want.
PROJECT_PATH_FOR_CI_UTIL_IMAGE: $CI_PROJECT_PATH
PROJECT_PATH_FOR_E2E_TEST_IMAGE: $CI_PROJECT_PATH
PROJECT_PATH_FOR_CI_DIND_IMAGE: $CI_PROJECT_PATH
# The image path for the helper CI util image that will be built and pushed to.
CI_UTIL_IMAGE: $CI_REGISTRY/$PROJECT_PATH_FOR_CI_UTIL_IMAGE/ci/ci_util_image:latest
# The image path for the E2E testing image that will be built and pushed to.
E2E_TEST_IMAGE: $CI_REGISTRY/$PROJECT_PATH_FOR_E2E_TEST_IMAGE/ci/baserow_e2e-tests:latest
# The image path for the dind CI image that will be built and pushed to.
CI_DIND_IMAGE: $CI_REGISTRY/$PROJECT_PATH_FOR_E2E_TEST_IMAGE/ci/ci_dind_image:latest
# ==================================== CI UTIL ====================================
# A simple util image used by the other jobs containing some helper tools like git, jq,
# coverage etc.
build-ci-util-image:
image: docker:20.10.18
stage: build
services:
- name: $CI_DIND_IMAGE
alias: docker
variables:
DOCKER_BUILDKIT: 1
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- cd .gitlab/ci_util_image
- docker build -t $CI_UTIL_IMAGE .
- docker push $CI_UTIL_IMAGE
# Only trigger this job manually to prevent it running every single time a new branch
# is made. See https://gitlab.com/gitlab-org/gitlab/-/issues/11427
when:
manual
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
only:
changes:
- .gitlab/ci_util_image/*
- .gitlab-ci.yml
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to rebuild.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to rebuild.
- tags
# An image used by the e2e tests.
build-e2e-tests-image:
image: docker:20.10.18
stage: build
services:
- name: $CI_DIND_IMAGE
alias: docker
variables:
DOCKER_BUILDKIT: 1
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- cd e2e-tests
- docker build -t $E2E_TEST_IMAGE .
- docker push $CI_REGISTRY/$PROJECT_PATH_FOR_E2E_TEST_IMAGE/ci/baserow_e2e-tests:latest
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
only:
changes:
- e2e-tests/
- .gitlab/ci_util_image/*
- .gitlab-ci.yml
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to rebuild.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to rebuild.
- tags
# An image used by the e2e tests.
build-ci-dind-image:
image: docker:20.10.18
stage: build
services:
- docker:20.10.18-dind
variables:
DOCKER_BUILDKIT: 1
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- cd .gitlab/ci_dind_image
- docker build -t $CI_DIND_IMAGE .
- docker push $CI_DIND_IMAGE
# Only trigger this job manually to prevent it running every single time a new branch
# is made. See https://gitlab.com/gitlab-org/gitlab/-/issues/11427
when:
manual
# We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
only:
changes:
- .gitlab/ci_dind_image/*
- .gitlab-ci.yml
except:
refs:
# When a pipeline is triggered by an upstream project we don't want to rebuild.
- pipelines
# When a pipeline is triggered by a git commit tag we don't want to rebuild.
- tags
# ==================================== BACKEND ====================================
# If pipeline not triggered by tag :
# - Builds the backend dev image and stores in ci repo for next stages.
build-backend-image:
extends: .build-baserow-image
variables:
DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH
# If pipeline not triggered by tag:
# - Runs the backend lint if changes to the backend, otherwise skips.
backend-lint:
extends:
- .docker-image-test-stage
- .skippable-job
variables:
RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/"
script:
- docker run --rm $BACKEND_CI_DEV_IMAGE lint
needs:
- job: build-backend-image
backend-check-startup-python:
extends:
- .docker-image-test-stage
- .skippable-job
services:
- name: $CI_DIND_IMAGE
alias: docker
- name: postgres:12
alias: db
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
POSTGRES_USER: baserow
POSTGRES_PASSWORD: baserow
POSTGRES_DB: baserow
RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/ .gitlab-ci.yml"
FF_NETWORK_PER_BUILD: 1
script:
- docker run --rm --network=host $BACKEND_CI_DEV_IMAGE ci-check-startup
needs:
- job: build-backend-image
# If pipeline not triggered by tag:
# - Runs the backend tests (the first 1/3) if changes to the backend, otherwise skips.
# - Generates coverage db's and stores as artifact for later coverage merge and report
backend-test-group-1:
extends:
- .docker-image-test-stage
- .skippable-job
services:
- name: $CI_DIND_IMAGE
alias: docker
- name: postgres:12
alias: db
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
POSTGRES_USER: baserow
POSTGRES_PASSWORD: baserow
POSTGRES_DB: baserow
PYTEST_SPLIT_GROUP: 1
RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/ .gitlab-ci.yml"
DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
FF_NETWORK_PER_BUILD: 1
script:
- mkdir -p reports
- TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
- PYTEST_EXTRA_ARGS=$([[ "$RUN_MORNING_TESTS" = "true" ]] && echo '--run-once-per-day-in-ci' || echo "")
- |
docker run \
-e PYTEST_SPLITS=10 \
-e PYTEST_SPLIT_GROUP=$PYTEST_SPLIT_GROUP \
--name=baserow_backend_test_container \
--network=host \
$BACKEND_CI_DEV_IMAGE $TEST_TYPE $PYTEST_EXTRA_ARGS;
- docker cp baserow_backend_test_container:/baserow/backend/reports .
- docker rm baserow_backend_test_container
needs:
- job: build-backend-image
artifacts:
name: "$CI_JOB_NAME-reports"
paths:
- reports/
reports:
junit: reports/report.xml
# Create 2 more separate groups to parallelize pytest by using separate groups to
# decrease overall build time. Pytest xdist doesn't help as the gitlab saas runners only
# have a single virtual core so `pytest -n 2+` slows things down.
backend-test-group-2:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 2
backend-test-group-3:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 3
backend-test-group-4:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 4
backend-test-group-5:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 5
backend-test-group-6:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 6
backend-test-group-7:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 7
backend-test-group-8:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 8
backend-test-group-9:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 9
backend-test-group-10:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 10
# Collects together all the separate backend coverage databases from previous jobs and
# combines them to generate a single report for gitlab to use. GitLab itself does not
# correctly merge these if you just add them all separately into artifacts->reports->
# cobertura.
collect-backend-coverage:
image: $CI_UTIL_IMAGE
stage: build-final
interruptible: true
only:
variables:
- $ENABLE_COVERAGE == "true"
# Prevent rebuilds when tagging as all we want to do is tag and push
except:
refs:
- tags
# Depend on the `reports` artifacts from the previous jobs
needs:
- job: backend-test-group-1
artifacts: true
- job: backend-test-group-2
artifacts: true
- job: backend-test-group-3
artifacts: true
- job: backend-test-group-4
artifacts: true
- job: backend-test-group-5
artifacts: true
script:
- . /baserow/venv/bin/activate
# The reports artifacts will be extracted before the script runs into reports by
# gitlab
- cp reports/.coverage.* .
- export COVERAGE_RCFILE=backend/.coveragerc
- coverage combine
- coverage report
- coverage xml -o coverage.xml
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
coverage: '/^TOTAL.+?(\d+\%)$/'
# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
build-final-backend-image:
extends: .build-final-baserow-image
needs:
- job: backend-test-group-1
- job: backend-test-group-2
- job: backend-test-group-3
- job: backend-test-group-4
- job: backend-test-group-5
- job: backend-test-group-6
- job: backend-test-group-7
- job: backend-test-group-8
- job: backend-test-group-9
- job: backend-lint
variables:
IMAGE_NAME: $BACKEND_IMAGE_NAME
DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH
build-final-backend-image-manual:
extends: .build-final-baserow-image
needs:
- job: build-backend-image
except:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
only:
when:
manual
variables:
IMAGE_NAME: $BACKEND_IMAGE_NAME
DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH
# ==================================== WEB-FRONTEND ====================================
# If pipeline not triggered by tag:
# - Builds the web-frontend dev image and stores in ci repo for next stages.
build-web-frontend-image:
extends: .build-baserow-image
variables:
DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
# If pipeline not triggered by tag:
# - Runs eslint and stylelint if the web-frontend code has changed, otherwise skips.
web-frontend-lint:
extends:
- .docker-image-test-stage
- .skippable-job
needs:
- job: build-web-frontend-image
variables:
RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
script:
- docker run --rm $WEBFRONTEND_CI_DEV_IMAGE lint
# If pipeline not triggered by tag:
# - Runs the web-frontend tests if the web-frontend has changed, otherwise skips.
# - Generates coverage and testing reports
web-frontend-test:
extends:
- .docker-image-test-stage
- .skippable-job
variables:
RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
needs:
- job: build-web-frontend-image
script:
- mkdir reports/ -p
- TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
- |
docker run --name=webfrontend_test $WEBFRONTEND_CI_DEV_IMAGE $TEST_TYPE \
| tee reports/stdout.txt;
- docker cp webfrontend_test:/baserow/reports .
- docker rm webfrontend_test
artifacts:
paths:
- reports/
reports:
junit: reports/junit.xml
coverage_report:
coverage_format: cobertura
path: coverage.xml
coverage: '/Lines\s*:\s*(\d+.?\d*)%/'
zapier-integration-test:
extends:
- .docker-image-test-stage
- .skippable-job
variables:
RUN_WHEN_CHANGES_MADE_IN: "integrations/zapier"
script:
- cd integrations/zapier
- yarn install
- yarn run zapier test
# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
build-final-web-frontend-image:
extends: .build-final-baserow-image
needs:
- job: web-frontend-test
- job: web-frontend-lint
variables:
IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
# Allow to manually build the image from any branch
build-final-web-frontend-image-manual:
extends: .build-final-baserow-image
needs:
- job: build-web-frontend-image
except:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
only:
when:
manual
variables:
IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
# ==================================== E2E TESTS ======================================
e2e-tests:
extends:
- .docker-image-test-stage
script:
- cd e2e-tests
- E2E_TEST_YARN_RUN_CMD=test-ci docker compose up --no-build --exit-code-from e2e-tests
artifacts:
paths:
# Save the HTML report as an artifact for download or visualization
- e2e-tests/playwright-report/
allow_failure: true
# ================================== ALL IN ONES ======================================
# If pipeline not triggered by tag:
# - Build and store the all-in-one image in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
build-all-in-one-image:
extends: .build-final-baserow-image
needs:
- job: build-final-web-frontend-image
- job: build-final-backend-image
tags:
- saas-linux-medium-amd64
only:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $BUILD_ALL_IN_ONE == "true"
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
variables:
IMAGE_NAME: $ALLINONE_IMAGE_NAME
DOCKERFILE_PATH: $ALLINONE_DOCKERFILE_PATH
BUILD_FROM_BACKEND_IMAGE: $TESTED_BACKEND_CI_IMAGE
BUILD_FROM_WEBFRONTEND_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
build-all-in-one-embedded-pg-11-image:
extends: .build-final-baserow-image
needs:
- job: build-all-in-one-image
tags:
- saas-linux-medium-amd64
only:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $BUILD_ALL_IN_ONE_PG_11 == "true"
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
variables:
IMAGE_NAME: $ALLINONE_PG_11_IMAGE_NAME
DOCKERFILE_PATH: $ALLINONE_PG_11_DOCKERFILE_PATH
BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE
build-all-in-one-embedded-pg-15-auto-upgrade-image:
extends: .build-final-baserow-image
needs:
- job: build-all-in-one-image
tags:
- saas-linux-medium-amd64
only:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
- $BUILD_ALL_IN_ONE_PGAUTOUPGRADE == "true"
- $CI_COMMIT_MESSAGE =~ /\[build-all]/
variables:
IMAGE_NAME: $ALLINONE_PGAUTOUPGRADE_IMAGE_NAME
DOCKERFILE_PATH: $ALLINONE_PGAUTOUPGRADE_DOCKERFILE_PATH
BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE
# If pipeline not triggered by tag:
# - Build and store cloudron image in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
build-cloudron-image:
extends: .build-final-baserow-image
needs:
- job: build-all-in-one-image
tags:
- saas-linux-medium-amd64
only:
variables:
- $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME && $CLOUDRON_IMAGE_NAME != "disable"
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME && $CLOUDRON_IMAGE_NAME != "disable"
# This rule and variable allows us to force build cloudron images on branches.
- ($BUILD_ALL_IN_ONE == "true" || $CI_COMMIT_MESSAGE =~ /\[build-all]/ ) && $CLOUDRON_IMAGE_NAME != "disable"
variables:
IMAGE_NAME: $CLOUDRON_IMAGE_NAME
DOCKERFILE_PATH: $CLOUDRON_DOCKERFILE_PATH
BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE
# We build the heroku image to simply test it builds and ensure we can have a smoke
# test in the future.
#
# If pipeline not triggered by tag:
# - Build and store heroku image in CI repo under the `ci-tested` tag so we know
# those images have passed the tests.
build-heroku-image:
extends: .build-final-baserow-image
needs:
- job: build-all-in-one-image
tags:
- saas-linux-medium-amd64
only:
variables:
- ( $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME || $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME ) && ( $BUILD_ALL_IN_ONE == "true" || $CI_COMMIT_MESSAGE =~ /\[build-all]/ ) && $HEROKU_IMAGE_NAME != "disable"
variables:
IMAGE_NAME: $HEROKU_IMAGE_NAME
DOCKERFILE_PATH: $HEROKU_DOCKERFILE_PATH
BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE
# ================================== TRIGGER SAAS =====================================
# Triggers a special pipeline in dependant project and passes various variables to it.
# Only on master and develop.
trigger-saas-build:
stage: publish
inherit:
variables:
- CI_COMMIT_BRANCH
- TESTED_BACKEND_CI_IMAGE
- TESTED_WEBFRONTEND_CI_IMAGE
- CI_COMMIT_SHA
- CI_COMMIT_SHORT_SHA
- DEVELOP_BRANCH_NAME
- MASTER_BRANCH_NAME
- CI_IMAGE_REPO
- BACKEND_IMAGE_NAME
- WEBFRONTEND_IMAGE_NAME
- TESTED_IMAGE_PREFIX
variables:
UPSTREAM_SHA: $CI_COMMIT_SHA
UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA
rules:
- if: $CI_PROJECT_ID == "10858056" && $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
allow_failure: true
trigger:
project: baserow/baserow-saas
branch: $CI_COMMIT_BRANCH
# ================================== PUSHING BACKEND ==================================
# Push baserow/backend:develop-latest
publish-backend-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/backend:$VERSION_GIT_TAG
publish-backend-release-tagged-image:
extends: .publish-baserow-image
only:
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/backend:latest
publish-backend-latest-release-image:
extends: .publish-baserow-image
only:
refs:
- tags
variables:
- $CI_COMMIT_TAG =~ /^[0-9.]+$/
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# ================================ PUSHING WEB-FRONTEND ===============================
# Push baserow/web-frontend:develop-latest
publish-webfrontend-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/web-frontend:$VERSION_GIT_TAG
publish-webfrontend-release-tagged-image:
extends: .publish-baserow-image
only:
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/web-frontend:latest
publish-webfrontend-latest-release-image:
extends: .publish-baserow-image
only:
refs:
- tags
variables:
- $CI_COMMIT_TAG =~ /^[0-9.]+$/
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# ================================ PUSHING ALL-IN-ONE ===============================
# Push baserow/baserow:develop-latest
publish-all-in-one-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow:$VERSION_GIT_TAG
publish-all-in-one-release-tagged-image:
extends: .publish-baserow-image
only:
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow:latest
publish-all-in-one-latest-release-image:
extends: .publish-baserow-image
only:
refs:
- tags
variables:
- $CI_COMMIT_TAG =~ /^[0-9.]+$/
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:latest"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pg11:develop-latest
publish-all-in-one-embedded-pg-11-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PG_11_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PG_11_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pg11:$VERSION_GIT_TAG
publish-all-in-one-embedded-pg-11-release-tagged-image:
extends: .publish-baserow-image
only:
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PG_11_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PG_11_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pg11:latest
publish-all-in-one-embedded-pg-11-latest-release-image:
extends: .publish-baserow-image
only:
refs:
- tags
variables:
- $CI_COMMIT_TAG =~ /^[0-9.]+$/
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PG_11_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PG_11_IMAGE_NAME:latest"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pgautoupgrade:develop-latest
publish-all-in-one-embedded-pg-15-upgrade-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PGAUTOUPGRADE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PGAUTOUPGRADE_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pgautoupgrade:$VERSION_GIT_TAG
publish-all-in-one-embedded-pg-15-auto-upgrade-release-tagged-image:
extends: .publish-baserow-image
only:
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PGAUTOUPGRADE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PGAUTOUPGRADE_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/baserow-pgautoupgrade:latest
publish-all-in-one-embedded-pg-15-auto-upgrade-latest-release-image:
extends: .publish-baserow-image
only:
refs:
- tags
variables:
- $CI_COMMIT_TAG =~ /^[0-9.]+$/
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_ALLINONE_PGAUTOUPGRADE_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_PGAUTOUPGRADE_IMAGE_NAME:latest"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/cloudron:develop-latest
publish-cloudron-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CLOUDRON_IMAGE_NAME != "disable" && $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
dependencies: []
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# Push baserow/cloudron:$VERSION_GIT_TAG
publish-cloudron-release-tagged-image:
extends: .publish-baserow-image
only:
variables:
- $CLOUDRON_IMAGE_NAME != "disable"
refs:
- tags
dependencies: []
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $RELEASE_REGISTRY
TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER
# We don't push any heroku images as Heroku itself will build and use heroku.Dockerfile
# ================================ MISC LINTS ===============================
docker-file-hadolint:
extends:
- .docker-image-test-stage
- .skippable-job
dependencies: []
variables:
RUN_WHEN_CHANGES_MADE_IN: "Dockerfile"
script:
- mkdir -p reports
# Ignore the version pinning as we want security upgrades ASAP in our docker images.
- |
docker run --rm -i -v "$(pwd)":/opt/hadolint/. -w /opt/hadolint \
hadolint/hadolint:2.9.3-debian \
hadolint --ignore DL3008 -f gitlab_codeclimate \
backend/Dockerfile \
web-frontend/Dockerfile \
heroku.Dockerfile \
e2e-tests/Dockerfile \
deploy/*/Dockerfile > reports/hadolint.json
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
when: always
reports:
codequality:
- "reports/*"
paths:
- "reports/*"
mjml-compiled-check:
extends:
- .docker-image-test-stage
- .skippable-job
variables:
RUN_WHEN_CHANGES_MADE_IN: ".eta"
dependencies: []
script:
- cd backend/email_compiler
- yarn install
- yarn run compile
- |
git diff --exit-code || \
(echo "Uncompiled changes found to mjml email templates, run the compiler in backend/email_compiler/ and committed the changes" && exit 1)
publish-helm-chart:
image: dtzar/helm-kubectl:3.15.2
stage: publish
variables:
HELM_REPO_PROJECT_ID: 59591209 # https://gitlab.com/baserow/baserow-chart
only:
refs:
- tags
artifacts:
paths:
- deploy/helm/baserow-*.tgz
script:
- cd deploy/helm
- rm baserow/Chart.lock
- helm repo add bitnami https://charts.bitnami.com/bitnami
- helm repo add caddy https://caddyserver.github.io/ingress
- helm dependency build baserow
- helm package baserow
- ls baserow-*.tgz
- echo "Upload to chart repository"
- CHART_FILE=$(ls baserow-*.tgz)
- |
HELM_CHART_VERSION=$(helm show chart baserow | grep version | tail -n 1 | awk -F ': ' '{print $2}')
- |
curl --request POST \
--form token=$CI_JOB_TOKEN \
--form ref=main \
--form variables[CHART_FILE]="$CHART_FILE" \
--form variables[HELM_CHART_VERSION]="$HELM_CHART_VERSION" \
--form variables[HELM_APP_VERSION]="$CI_COMMIT_TAG" \
--form variables[TRIGGER_ID]="$CI_JOB_ID" \
--url "https://gitlab.com/api/v4/projects/$HELM_REPO_PROJECT_ID/trigger/pipeline"
- echo "Triggered pipeline to upload helm chart in project $HELM_REPO_PROJECT_ID"
Reference in a new issue View git blame Copy permalink