archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-17 10:22:36 +00:00
Code Issues Projects Releases Wiki Activity

Revert changes to be introduced by future commit

Browse source
This commit is contained in:
Nigel Gott 2022-02-02 15:34:16 +00:00
parent 241958a8ab
commit 426c4791fd
11 changed files with 20 additions and 1710 deletions

4
.dockerignore
View file

@ -2,6 +2,4 @@
**/node_modules **/node_modules
**/.pytest_cache **/.pytest_cache
*.iml *.iml
.idea/ .idea/
web-frontend/reports/
backend/reports/

5
.gitignore vendored
View file

@ -96,8 +96,6 @@ venv/
web-frontend/plugins/ web-frontend/plugins/
backend/plugins/ backend/plugins/
web-frontend/reports/
backend/reports/
.idea/ .idea/
*.iml *.iml
@ -117,6 +115,3 @@ out/
vetur.config.js vetur.config.js
formula/out/ formula/out/
.coverage
junit.xml

341
.gitlab-ci.yml
View file

@ -89,12 +89,6 @@ stages:
- publish - publish
variables: variables:
ENABLE_JOB_SKIPPING:
value: "true"
description: "If set to false then tests and lints will be forced to run and not use previously cached results."
ENABLE_COVERAGE:
value: "true"
description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
# An image repo which is used for storing and passing images between ci pipeline jobs # An image repo which is used for storing and passing images between ci pipeline jobs
# and also speeding up ci builds by caching from the latest ci image when building. # and also speeding up ci builds by caching from the latest ci image when building.
CI_IMAGE_REPO: $CI_REGISTRY_IMAGE/ci CI_IMAGE_REPO: $CI_REGISTRY_IMAGE/ci
@ -179,23 +173,6 @@ backend-lint:
- backend/**/* - backend/**/*
- premium/backend/**/* - premium/backend/**/*
# If pipeline not triggered by tag and backend code has not changed:
# - If there is a previous successful backend lint run in the cache then skip.
# - Otherwise runs backend lint.
no-backend-changes-so-try-skip-lint:
extends:
- .skippable-job
- backend-lint
variables:
SKIP_JOB_NAME: backend-lint
# Override inherited only block, so we can run this job in the
# exact opposite situations.
only: null
except:
changes:
- backend/**/*
- premium/backend/**/*
# If pipeline not triggered by tag and backend code has changed: # If pipeline not triggered by tag and backend code has changed:
# - Runs the backend tests (the first 1/3) # - Runs the backend tests (the first 1/3)
# - Generates coverage db's and stores as artifact for later coverage merge and report # - Generates coverage db's and stores as artifact for later coverage merge and report
@ -214,127 +191,31 @@ backend-test-group-1:
POSTGRES_USER: baserow POSTGRES_USER: baserow
POSTGRES_PASSWORD: baserow POSTGRES_PASSWORD: baserow
POSTGRES_DB: baserow POSTGRES_DB: baserow
PYTEST_SPLIT_GROUP: 1
script: script:
- MJML_IP=$(cat /etc/hosts | awk '{if ($2 == "mjml") print $1;}') - MJML_IP=$(cat /etc/hosts | awk '{if ($2 == "mjml") print $1;}')
- ping -w 2 $MJML_IP - ping -w 2 $MJML_IP
- DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}') - DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}')
- ping -w 2 $DB_IP - ping -w 2 $DB_IP
- mkdir -p reports
- TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
- | - |
docker run \ docker run \
-e PYTEST_SPLITS=3 \
-e PYTEST_SPLIT_GROUP=$PYTEST_SPLIT_GROUP \
--name=baserow_backend_test_container \ --name=baserow_backend_test_container \
--add-host="db:$DB_IP" \ --add-host="db:$DB_IP" \
--add-host="mjml:$MJML_IP" \ --add-host="mjml:$MJML_IP" \
$BACKEND_CI_DEV_IMAGE $TEST_TYPE; $BACKEND_CI_DEV_IMAGE ci-test;
- docker cp baserow_backend_test_container:/baserow/backend/reports .
- docker rm baserow_backend_test_container
- | - |
if [[ $PYTEST_SPLIT_GROUP = 1 ]]; then docker run -e DATABASE_USER=baserow \
docker run -e DATABASE_USER=baserow \ -e DATABASE_NAME=baserow \
-e DATABASE_NAME=baserow \ -e DATABASE_HOST=db \
-e DATABASE_HOST=db \ -e DATABASE_PASSWORD=baserow \
-e DATABASE_PASSWORD=baserow \ --rm \
--rm \ --add-host="db:$DB_IP" \
--add-host="db:$DB_IP" \ --add-host="mjml:$MJML_IP" \
--add-host="mjml:$MJML_IP" \ $BACKEND_CI_DEV_IMAGE ci-check-startup;
$BACKEND_CI_DEV_IMAGE ci-check-startup;
fi
only: only:
# Skip testing on if no change to backend files # Skip testing on if no change to backend files
changes: changes:
- backend/**/* - backend/**/*
- premium/backend/**/* - premium/backend/**/*
artifacts:
paths:
- reports/
reports:
junit: reports/report.xml
# If pipeline not triggered by tag and backend code has not changed:
# - If there is a previous successful backend test run then download and reuse its
# artifacts (coverage etc).
# - Otherwise runs the backend-test job like normal.
no-backend-changes-so-try-skip-tests-group-1:
extends:
- backend-test-group-1
- .skippable-job
variables:
SKIP_JOB_NAME: backend-test-group-1
DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
# Override inherited only block, so we can run this job in the
# exact opposite situations.
only: null
except:
changes:
- backend/**/*
- premium/backend/**/*
# Create 2 more separate groups to parallelize pytest by using separate groups to
# decrease overall build time. Pytest xdist doesn't help as the gitlab saas runners only
# have a single virtual core so `pytest -n 2+` slows things down.
backend-test-group-2:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 2
no-backend-changes-so-try-skip-tests-group-2:
extends: no-backend-changes-so-try-skip-tests-group-1
variables:
SKIP_JOB_NAME: backend-test-group-2
PYTEST_SPLIT_GROUP: 2
backend-test-group-3:
extends: backend-test-group-1
variables:
PYTEST_SPLIT_GROUP: 3
no-backend-changes-so-try-skip-tests-group-3:
extends: no-backend-changes-so-try-skip-tests-group-1
variables:
SKIP_JOB_NAME: backend-test-group-3
PYTEST_SPLIT_GROUP: 3
# Collects together all the separate backend coverage databases from previous jobs and
# combines them to generate a single report for gitlab to use. Gitlab itself does not
# correctly merge these if you just add them all separately into artifacts->reports->
# cobertura.
backend-coverage:
image: $CI_UTIL_IMAGE
stage: build-final
interruptible: true
only:
variables:
- $ENABLE_COVERAGE == "true"
# Prevent rebuilds when tagging as all we want to do is tag and push
except:
variables:
- $CI_COMMIT_TAG
# Depend on the `reports` artifacts from the previous jobs
dependencies:
- backend-test-group-1
- backend-test-group-2
- backend-test-group-3
# If the actual tests were skipped then the artifacts will be on these jobs instead
- no-backend-changes-so-try-skip-tests-group-1
- no-backend-changes-so-try-skip-tests-group-2
- no-backend-changes-so-try-skip-tests-group-3
script:
- . /baserow/venv/bin/activate
# The reports artifacts will be extracted before the script runs into reports by
# gitlab
- cp reports/.coverage.* .
- export COVERAGE_RCFILE=backend/.coveragerc
- coverage combine
- coverage report
- coverage xml -o coverage.xml
artifacts:
reports:
cobertura: coverage.xml
coverage: '/^TOTAL.+?(\d+\%)$/'
# If pipeline not triggered by tag: # If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know # - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
@ -370,23 +251,6 @@ web-frontend-lint:
- web-frontend/**/* - web-frontend/**/*
- premium/web-frontend/**/* - premium/web-frontend/**/*
# If pipeline not triggered by tag and web-frontend code has not changed:
# - If there is a previous successful lint run in the cache then skip.
# - otherwise runs lint and stores success file in cache if successful.
no-web-frontend-changes-so-try-skip-lint:
extends:
- web-frontend-lint
- .skippable-job
variables:
SKIP_JOB_NAME: web-frontend-lint
# Override inherited only block so we can run this job in the
# exact opposite situations.
only: null
except:
changes:
- web-frontend/**/*
- premium/web-frontend/**/*
# If pipeline not triggered by tag and web-frontend code has changed: # If pipeline not triggered by tag and web-frontend code has changed:
# - Runs the web-frontend tests # - Runs the web-frontend tests
# - Generates coverage and testing reports # - Generates coverage and testing reports
@ -395,45 +259,13 @@ web-frontend-test:
extends: extends:
- .docker-image-test-stage - .docker-image-test-stage
script: script:
- mkdir reports/ -p
- TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
- | - |
docker run --name=webfrontend_test $WEBFRONTEND_CI_DEV_IMAGE $TEST_TYPE \ docker run --name=webfrontend_test $WEBFRONTEND_CI_DEV_IMAGE ci-test;
| tee reports/stdout.txt;
- docker cp webfrontend_test:/baserow/reports .
- docker rm webfrontend_test
only: only:
# Skip testing on if no change to web-frontend files # Skip testing on if no change to web-frontend files
changes: changes:
- web-frontend/**/* - web-frontend/**/*
- premium/web-frontend/**/* - premium/web-frontend/**/*
artifacts:
paths:
- reports/
reports:
cobertura: reports/coverage/cobertura-coverage.xml
junit: reports/junit.xml
coverage: '/Lines\s*:\s*(\d+.?\d*)%/'
# If pipeline not triggered by tag and web-frontend code has not changed:
# - If there is a previous successful webfrontend test run in the cache then skip and
# unpack its coverage reports.
# - Otherwise runs the tests, coverage reporting and stores in cache if successful
# as normal
no-web-frontend-changes-so-try-skip-tests:
extends:
- web-frontend-test
- .skippable-job
variables:
SKIP_JOB_NAME: web-frontend-test
DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
# Override inherited only block so we can run this job in the
# exact opposite situations.
only: null
except:
changes:
- web-frontend/**/*
- premium/web-frontend/**/*
# If pipeline not triggered by tag: # If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know # - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
@ -444,156 +276,3 @@ build-final-web-frontend-image:
IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH
# ================================== TRIGGER SAAS =====================================
# Triggers a special pipeline in dependant project and passes various variables to it.
# Only on master and develop.
trigger-saas-build:
stage: publish
inherit:
variables:
- CI_COMMIT_BRANCH
- TESTED_BACKEND_CI_IMAGE
- TESTED_WEBFRONTEND_CI_IMAGE
- CI_COMMIT_SHA
- CI_COMMIT_SHORT_SHA
- DEVELOP_BRANCH_NAME
- MASTER_BRANCH_NAME
variables:
UPSTREAM_SHA: $CI_COMMIT_SHA
UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA
UPSTREAM_TESTED_BACKEND_CI_IMAGE: $TESTED_BACKEND_CI_IMAGE
UPSTREAM_TESTED_WEBFRONTEND_CI_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
only:
changes:
- web-frontend/**/*
- premium/web-frontend/**/*
- backend/**/*
- premium/backend/**/*
variables:
- ($CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME || $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME)
allow_failure: true
trigger:
project: bramw/baserow-saas
branch: $CI_COMMIT_BRANCH
# ================================== PUSHING BACKEND ==================================
# Push baserow/backend:develop_latest
publish-backend-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/backend_dev:develop_latest
publish-backend-develop-latest-dev-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_DEV_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/backend:$VERSION_GIT_TAG
publish-backend-release-tagged-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_TAG
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/backend:latest
publish-backend-latest-release-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_TAG
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# ================================ PUSHING WEB-FRONTEND ===============================
# Push baserow/web-frontend:develop_latest
publish-webfrontend-develop-latest-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/web-frontend_dev:develop_latest
publish-webfrontend-develop-latest-dev-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_DEV_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$DEVELOP_LATEST_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/web-frontend:$VERSION_GIT_TAG
publish-webfrontend-release-tagged-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_TAG
variables:
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER
# Push baserow/web-frontend:latest
publish-webfrontend-latest-release-image:
extends: .publish-baserow-image
only:
variables:
- $CI_COMMIT_TAG
variables:
SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest"
TARGET_REGISTRY: $CI_REGISTRY
TARGET_REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD
TARGET_REGISTRY_USER: $CI_REGISTRY_USER

145
.gitlab/ci_includes/jobs.yml
View file

@ -262,148 +262,3 @@
- docker:20.10.12-dind - docker:20.10.12-dind
# Pushes $SOURCE_IMAGE to $TARGET_IMAGE using the $TARGET_REGISTRY_PASSWORD,
# $TARGET_REGISTRY_USER and $TARGET_REGISTRY credentials.
#
# Set $SKIP_IF_TAG_NOT_ON_BRANCH to make the job skip if the commit is not on
# the specified branch. Useful for TAG pipelines when $CI_COMMIT_BRANCH is not set
# and so we need to do some extra git work to figure out what branches this commit is
# on.
#
# Set $SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH to a branch name. If the job is not
# for a commit which is the latest on the specified branch name (for example due to
# someone re-running a pipeline for an old commit) this job will be skipped.
.publish-baserow-image:
image: $CI_UTIL_IMAGE
stage: publish
services:
- docker:20.10.12-dind
except:
refs:
- pipelines
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
allow_failure:
# By exiting with this code we can skip this step without failing the build,
# but still fail if something else goes wrong.
exit_codes: 137
script:
- |
if [[ -n "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
# Query for all the branches that this commit is part of.
curl -s --header "JOB-TOKEN: $CI_JOB_TOKEN" \
"https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$CI_COMMIT_SHA/refs?type=branch" \
-o this_commits_branches.json;
# Extract just the branch names from the json so we can assert it matches.
TAG_BRANCH_NAMES=$(cat this_commits_branches.json | jq -r ".[].name")
NUM_BRANCHES=$(cat this_commits_branches.json | jq length)
# Ensure the commit is only on master and no other branches, otherwise someone
# could checkout a master commit as a new branch and tag it to cause an image
# upload.
if [[ "$NUM_BRANCHES" != "1" || "$TAG_BRANCH_NAMES" != "$SKIP_IF_TAG_NOT_ON_BRANCH" ]]; then
echo "Tags should never be applied to non $SKIP_IF_TAG_NOT_ON_BRANCH branches!" 2>&1;
echo "Pipeline is running for tag: $CI_COMMIT_TAG which for a commit that only appears on $SKIP_IF_TAG_NOT_ON_BRANCH and no other branches." 2>&1;
echo "Instead this commit appears on $NUM_BRANCHES branches called $TAG_BRANCH_NAMES" 2>&1;
exit 1;
fi
fi
if [[ -n "$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH" ]]; then
LATEST_COMMIT_HASH=$(git rev-parse origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH)
HEAD_COMMIT_HASH=$CI_COMMIT_SHA
if [[ "$LATEST_COMMIT_HASH" != "$HEAD_COMMIT_HASH" ]]; then
echo "Pipeline is not running for latest commit on origin/$SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH";
echo " which has commit $LATEST_COMMIT_HASH.";
echo "Instead pipeline is running on commit $HEAD_COMMIT_HASH, exitting as configured to do so in this situation...";
exit 137;
fi
fi
echo "$TARGET_REGISTRY_PASSWORD" | docker login -u "$TARGET_REGISTRY_USER" "$TARGET_REGISTRY" --password-stdin
if ! docker pull $SOURCE_IMAGE; then
echo "Could not pull $SOURCE_IMAGE, has the build pipeline finished yet?" 2>&1;
exit 1
fi
docker tag $SOURCE_IMAGE $TARGET_IMAGE
docker push $TARGET_IMAGE
.skippable-job:
before_script:
- |
if [[ -z "$SKIP_JOB_NAME" ]]; then
echo "Must provide SKIP_JOB_NAME as a job variable" 2>&1
exit 1
fi
if [[ "$ENABLE_JOB_SKIPPING" = "true" ]]; then
try_download_latest_successful_artifacts_for_commit(){
COMMIT_HASH=$1
JOB_NAME=$2
echo -e "\e[0Ksection_start:`date +%s`:$COMMIT_HASH$JOB_NAME[collapsed=true]\r\e[0KPrevious successful run check for $JOB_NAME and $COMMIT_HASH"
URL="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/commits/$COMMIT_HASH/statuses?name=$JOB_NAME"
COMMIT_GITLAB_JOBS=$(curl --header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" $URL)
if [[ "$COMMIT_GITLAB_JOBS" ]]; then
echo "Got these job statuses: $COMMIT_GITLAB_JOBS"
JOB_ID=$(echo $COMMIT_GITLAB_JOBS| jq "[.[] | select(.status == \"success\")][0].id")
# Check if JOB_ID is an integer (POSIX compliant way)
if [ "$JOB_ID" -eq "$JOB_ID" ] 2> /dev/null; then
if [[ -n "$DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP" ]] ; then
exit_code=0
curl --fail --location --output artifacts.zip \
--header "PRIVATE-TOKEN: $PROJECT_READ_ONLY_API_TOKEN" \
"https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/jobs/$JOB_ID/artifacts" \
|| exit_code=$?;
if [ ${exit_code} -ne 0 ]; then
echo "Failed to get artifacts from successful run $JOB_ID"
else
unzip artifacts.zip || exit_code=$?
if [ ${exit_code} -ne 0 ]; then
echo "Failed to unzip artifacts"
else
if [[ -f "reports/stdout.txt" ]]; then
cat reports/stdout.txt;
fi
echo "Skipping $JOB_NAME as previous successful run for $COMMIT_HASH and it's artifacts were found."
exit 0;
fi
fi
else
echo "Skipping $JOB_NAME as previous successful build for $COMMIT_HASH were found.".
exit 0;
fi
else
echo "Failed to find successful run of $JOB_NAME in $COMMIT_GITLAB_JOBS"
fi
else
echo "Failed to query gitlab for jobs";
fi
echo -e "\e[0Ksection_end:`date +%s`:$COMMIT_HASH$JOB_NAME\r\e[0K"
}
SECOND_PARENT_COMMIT=$(git rev-list -1 --merges ${CI_COMMIT_SHA}~1..${CI_COMMIT_SHA})
if [[ -z "$SECOND_PARENT_COMMIT" ]] ; then
# If there is no second parent commit then there is only one parent commit
# and so we can safely look for its artifacts.
PREVIOUS_COMMIT_SHA=$(git rev-parse HEAD~1)
# Search for successful runs of either the normal job or this job itself
# for either this or previous commit.
try_download_latest_successful_artifacts_for_commit $CI_COMMIT_SHA $SKIP_JOB_NAME
try_download_latest_successful_artifacts_for_commit $CI_COMMIT_SHA $CI_JOB_NAME
try_download_latest_successful_artifacts_for_commit $PREVIOUS_COMMIT_SHA $SKIP_JOB_NAME
try_download_latest_successful_artifacts_for_commit $PREVIOUS_COMMIT_SHA $CI_JOB_NAME
echo "Actually running job as successful run for previous or this commit not found"
else
# There is a second (or more) parent commit meaning we should re-run this job
# as a merge has happened.
echo "Running full job as this is a merge commit."
fi
else
echo "Force running job regardless of previous runs."
fi

12
backend/.coveragerc
View file

@ -1,12 +0,0 @@
[run]
# Also required for gitlab MR coverage to be shown correctly.
relative_files = True
omit =
*/generated/*
# We can't set source as it changes the xml reports file paths to be relative from
# say `backend/src` instead of the root of the repo. Gitlab needs paths to be relative
# from the root so instead we just set include which ensures in gitlab MR coverage is
# shown correctly.
include =
backend/src/**/*
premium/backend/src/**/*

1140
backend/.test_durations
View file

File diff suppressed because one or more lines are too long

24
backend/Makefile
View file

@ -16,36 +16,14 @@ format:
test: test:
pytest tests ../premium/backend/tests || exit; pytest tests ../premium/backend/tests || exit;
test-regenerate-ci-durations:
pytest tests ../premium/backend/tests --store-durations || exit;
test-parallel: test-parallel:
pytest tests ../premium/backend/tests -n 10 || exit; pytest tests ../premium/backend/tests -n 10 || exit;
PYTEST_SPLITS:=1
PYTEST_SPLIT_GROUP:=1
ci-test-python: ci-test-python:
mkdir reports/ -p; \ pytest \
cd ..; \
COVERAGE_FILE=backend/reports/.coverage.$(PYTEST_SPLIT_GROUP) \
coverage run \
--rcfile=backend/.coveragerc \
-m pytest \
--durations-path=backend/.test_durations \
--splits $(PYTEST_SPLITS) \
--group $(PYTEST_SPLIT_GROUP) \
--junitxml=backend/reports/report.xml \
backend/tests \ backend/tests \
premium/backend/tests; premium/backend/tests;
generate-html-coverage-report:
mkdir html_coverage/ -p; \
cd ..; \
coverage run --rcfile=backend/.coveragerc -m pytest \
backend/tests \
premium/backend/tests; \
coverage html -d html_coverage/;
ci-check-startup-python: ci-check-startup-python:
timeout --preserve-status 10s \ timeout --preserve-status 10s \
gunicorn --workers=1 -b 0.0.0.0:8002 \ gunicorn --workers=1 -b 0.0.0.0:8002 \

2
backend/docker/docker-entrypoint.sh
View file

@ -124,7 +124,7 @@ case "$1" in
exec make lint-python exec make lint-python
;; ;;
ci-test) ci-test)
exec make ci-test-python PYTEST_SPLITS="${PYTEST_SPLITS:-1}" PYTEST_SPLIT_GROUP="${PYTEST_SPLIT_GROUP:-1}" exec make ci-test-python
;; ;;
ci-check-startup) ci-check-startup)
exec make ci-check-startup-python exec make ci-check-startup-python

2
web-frontend/Makefile
View file

@ -20,7 +20,7 @@ unit-test:
npx jest --selectProjects unit --selectProjects premium || exit; npx jest --selectProjects unit --selectProjects premium || exit;
ci-test-javascript: ci-test-javascript:
JEST_JUNIT_OUTPUT_DIR=../reports/ npx jest --ci --collectCoverage --coverageDirectory="./reports/coverage/" || exit; npx jest --ci || exit;
unit-test-watch: unit-test-watch:
npx jest test/unit --watch || exit; npx jest test/unit --watch || exit;

26
web-frontend/jest.config.js
View file

@ -1,11 +1,9 @@
// Setting reporters on the command line does not work so enable via this env variable // The main jest config file used to run all of our tests.
// we have to set anyway when using the junit reporter in CI.
const junitReporterConfig = process.env.JEST_JUNIT_OUTPUT_DIR
? {
reporters: ['default', '<rootDir>/web-frontend/node_modules/jest-junit'],
}
: {}
module.exports = { module.exports = {
// The rootDir used by jest must be the root of the repository so the premium tests
// and frontend code are contained within jest's rootDir. This is because:
// - Jest cannot collect coverage for files outside of its rootDir
// - Jest struggles to run tests which are outside of its rootDir.
rootDir: '..', rootDir: '..',
roots: ['<rootDir>/web-frontend/', '<rootDir>/premium/web-frontend'], roots: ['<rootDir>/web-frontend/', '<rootDir>/premium/web-frontend'],
moduleDirectories: ['<rootDir>/web-frontend/node_modules/'], moduleDirectories: ['<rootDir>/web-frontend/node_modules/'],
@ -15,18 +13,4 @@ module.exports = {
'<rootDir>/premium/web-frontend/test/unit', '<rootDir>/premium/web-frontend/test/unit',
'<rootDir>/web-frontend/test/server', '<rootDir>/web-frontend/test/server',
], ],
coverageReporters: [
'text-summary',
['cobertura', { projectRoot: '/baserow/' }],
],
collectCoverageFrom: [
'<rootDir>/premium/web-frontend/modules/**/*.{js,Vue,vue}',
'<rootDir>/web-frontend/modules/**/*.{js,Vue,vue}',
'!**/node_modules/**',
'!**/.nuxt/**',
'!**/reports/**',
'!**/test/**',
'!**/generated/**',
],
...junitReporterConfig,
} }

29
web-frontend/yarn.lock
View file

@ -2423,11 +2423,6 @@ ansi-regex@^5.0.0:
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
ansi-styles@^2.2.1: ansi-styles@^2.2.1:
version "2.2.1" version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@ -6930,16 +6925,6 @@ jest-jasmine2@^26.6.3:
pretty-format "^26.6.2" pretty-format "^26.6.2"
throat "^5.0.0" throat "^5.0.0"
jest-junit@^13.0.0:
version "13.0.0"
resolved "https://registry.yarnpkg.com/jest-junit/-/jest-junit-13.0.0.tgz#479be347457aad98ae8a5983a23d7c3ec526c9a3"
integrity sha512-JSHR+Dhb32FGJaiKkqsB7AR3OqWKtldLd6ZH2+FJ8D4tsweb8Id8zEVReU4+OlrRO1ZluqJLQEETm+Q6/KilBg==
dependencies:
mkdirp "^1.0.4"
strip-ansi "^6.0.1"
uuid "^8.3.2"
xml "^1.0.1"
jest-leak-detector@^26.6.2: jest-leak-detector@^26.6.2:
version "26.6.2" version "26.6.2"
resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-26.6.2.tgz#7717cf118b92238f2eba65054c8a0c9c653a91af" resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-26.6.2.tgz#7717cf118b92238f2eba65054c8a0c9c653a91af"
@ -11132,13 +11117,6 @@ strip-ansi@^6.0.0:
dependencies: dependencies:
ansi-regex "^5.0.0" ansi-regex "^5.0.0"
strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-bom@^3.0.0: strip-bom@^3.0.0:
version "3.0.0" version "3.0.0"
resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
@ -12055,7 +12033,7 @@ uuid@^3.3.2:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
uuid@^8.3.0, uuid@^8.3.2: uuid@^8.3.0:
version "8.3.2" version "8.3.2"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
@ -12556,11 +12534,6 @@ xml-name-validator@^3.0.0:
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
xml@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
xmlchars@^2.2.0: xmlchars@^2.2.0:
version "2.2.0" version "2.2.0"
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"