1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-07 22:35:36 +00:00

Pin our Python dependencies using pip-tools

This commit is contained in:
Nigel Gott 2022-03-10 11:48:19 +00:00
parent c00dc4e736
commit b7e140fc84
9 changed files with 686 additions and 277 deletions

View file

@ -6,15 +6,11 @@ install-dev-dependencies:
pip install -r requirements/dev.txt
lint:
./python_dep_check.sh check && \
flake8 src tests ../premium/backend && \
black . ../premium/backend --extend-exclude='/generated/' --check && \
bandit -r --exclude src/baserow/test_utils src/ ../premium/backend/src/ \
|| exit;
pip-clean-reinstall-and-freeze:
./python_dep_check.sh freeze
lint-python: lint
format:

View file

@ -1,65 +0,0 @@
#!/bin/bash
# Bash strict mode: http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail
# Currently we only define our direct dependencies in requirements/base.txt and
# requirements/dev.txt. This means that pip figures out and resolves the versions of
# dependant libraries for us based on the constraints of our direct dependencies.
# This means a seemingly simple change to base.txt or dev.txt can result in important
# dependencies having their versions change under the hood with us being non the wiser.
#
# This file is a temporary solution and is used by our `make lint` command and a new
# file requirements/dev_frozen.txt to clearly show and enforce what exactly all of our
# python dependencies, direct and indirect are. Really we should switch away from just
# using pip to something like Poetry which has a lock file built in.
FROZEN_DEP_FILE="requirements/dev_frozen.txt"
if [ -t 0 ]; then
TPUTTERM=()
else
# if we are in a non-interactive environment set a default -T for tput so it doesn't
# crash
TPUTTERM=(-T xterm-256color)
fi
safe_tput(){
tput "${TPUTTERM[@]}" "$@"
}
RED=$(safe_tput setaf 1)
GREEN=$(safe_tput setaf 2)
YELLOW=$(safe_tput setaf 3)
NC=$(safe_tput sgr0) # No Color
if [[ "${1:-}" == "check" ]]; then
FROZEN_DEPS=$(< "$FROZEN_DEP_FILE")
NEW_DEPS=$(pip freeze)
if [[ "$FROZEN_DEPS" != "$NEW_DEPS" ]]; then
echo "$RED Python dependencies have changed but $FROZEN_DEP_FILE " \
"has not been updated, please run$NC$GREEN make pip-clean-reinstall-and-freeze and commit " \
"$NC$RED so we can clearly see any changed python dependencies. See below " \
"for diff:${NC}"
diff <(echo "$FROZEN_DEPS" ) <(echo "$NEW_DEPS")
exit 1
else
echo "$GREEN No python dep changes detected $NC"
fi
elif [[ "${1:-}" == "freeze" ]]; then
if [ ! -d /baserow/venv ]; then
echo "Please run inside the backend docker container, couldn't find the venv."
exit 1
fi
rm -rf /baserow/venv
python3 -m virtualenv /baserow/venv
. /baserow/venv/bin/activate
pip install -r requirements/base.txt
pip install -r requirements/dev.txt
pip freeze > "$FROZEN_DEP_FILE"
echo "$GREEN Successfully froze current python deps to $FROZEN_DEP_FILE $NC"
exit 0
else
echo "$YELLOW Supported arguments are check or freeze, unknown args provided. $NC"
exit 1
fi

View file

@ -0,0 +1,48 @@
# Readme
We use [pip-tools](https://github.com/jazzband/pip-tools) to manage our pip requirement
files.
## Base Requirements
`base.in` contains our non-dev requirements for our backend python environment and
`base.txt` is the corresponding pip requirements file generated by `pip-tools`:
```
pip-compile --output-file=base.txt base.in
```
We install the `base.txt` requirements into the [`baserow/backend`](../Dockerfile)
docker image. You can launch an environment using these images by running
`./dev.sh local restart --build`.
## Dev Requirements
`dev.in` contains our extra dev requirements on-top of `base.in`.
`dev.txt` is the corresponding pip requirements file generated by `pip-tools`:
```
pip-compile --output-file=dev.txt dev.in
```
We install the `dev.txt` requirements into the [`baserow/backend`](../Dockerfile)
docker image when built using the dev target (`docker build ... --target dev`). This
dev backend image is the one used when running `./dev.sh restart --build` etc.
## Common Operations
### Add a new base dependency
1. Add a line to `base.in` containing your new dependency
2. In the `backend lint` tab opened by running `./dev.sh --build`, or an active virtual
environment with `pip-tools` installed.
3. Ensure you are using python 3.7 if not using `./dev.sh --build`
4. `cd requirements`
5. Run `pip-compile --output-file=base.txt base.in`, review the changes to base.txt,
commit and push them to your MR.
### Add a new dev dependency
1. Add a line to `dev.in` containing your new dependency
2. In the `backend lint` tab opened by running `./dev.sh --build`, or an active virtual
environment with `pip-tools` installed.
3. Ensure you are using python 3.7 if not using `./dev.sh --build`
4. `cd requirements`
4. Run `pip-compile --output-file=dev.txt dev.in`, review the changes to dev.txt,
commit and push them to your MR.
### Upgrade an existing dependency
1. Change the version in the corresponding `.in` file.
2. Follow from step 2 above depending on which `.in` file you edited.

View file

@ -0,0 +1,35 @@
Django==3.2.12
django-cors-headers==3.8.0
djangorestframework==3.13.1
drf-jwt==1.19.1
psycopg2==2.9.1
Faker==8.11.0
Twisted==22.1
gunicorn==20.1.0
uvicorn[standard]==0.15.0
requests==2.26.0
itsdangerous==2.0.1
Pillow==9.0.0
drf-spectacular==0.21.2
asgiref==3.4.1
channels==3.0.4
channels-redis==3.3.0
celery[redis]==5.2.3
django-redis==5.2.0
django-celery-email==3.0.0
advocate==1.0.0
zipp==3.5.0
unicodecsv==0.14.1
django-celery-beat==2.2.1
celery-redbeat==2.0.0
service-identity==21.1.0
regex==2021.8.3
cryptography==36.0.1
antlr4-python3-runtime==4.8.0
tqdm==4.62.3
boto3==1.20.38
django-storages==1.12.3
django-health-check==3.16.5
psutil==5.9.0
dj-database-url==0.5.0
redis==4.1.4

View file

@ -1,35 +1,315 @@
Django==3.2.12
django-cors-headers==3.8.0
djangorestframework==3.13.1
drf-jwt==1.19.1
psycopg2==2.9.1
Faker==8.11.0
Twisted==22.1
gunicorn==20.1.0
uvicorn[standard]==0.15.0
requests==2.26.0
itsdangerous==2.0.1
Pillow==9.0.0
drf-spectacular==0.21.2
asgiref==3.4.1
channels==3.0.4
channels-redis==3.3.0
celery[redis]==5.2.3
django-redis==5.2.0
django-celery-email==3.0.0
#
# This file is autogenerated by pip-compile with python 3.7
# To update, run:
#
# pip-compile --output-file=base.txt base.in
#
advocate==1.0.0
zipp==3.5.0
unicodecsv==0.14.1
django-celery-beat==2.2.1
celery-redbeat==2.0.0
service-identity==21.1.0
regex==2021.8.3
cryptography==36.0.1
# via -r base.in
aioredis==1.3.1
# via channels-redis
amqp==5.1.0
# via kombu
antlr4-python3-runtime==4.8.0
tqdm==4.62.3
# via -r base.in
asgiref==3.4.1
# via
# -r base.in
# channels
# channels-redis
# daphne
# django
# uvicorn
async-timeout==4.0.2
# via aioredis
attrs==21.4.0
# via
# automat
# jsonschema
# service-identity
# twisted
autobahn==22.2.2
# via daphne
automat==20.2.0
# via twisted
billiard==3.6.4.0
# via celery
boto3==1.20.38
django-storages==1.12.3
# via -r base.in
botocore==1.23.54
# via
# boto3
# s3transfer
cached-property==1.5.2
# via kombu
celery[redis]==5.2.3
# via
# -r base.in
# celery-redbeat
# django-celery-beat
# django-celery-email
celery-redbeat==2.0.0
# via -r base.in
certifi==2021.10.8
# via requests
cffi==1.15.0
# via cryptography
channels==3.0.4
# via
# -r base.in
# channels-redis
channels-redis==3.3.0
# via -r base.in
charset-normalizer==2.0.12
# via requests
click==8.0.4
# via
# celery
# click-didyoumean
# click-plugins
# click-repl
# uvicorn
click-didyoumean==0.3.0
# via celery
click-plugins==1.1.1
# via celery
click-repl==0.2.0
# via celery
constantly==15.1.0
# via twisted
cryptography==36.0.1
# via
# -r base.in
# autobahn
# pyjwt
# pyopenssl
# service-identity
daphne==3.0.2
# via channels
deprecated==1.2.13
# via redis
dj-database-url==0.5.0
# via -r base.in
django==3.2.12
# via
# -r base.in
# channels
# django-appconf
# django-celery-beat
# django-celery-email
# django-cors-headers
# django-health-check
# django-redis
# django-storages
# django-timezone-field
# djangorestframework
# drf-jwt
# drf-spectacular
django-appconf==1.0.5
# via django-celery-email
django-celery-beat==2.2.1
# via -r base.in
django-celery-email==3.0.0
# via -r base.in
django-cors-headers==3.8.0
# via -r base.in
django-health-check==3.16.5
# via -r base.in
django-redis==5.2.0
# via -r base.in
django-storages==1.12.3
# via -r base.in
django-timezone-field==4.2.3
# via django-celery-beat
djangorestframework==3.13.1
# via
# -r base.in
# drf-jwt
# drf-spectacular
drf-jwt==1.19.1
# via -r base.in
drf-spectacular==0.21.2
# via -r base.in
faker==8.11.0
# via -r base.in
gunicorn==20.1.0
# via -r base.in
h11==0.13.0
# via uvicorn
hiredis==2.0.0
# via aioredis
httptools==0.2.0
# via uvicorn
hyperlink==21.0.0
# via
# autobahn
# twisted
idna==3.3
# via
# hyperlink
# requests
# twisted
importlib-metadata==4.11.2
# via
# click
# jsonschema
# kombu
# redis
importlib-resources==5.4.0
# via jsonschema
incremental==21.3.0
# via twisted
inflection==0.5.1
# via drf-spectacular
itsdangerous==2.0.1
# via -r base.in
jmespath==0.10.0
# via
# boto3
# botocore
jsonschema==4.4.0
# via drf-spectacular
kombu==5.2.4
# via celery
msgpack==1.0.3
# via channels-redis
ndg-httpsclient==0.5.1
# via advocate
netifaces==0.11.0
# via advocate
packaging==21.3
# via redis
pillow==9.0.0
# via -r base.in
prompt-toolkit==3.0.28
# via click-repl
psutil==5.9.0
dj_database_url==0.5.0
# via -r base.in
psycopg2==2.9.1
# via -r base.in
pyasn1==0.4.8
# via
# advocate
# ndg-httpsclient
# pyasn1-modules
# service-identity
pyasn1-modules==0.2.8
# via service-identity
pycparser==2.21
# via cffi
pyjwt[crypto]==2.3.0
# via drf-jwt
pyopenssl==22.0.0
# via
# advocate
# ndg-httpsclient
# twisted
pyparsing==3.0.7
# via packaging
pyrsistent==0.18.1
# via jsonschema
python-crontab==2.6.0
# via django-celery-beat
python-dateutil==2.8.2
# via
# botocore
# celery-redbeat
# faker
# python-crontab
python-dotenv==0.19.2
# via uvicorn
pytz==2021.3
# via
# celery
# django
# django-timezone-field
# djangorestframework
pyyaml==6.0
# via
# drf-spectacular
# uvicorn
redis==4.1.4
# via
# -r base.in
# celery
# celery-redbeat
# django-redis
regex==2021.8.3
# via -r base.in
requests==2.26.0
# via
# -r base.in
# advocate
s3transfer==0.5.2
# via boto3
service-identity==21.1.0
# via
# -r base.in
# twisted
six==1.16.0
# via
# advocate
# automat
# click-repl
# python-dateutil
# service-identity
sqlparse==0.4.2
# via django
tenacity==8.0.1
# via celery-redbeat
text-unidecode==1.3
# via faker
tqdm==4.62.3
# via -r base.in
twisted[tls]==22.1.0
# via
# -r base.in
# daphne
txaio==22.2.1
# via autobahn
typing-extensions==4.1.1
# via
# asgiref
# async-timeout
# drf-spectacular
# h11
# importlib-metadata
# jsonschema
# twisted
# uvicorn
unicodecsv==0.14.1
# via -r base.in
uritemplate==4.1.1
# via drf-spectacular
urllib3==1.26.8
# via
# advocate
# botocore
# requests
uvicorn[standard]==0.15.0
# via -r base.in
uvloop==0.16.0
# via uvicorn
vine==5.0.0
# via
# amqp
# celery
# kombu
watchgod==0.7
# via uvicorn
wcwidth==0.2.5
# via prompt-toolkit
websockets==10.2
# via uvicorn
wrapt==1.13.3
# via deprecated
zipp==3.5.0
# via
# -r base.in
# importlib-metadata
# importlib-resources
zope-interface==5.4.0
# via twisted
# The following packages are considered to be unsafe in a requirements file:
# setuptools

View file

@ -0,0 +1,28 @@
-c base.txt
flake8==3.9.2
pytest==6.2.5
pytest-django==4.4.0
pytest-env==0.6.2
pytest-asyncio==0.15.1
pytest-ordering==0.6
pytest-mock==3.6.1
pytest-icdiff==0.5
freezegun==1.1.0
responses==0.13.4
watchdog==2.1.4
argh==0.26.2
black==21.7b0
pyinstrument==4.0.3
pyfakefs==4.5.5
pytest-xdist==2.3.0
responses==0.13.4
django-silk==4.2.0
django-extensions==3.1.5
snoop==0.4.1
openapi-spec-validator==0.4.0
pytest-html==3.1.1
coverage==6.2
pytest-split==0.6.0
bandit==1.7.2
pip-tools==6.5.1
autopep8==1.5.7

View file

@ -1,25 +1,267 @@
flake8==3.9.2
pytest==6.2.5
pytest-django==4.4.0
pytest-env==0.6.2
pytest-asyncio==0.15.1
pytest-ordering==0.6
pytest-mock==3.6.1
pytest-icdiff==0.5
freezegun==1.1.0
responses==0.13.4
watchdog==2.1.4
#
# This file is autogenerated by pip-compile with python 3.7
# To update, run:
#
# pip-compile --output-file=dev.txt dev.in
#
appdirs==1.4.4
# via black
argh==0.26.2
black==21.7b0
pyinstrument==4.0.3
pyfakefs==4.5.0
pytest-xdist==2.3.0
responses==0.13.4
django-silk==4.2.0
django-extensions==3.1.5
snoop==0.4.1
openapi-spec-validator==0.4.0
pytest-html==3.1.1
coverage==6.2
pytest-split==0.6.0
# via -r dev.in
asgiref==3.4.1
# via
# -c base.txt
# django
asttokens==2.0.5
# via snoop
attrs==21.4.0
# via
# -c base.txt
# jsonschema
# pytest
autopep8==1.5.7
# via
# -r dev.in
# django-silk
bandit==1.7.2
# via -r dev.in
black==21.7b0
# via -r dev.in
certifi==2021.10.8
# via
# -c base.txt
# requests
charset-normalizer==2.0.12
# via
# -c base.txt
# requests
cheap-repr==0.5.1
# via snoop
click==8.0.4
# via
# -c base.txt
# black
# pip-tools
coverage==6.2
# via -r dev.in
django==3.2.12
# via
# -c base.txt
# django-extensions
# django-silk
django-extensions==3.1.5
# via -r dev.in
django-silk==4.2.0
# via -r dev.in
execnet==1.9.0
# via pytest-xdist
executing==0.8.3
# via snoop
flake8==3.9.2
# via -r dev.in
freezegun==1.1.0
# via -r dev.in
gitdb==4.0.9
# via gitpython
gitpython==3.1.27
# via bandit
gprof2dot==2021.2.21
# via django-silk
icdiff==2.0.4
# via pytest-icdiff
idna==3.3
# via
# -c base.txt
# requests
importlib-metadata==4.11.2
# via
# -c base.txt
# click
# flake8
# jsonschema
# pep517
# pluggy
# pytest
# stevedore
importlib-resources==5.4.0
# via
# -c base.txt
# jsonschema
iniconfig==1.1.1
# via pytest
jinja2==3.0.3
# via django-silk
jsonschema==4.4.0
# via
# -c base.txt
# openapi-schema-validator
# openapi-spec-validator
markupsafe==2.1.0
# via jinja2
mccabe==0.6.1
# via flake8
mypy-extensions==0.4.3
# via black
openapi-schema-validator==0.2.3
# via openapi-spec-validator
openapi-spec-validator==0.4.0
# via -r dev.in
packaging==21.3
# via
# -c base.txt
# pytest
pathspec==0.9.0
# via black
pbr==5.8.1
# via stevedore
pep517==0.12.0
# via pip-tools
pip-tools==6.5.1
# via -r dev.in
pluggy==1.0.0
# via pytest
pprintpp==0.4.0
# via pytest-icdiff
py==1.11.0
# via
# pytest
# pytest-forked
pycodestyle==2.7.0
# via
# autopep8
# flake8
pyfakefs==4.5.5
# via -r dev.in
pyflakes==2.3.1
# via flake8
pygments==2.11.2
# via
# django-silk
# snoop
pyinstrument==4.0.3
# via -r dev.in
pyparsing==3.0.7
# via
# -c base.txt
# packaging
pyrsistent==0.18.1
# via
# -c base.txt
# jsonschema
pytest==6.2.5
# via
# -r dev.in
# pytest-asyncio
# pytest-django
# pytest-env
# pytest-forked
# pytest-html
# pytest-icdiff
# pytest-metadata
# pytest-mock
# pytest-ordering
# pytest-split
# pytest-xdist
pytest-asyncio==0.15.1
# via -r dev.in
pytest-django==4.4.0
# via -r dev.in
pytest-env==0.6.2
# via -r dev.in
pytest-forked==1.4.0
# via pytest-xdist
pytest-html==3.1.1
# via -r dev.in
pytest-icdiff==0.5
# via -r dev.in
pytest-metadata==1.11.0
# via pytest-html
pytest-mock==3.6.1
# via -r dev.in
pytest-ordering==0.6
# via -r dev.in
pytest-split==0.6.0
# via -r dev.in
pytest-xdist==2.3.0
# via -r dev.in
python-dateutil==2.8.2
# via
# -c base.txt
# django-silk
# freezegun
pytz==2021.3
# via
# -c base.txt
# django
# django-silk
pyyaml==6.0
# via
# -c base.txt
# bandit
# openapi-spec-validator
regex==2021.8.3
# via
# -c base.txt
# black
requests==2.26.0
# via
# -c base.txt
# django-silk
# responses
responses==0.13.4
# via -r dev.in
six==1.16.0
# via
# -c base.txt
# asttokens
# python-dateutil
# responses
# snoop
smmap==5.0.0
# via gitdb
snoop==0.4.1
# via -r dev.in
sqlparse==0.4.2
# via
# -c base.txt
# django
# django-silk
stevedore==3.5.0
# via bandit
toml==0.10.2
# via
# autopep8
# pytest
tomli==1.2.3
# via
# black
# pep517
typed-ast==1.5.2
# via black
typing-extensions==4.1.1
# via
# -c base.txt
# asgiref
# black
# gitpython
# importlib-metadata
# jsonschema
urllib3==1.26.8
# via
# -c base.txt
# requests
# responses
watchdog==2.1.4
# via -r dev.in
wheel==0.37.1
# via pip-tools
zipp==3.5.0
# via
# -c base.txt
# importlib-metadata
# importlib-resources
# pep517
# The following packages are considered to be unsafe in a requirements file:
# pip
# setuptools

View file

@ -1,156 +0,0 @@
advocate==1.0.0
aioredis==1.3.1
amqp==5.1.0
antlr4-python3-runtime==4.8
appdirs==1.4.4
argh==0.26.2
asgiref==3.4.1
asttokens==2.0.5
async-timeout==4.0.2
attrs==21.4.0
autobahn==22.2.2
Automat==20.2.0
autopep8==1.5.7
bandit==1.7.2
billiard==3.6.4.0
black==21.7b0
boto3==1.20.38
botocore==1.23.54
cached-property==1.5.2
celery==5.2.3
celery-redbeat==2.0.0
certifi==2021.10.8
cffi==1.15.0
channels==3.0.4
channels-redis==3.3.0
charset-normalizer==2.0.12
cheap-repr==0.5.1
click==8.0.4
click-didyoumean==0.3.0
click-plugins==1.1.1
click-repl==0.2.0
constantly==15.1.0
coverage==6.2
cryptography==36.0.1
daphne==3.0.2
Deprecated==1.2.13
dj-database-url==0.5.0
Django==3.2.12
django-appconf==1.0.5
django-celery-beat==2.2.1
django-celery-email==3.0.0
django-cors-headers==3.8.0
django-extensions==3.1.5
django-health-check==3.16.5
django-redis==5.2.0
django-silk==4.2.0
django-storages==1.12.3
django-timezone-field==4.2.3
djangorestframework==3.13.1
drf-jwt==1.19.1
drf-spectacular==0.21.2
execnet==1.9.0
executing==0.8.3
Faker==8.11.0
flake8==3.9.2
freezegun==1.1.0
gitdb==4.0.9
GitPython==3.1.27
gprof2dot==2021.2.21
gunicorn==20.1.0
h11==0.13.0
hiredis==2.0.0
httptools==0.2.0
hyperlink==21.0.0
icdiff==2.0.4
idna==3.3
importlib-metadata==4.11.2
importlib-resources==5.4.0
incremental==21.3.0
inflection==0.5.1
iniconfig==1.1.1
itsdangerous==2.0.1
Jinja2==3.0.3
jmespath==0.10.0
jsonschema==4.4.0
kombu==5.2.4
MarkupSafe==2.1.0
mccabe==0.6.1
msgpack==1.0.3
mypy-extensions==0.4.3
ndg-httpsclient==0.5.1
netifaces==0.11.0
openapi-schema-validator==0.2.3
openapi-spec-validator==0.4.0
packaging==21.3
pathspec==0.9.0
pbr==5.8.1
Pillow==9.0.0
pluggy==1.0.0
pprintpp==0.4.0
prompt-toolkit==3.0.28
psutil==5.9.0
psycopg2==2.9.1
py==1.11.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.7.0
pycparser==2.21
pyfakefs==4.5.0
pyflakes==2.3.1
Pygments==2.11.2
pyinstrument==4.0.3
PyJWT==2.3.0
pyOpenSSL==22.0.0
pyparsing==3.0.7
pyrsistent==0.18.1
pytest==6.2.5
pytest-asyncio==0.15.1
pytest-django==4.4.0
pytest-env==0.6.2
pytest-forked==1.4.0
pytest-html==3.1.1
pytest-icdiff==0.5
pytest-metadata==1.11.0
pytest-mock==3.6.1
pytest-ordering==0.6
pytest-split==0.6.0
pytest-xdist==2.3.0
python-crontab==2.6.0
python-dateutil==2.8.2
python-dotenv==0.19.2
pytz==2021.3
PyYAML==6.0
redis==4.1.4
regex==2021.8.3
requests==2.26.0
responses==0.13.4
s3transfer==0.5.2
service-identity==21.1.0
six==1.16.0
smmap==5.0.0
snoop==0.4.1
sqlparse==0.4.2
stevedore==3.5.0
tenacity==8.0.1
text-unidecode==1.3
toml==0.10.2
tomli==1.2.3
tqdm==4.62.3
Twisted==22.1.0
txaio==22.2.1
typed-ast==1.5.2
typing_extensions==4.1.1
unicodecsv==0.14.1
uritemplate==4.1.1
urllib3==1.26.8
uvicorn==0.15.0
uvloop==0.16.0
vine==5.0.0
watchdog==2.1.4
watchgod==0.7
wcwidth==0.2.5
websockets==10.2
wrapt==1.13.3
zipp==3.5.0
zope.interface==5.4.0

View file

@ -4,6 +4,7 @@
* Added group context menu to sidebar.
* Fixed Airtable import bug where the import would fail if a row is empty.
* Pin backend python dependencies using pip-tools.
## Released (2022-03-03 1.9.1)