archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-05 05:35:25 +00:00
Code Issues Projects Releases Wiki Activity

PostgreSQL data sync

Browse source
This commit is contained in:
Bram Wiepjes 2024-11-11 09:41:45 +00:00
parent 1ab02d416d
commit 9fd4f23608
27 changed files with 1457 additions and 176 deletions

14
backend/src/baserow/config/settings/base.py
View file

@ -817,7 +817,7 @@ BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT = int(
os.getenv("BASEROW_UNIQUE_ROW_VALUES_SIZE_LIMIT", 100)
)
# The amount of rows that can be imported when creating a table.
# The amount of rows that can be imported when creating a table or data sync.
INITIAL_TABLE_DATA_LIMIT = None
if "INITIAL_TABLE_DATA_LIMIT" in os.environ:
INITIAL_TABLE_DATA_LIMIT = int(os.getenv("INITIAL_TABLE_DATA_LIMIT"))
@ -1277,3 +1277,15 @@ BASEROW_OLLAMA_MODELS = os.getenv("BASEROW_OLLAMA_MODELS", "")
BASEROW_OLLAMA_MODELS = (
BASEROW_OLLAMA_MODELS.split(",") if BASEROW_OLLAMA_MODELS else []
)
BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE = str_to_bool(
os.getenv("BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE", "true")
)
BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST = os.getenv(
"BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST", ""
)
BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST = (
BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST.split(",")
if BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST
else []
)

3
backend/src/baserow/config/settings/test.py
View file

@ -83,3 +83,6 @@ PUBLIC_WEB_FRONTEND_URL = "http://localhost:3000"
BASEROW_EMBEDDED_SHARE_URL = "http://localhost:3000"
FEATURE_FLAGS = "*"
# We must allow this because we're connecting to the same database in the tests.
BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE = False

6
backend/src/baserow/contrib/database/apps.py
View file

@ -592,9 +592,13 @@ class DatabaseConfig(AppConfig):
airtable_column_type_registry.register(RichTextTextAirtableColumnType())
airtable_column_type_registry.register(CountAirtableColumnType())
from .data_sync.data_sync_types import ICalCalendarDataSyncType
from .data_sync.data_sync_types import (
ICalCalendarDataSyncType,
PostgreSQLDataSyncType,
)
data_sync_type_registry.register(ICalCalendarDataSyncType())
data_sync_type_registry.register(PostgreSQLDataSyncType())
from baserow.contrib.database.table.usage_types import (
TableWorkspaceStorageUsageItemType,

145
backend/src/baserow/contrib/database/data_sync/data_sync_types.py
View file

@ -1,143 +1,2 @@
from datetime import date, datetime, timezone
from typing import Any, Dict, List, Optional
import advocate
from advocate.exceptions import UnacceptableAddressException
from icalendar import Calendar
from requests.exceptions import RequestException
from baserow.contrib.database.fields.models import DateField, TextField
from baserow.core.utils import ChildProgressBuilder
from .exceptions import SyncError
from .models import ICalCalendarDataSync
from .registries import DataSyncProperty, DataSyncType
def normalize_datetime(d):
if d.tzinfo is None:
d = d.replace(tzinfo=timezone.utc)
else:
d = d.astimezone(timezone.utc)
d = d.replace(second=0, microsecond=0)
return d
def normalize_date(d):
if isinstance(d, datetime):
d = d.date()
return d
def compare_date(date1, date2):
if isinstance(date1, datetime) and isinstance(date2, datetime):
date1 = normalize_datetime(date1)
date2 = normalize_datetime(date2)
elif isinstance(date1, date) or isinstance(date2, date):
date1 = normalize_date(date1)
date2 = normalize_date(date2)
return date1 == date2
class UIDICalCalendarDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class DateStartICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = False
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class DateEndICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = False
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class SummaryICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class ICalCalendarDataSyncType(DataSyncType):
type = "ical_calendar"
model_class = ICalCalendarDataSync
allowed_fields = ["ical_url"]
serializer_field_names = ["ical_url"]
def get_properties(self, instance) -> List[DataSyncProperty]:
return [
UIDICalCalendarDataSyncProperty("uid", "Unique ID"),
DateStartICalCalendarDataSyncProperty("dtstart", "Start date"),
DateEndICalCalendarDataSyncProperty("dtend", "End date"),
SummaryICalCalendarDataSyncProperty("summary", "Summary"),
]
def get_all_rows(
self,
instance,
progress_builder: Optional[ChildProgressBuilder] = None,
) -> List[Dict]:
# The progress bar is difficult to setup because there are only three steps
# that must completed. We're therefore using working with a total of three
# because it gives some sense of what's going on.
progress = ChildProgressBuilder.build(progress_builder, child_total=3)
try:
response = advocate.get(instance.ical_url, timeout=60)
except (RequestException, UnacceptableAddressException, ConnectionError):
raise SyncError("The provided URL could not be reached.")
if not response.ok:
raise SyncError(
"The request to the URL didn't respond with an OK response code."
)
progress.increment(by=1) # makes the total `1`
try:
calendar = Calendar.from_ical(response.content)
except ValueError as e:
raise SyncError(f"Could not read calendar file: {str(e)}")
progress.increment(by=1) # makes the total `2`
events = [
{
"uid": str(component.get("uid")),
"dtstart": getattr(component.get("dtstart"), "dt", None),
"dtend": getattr(component.get("dtend"), "dt", None),
"summary": str(component.get("summary") or ""),
}
for component in calendar.walk()
if component.name == "VEVENT"
]
progress.increment(by=1) # makes the total `3`
return events
from .ical_data_sync_type import ICalCalendarDataSyncType # noqa: F401
from .postgresql_data_sync_type import PostgreSQLDataSyncType # noqa: F401

117
backend/src/baserow/contrib/database/data_sync/ical_data_sync_type.py Normal file
View file

@ -0,0 +1,117 @@
from typing import Any, Dict, List, Optional
import advocate
from advocate.exceptions import UnacceptableAddressException
from icalendar import Calendar
from requests.exceptions import RequestException
from baserow.contrib.database.fields.models import DateField, TextField
from baserow.core.utils import ChildProgressBuilder
from .exceptions import SyncError
from .models import ICalCalendarDataSync
from .registries import DataSyncProperty, DataSyncType
from .utils import compare_date
class UIDICalCalendarDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class DateStartICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = False
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class DateEndICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = False
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class SummaryICalCalendarDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class ICalCalendarDataSyncType(DataSyncType):
type = "ical_calendar"
model_class = ICalCalendarDataSync
allowed_fields = ["ical_url"]
serializer_field_names = ["ical_url"]
def get_properties(self, instance) -> List[DataSyncProperty]:
return [
UIDICalCalendarDataSyncProperty("uid", "Unique ID"),
DateStartICalCalendarDataSyncProperty("dtstart", "Start date"),
DateEndICalCalendarDataSyncProperty("dtend", "End date"),
SummaryICalCalendarDataSyncProperty("summary", "Summary"),
]
def get_all_rows(
self,
instance,
progress_builder: Optional[ChildProgressBuilder] = None,
) -> List[Dict]:
# The progress bar is difficult to setup because there are only three steps
# that must completed. We're therefore using working with a total of three
# because it gives some sense of what's going on.
progress = ChildProgressBuilder.build(progress_builder, child_total=3)
try:
response = advocate.get(instance.ical_url, timeout=60)
except (RequestException, UnacceptableAddressException, ConnectionError):
raise SyncError("The provided URL could not be reached.")
if not response.ok:
raise SyncError(
"The request to the URL didn't respond with an OK response code."
)
progress.increment(by=1) # makes the total `1`
try:
calendar = Calendar.from_ical(response.content)
except ValueError as e:
raise SyncError(f"Could not read calendar file: {str(e)}")
progress.increment(by=1) # makes the total `2`
events = [
{
"uid": str(component.get("uid")),
"dtstart": getattr(component.get("dtstart"), "dt", None),
"dtend": getattr(component.get("dtend"), "dt", None),
"summary": str(component.get("summary") or ""),
}
for component in calendar.walk()
if component.name == "VEVENT"
]
progress.increment(by=1) # makes the total `3`
return events

22
backend/src/baserow/contrib/database/data_sync/models.py
View file

@ -83,3 +83,25 @@ class SyncDataSyncTableJob(Job):
class ICalCalendarDataSync(DataSync):
ical_url = models.URLField(max_length=2000)
class PostgreSQLDataSync(DataSync):
postgresql_host = models.CharField(max_length=255)
postgresql_username = models.CharField(max_length=255)
postgresql_password = models.CharField(max_length=255)
postgresql_port = models.PositiveSmallIntegerField(default=5432)
postgresql_database = models.CharField(max_length=255)
postgresql_schema = models.CharField(max_length=255, default="public")
postgresql_table = models.CharField(max_length=255)
postgresql_sslmode = models.CharField(
max_length=12,
default="prefer",
choices=(
("disable", "disable"),
("allow", "allow"),
("prefer", "prefer"),
("require", "require"),
("verify-ca", "verify-ca"),
("verify-full", "verify-full"),
),
)

286
backend/src/baserow/contrib/database/data_sync/postgresql_data_sync_type.py Normal file
View file

@ -0,0 +1,286 @@
import contextlib
from typing import Any, Dict, List, Optional
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS
import psycopg2
from psycopg2 import sql
from baserow.contrib.database.fields.models import (
NUMBER_MAX_DECIMAL_PLACES,
BooleanField,
DateField,
LongTextField,
NumberField,
TextField,
)
from baserow.core.utils import ChildProgressBuilder, are_hostnames_same
from .exceptions import SyncError
from .models import PostgreSQLDataSync
from .registries import DataSyncProperty, DataSyncType
from .utils import compare_date
class BasePostgreSQLSyncProperty(DataSyncProperty):
decimal_places = None
def prepare_value(self, value):
return value
class TextPostgreSQLSyncProperty(BasePostgreSQLSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class LongTextPostgreSQLSyncProperty(BasePostgreSQLSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> LongTextField:
return LongTextField(name=self.name)
class BooleanPostgreSQLSyncProperty(BasePostgreSQLSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> BooleanField:
return BooleanField(name=self.name)
def prepare_value(self, value):
return bool(value)
class NumberPostgreSQLSyncProperty(BasePostgreSQLSyncProperty):
immutable_properties = False
decimal_places = 0
def __init__(self, key, name):
super().__init__(key, name)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
try:
return round(baserow_row_value, self.decimal_places) == round(
data_sync_row_value, self.decimal_places
)
except TypeError:
return super().is_equal(baserow_row_value, data_sync_row_value)
def to_baserow_field(self) -> NumberField:
return NumberField(
name=self.name,
number_decimal_places=min(
self.decimal_places or 0, NUMBER_MAX_DECIMAL_PLACES
),
number_negative=True,
)
class DatePostgreSQLSyncProperty(BasePostgreSQLSyncProperty):
immutable_properties = False
include_time = False
def is_equal(self, baserow_row_value, data_sync_row_value) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=self.include_time,
date_time_format="24",
date_show_tzinfo=True,
)
class DateTimePostgreSQLSyncProperty(DatePostgreSQLSyncProperty):
include_time = True
# The key of the mapping must be in the column type to map to it. `integer` column
# type would therefore map to `NumberPostgreSQLSyncProperty`.
column_type_to_baserow_field_type = {
"int": NumberPostgreSQLSyncProperty,
"serial": NumberPostgreSQLSyncProperty,
"boolean": BooleanPostgreSQLSyncProperty,
"real": NumberPostgreSQLSyncProperty,
"double": NumberPostgreSQLSyncProperty,
"numeric": NumberPostgreSQLSyncProperty,
"char": TextPostgreSQLSyncProperty,
"text": LongTextPostgreSQLSyncProperty,
"date": DatePostgreSQLSyncProperty,
"timestamp": DateTimePostgreSQLSyncProperty,
"uuid": TextPostgreSQLSyncProperty,
}
class PostgreSQLDataSyncType(DataSyncType):
type = "postgresql"
model_class = PostgreSQLDataSync
allowed_fields = [
"postgresql_host",
"postgresql_username",
"postgresql_password",
"postgresql_port",
"postgresql_database",
"postgresql_schema",
"postgresql_table",
"postgresql_sslmode",
]
serializer_field_names = [
"postgresql_host",
"postgresql_username",
"postgresql_password",
"postgresql_port",
"postgresql_database",
"postgresql_schema",
"postgresql_table",
"postgresql_sslmode",
]
@contextlib.contextmanager
def _connection(self, instance):
cursor = None
connection = None
baserow_postgresql_connection = (
settings.BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE
and are_hostnames_same(
instance.postgresql_host, settings.DATABASES[DEFAULT_DB_ALIAS]["HOST"]
)
)
data_sync_blacklist = any(
are_hostnames_same(instance.postgresql_host, hostname)
for hostname in settings.BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST
)
if baserow_postgresql_connection or data_sync_blacklist:
raise SyncError("It's not allowed to connect to this hostname.")
try:
connection = psycopg2.connect(
host=instance.postgresql_host,
dbname=instance.postgresql_database,
user=instance.postgresql_username,
password=instance.postgresql_password,
port=instance.postgresql_port,
sslmode=instance.postgresql_sslmode,
)
cursor = connection.cursor()
yield cursor
except psycopg2.Error as e:
raise SyncError(str(e))
finally:
if cursor:
cursor.close()
if connection:
connection.close()
def _get_table_columns(self, instance):
with self._connection(instance) as cursor:
exists_query = """
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = %s
AND table_name = %s
);
"""
cursor.execute(
exists_query, (instance.postgresql_schema, instance.postgresql_table)
)
exists = cursor.fetchone()
if not exists[0]:
raise SyncError(
f"The table {instance.postgresql_table} does not exist."
)
primary_query = """
SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
WHERE tc.table_schema = %s
AND tc.table_name = %s
AND tc.constraint_type = 'PRIMARY KEY';
"""
cursor.execute(
primary_query, (instance.postgresql_schema, instance.postgresql_table)
)
primary_columns = [result[0] for result in cursor.fetchall()]
columns_query = """
SELECT column_name, data_type, numeric_scale
FROM information_schema.columns
WHERE table_schema = %s
AND table_name = %s;
"""
cursor.execute(
columns_query, (instance.postgresql_schema, instance.postgresql_table)
)
columns = cursor.fetchall()
return primary_columns, columns
def get_properties(self, instance) -> List[DataSyncProperty]:
primary_columns, columns = self._get_table_columns(instance)
properties = []
for column_name, column_type, decimal_places in columns:
is_primary = column_name in primary_columns
property_class = [
column_type_to_baserow_field_type[key]
for key in column_type_to_baserow_field_type.keys()
if key.lower() in column_type.lower()
]
if len(property_class) == 0:
continue
property_class = property_class[0]
property_instance = property_class(column_name, column_name)
property_instance.unique_primary = is_primary
property_instance.decimal_places = decimal_places or 0
properties.append(property_instance)
return properties
def get_all_rows(
self,
instance,
progress_builder: Optional[ChildProgressBuilder] = None,
) -> List[Dict]:
table_name = instance.postgresql_table
properties = self.get_properties(instance)
order_names = [p.key for p in properties if p.unique_primary]
column_names = [p.key for p in properties]
with self._connection(instance) as cursor:
count_query = sql.SQL("SELECT count(*) FROM {}").format(
sql.Identifier(table_name)
)
cursor.execute(count_query)
count = cursor.fetchone()[0]
limit = settings.INITIAL_TABLE_DATA_LIMIT
if limit and count > settings.INITIAL_TABLE_DATA_LIMIT:
raise SyncError(f"The table can't contain more than {limit} records.")
select_query = sql.SQL("SELECT {} FROM {} ORDER BY {}").format(
sql.SQL(", ").join(map(sql.Identifier, column_names)),
sql.Identifier(table_name),
sql.SQL(", ").join(map(sql.Identifier, order_names)),
)
cursor.execute(select_query)
records = cursor.fetchall()
return [
{
p.key: p.prepare_value(record[index])
for index, p in enumerate(properties)
}
for record in records
]

27
backend/src/baserow/contrib/database/data_sync/utils.py Normal file
View file

@ -0,0 +1,27 @@
from datetime import date, datetime, timezone
def normalize_datetime(d):
if d.tzinfo is None:
d = d.replace(tzinfo=timezone.utc)
else:
d = d.astimezone(timezone.utc)
d = d.replace(second=0, microsecond=0)
return d
def normalize_date(d):
if isinstance(d, datetime):
d = d.date()
return d
def compare_date(date1, date2):
if isinstance(date1, datetime) and isinstance(date2, datetime):
date1 = normalize_datetime(date1)
date2 = normalize_datetime(date2)
elif isinstance(date1, date) or isinstance(date2, date):
date1 = normalize_date(date1)
date2 = normalize_date(date2)
return date1 == date2

58
backend/src/baserow/contrib/database/migrations/0172_postgresqldatasync.py Normal file
View file

@ -0,0 +1,58 @@
# Generated by Django 5.0.9 on 2024-11-06 20:48
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("database", "0171_alter_formview_submit_action_redirect_url"),
]
operations = [
migrations.CreateModel(
name="PostgreSQLDataSync",
fields=[
(
"datasync_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="database.datasync",
),
),
("postgresql_host", models.CharField(max_length=255)),
("postgresql_username", models.CharField(max_length=255)),
("postgresql_password", models.CharField(max_length=255)),
("postgresql_port", models.PositiveSmallIntegerField(default=5432)),
("postgresql_database", models.CharField(max_length=255)),
(
"postgresql_schema",
models.CharField(default="public", max_length=255),
),
("postgresql_table", models.CharField(max_length=255)),
(
"postgresql_sslmode",
models.CharField(
choices=[
("disable", "disable"),
("allow", "allow"),
("prefer", "prefer"),
("require", "require"),
("verify-ca", "verify-ca"),
("verify-full", "verify-full"),
],
default="prefer",
max_length=12,
),
),
],
options={
"abstract": False,
},
bases=("database.datasync",),
),
]

36
backend/src/baserow/core/utils.py
View file

@ -7,13 +7,14 @@ import math
import os
import random
import re
import socket
import string
from collections import defaultdict, namedtuple
from decimal import Decimal
from fractions import Fraction
from itertools import chain, islice
from numbers import Number
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
from django.conf import settings
from django.db import transaction
@ -1139,3 +1140,36 @@ def merge_dicts_no_duplicates(*dicts):
merged_dict[key] = dictionary[key]
return merged_dict
def get_all_ips(hostname: str) -> Set:
"""
Returns a set of all IP addresses of the provided hostname.
:param hostname: The hostname where to get the IP addresses from.
:return: A set containing the IP addresses of the hostname.
"""
try:
addr_info = socket.getaddrinfo(hostname, None)
# Extract unique IP addresses from addr_info (both IPv4 and IPv6)
ips = {info[4][0] for info in addr_info}
return ips
except socket.gaierror:
return set()
def are_hostnames_same(hostname1: str, hostname2: str) -> bool:
"""
Resolves the IP addresses of both hostnames, and checks they resolve to the same IP
address. In this case, `are_hostnames_same("localhost", "localhost") == True`
because they're both resolving to the same IP.
:param hostname1: First hostname to compare.
:param hostname2: Second hostname to compare
:return: True if the hostnames point to the same IP.
"""
ips1 = get_all_ips(hostname1)
ips2 = get_all_ips(hostname2)
return not ips1.isdisjoint(ips2)

10
backend/tests/baserow/contrib/database/data_sync/test_data_sync_handler.py
View file

@ -7,16 +7,16 @@ import pytest
import responses
from freezegun import freeze_time
from baserow.contrib.database.data_sync.data_sync_types import (
ICalCalendarDataSyncType,
UIDICalCalendarDataSyncProperty,
)
from baserow.contrib.database.data_sync.exceptions import (
PropertyNotFound,
SyncDataSyncTableAlreadyRunning,
UniquePrimaryPropertyNotFound,
)
from baserow.contrib.database.data_sync.handler import DataSyncHandler
from baserow.contrib.database.data_sync.ical_data_sync_type import (
ICalCalendarDataSyncType,
UIDICalCalendarDataSyncProperty,
)
from baserow.contrib.database.data_sync.models import (
DataSync,
DataSyncSyncedProperty,
@ -752,7 +752,7 @@ def test_sync_data_sync_table_sync_error(data_fixture):
@pytest.mark.django_db
@patch(
"baserow.contrib.database.data_sync.data_sync_types.ICalCalendarDataSyncType.get_all_rows"
"baserow.contrib.database.data_sync.ical_data_sync_type.ICalCalendarDataSyncType.get_all_rows"
)
def test_sync_data_sync_table_exception_raised(mock_get_all_rows, data_fixture):
mock_get_all_rows.side_effect = ValueError

2
backend/tests/baserow/contrib/database/data_sync/test_data_sync_ical_calendar_type.py
View file

@ -5,9 +5,9 @@ import pytest
import responses
from freezegun import freeze_time
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.handler import DataSyncHandler
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.utils import compare_date
ICAL_FEED_WITH_ONE_ITEMS_WITHOUT_DTEND = """BEGIN:VCALENDAR
VERSION:2.0

619
backend/tests/baserow/contrib/database/data_sync/test_data_sync_postgresql.py Normal file
View file

@ -0,0 +1,619 @@
from datetime import date, datetime, timezone
from decimal import Decimal
from unittest.mock import patch
from django.conf import settings
from django.db import connection, transaction
from django.test.utils import override_settings
from django.urls import reverse
import pytest
from rest_framework.status import HTTP_200_OK
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.handler import DataSyncHandler
from baserow.contrib.database.data_sync.models import PostgreSQLDataSync
from baserow.contrib.database.data_sync.postgresql_data_sync_type import (
TextPostgreSQLSyncProperty,
)
from baserow.contrib.database.fields.models import NumberField
from baserow.core.db import specific_iterator
# Fixture to create a test table
@pytest.fixture
def create_postgresql_test_table():
table_name = "test_table"
column_definitions = {
"text_col": "TEXT",
"char_col": "CHAR(10)",
"int_col": "INTEGER",
"float_col": "REAL",
"numeric_col": "NUMERIC",
"numeric2_col": "NUMERIC(100, 4)",
"smallint_col": "SMALLINT",
"bigint_col": "BIGINT",
"decimal_col": "DECIMAL",
"date_col": "DATE",
"datetime_col": "TIMESTAMP",
"boolean_col": "BOOLEAN",
}
# Create the schema of the initial table.
create_table_sql = f"""
CREATE TABLE {table_name} (
id SERIAL PRIMARY KEY,
{', '.join([f"{col_name} {col_type}" for col_name, col_type in column_definitions.items()])}
)
"""
# Inserts a couple of random rows for testing purposes.
insert_sql = f"""
INSERT INTO {table_name} ({', '.join(column_definitions.keys())})
VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
)
"""
try:
with connection.cursor() as cursor:
cursor.execute(create_table_sql)
cursor.execute(
insert_sql,
(
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas non nunc et sapien ultricies blandit. ",
"Short char",
10,
10.10,
100,
"10.4444",
200,
99999999,
Decimal("99999999.22"),
date(2023, 1, 17),
datetime(2022, 2, 28, 12, 00),
True,
),
)
cursor.execute(
insert_sql,
(
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
),
)
transaction.commit()
yield table_name # Provide table name to tests that need to access it
finally:
# Drop the table after test completes or fails
with connection.cursor() as cursor:
cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
transaction.commit()
@pytest.mark.django_db(transaction=True)
def test_create_postgresql_data_sync(data_fixture, create_postgresql_test_table):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=[
"id",
"text_col",
"char_col",
"int_col",
"numeric_col",
"numeric2_col",
"smallint_col",
"bigint_col",
"decimal_col",
"date_col",
"datetime_col",
"boolean_col",
],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table=create_postgresql_test_table,
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
assert isinstance(data_sync, PostgreSQLDataSync)
assert data_sync.postgresql_host == default_database["HOST"]
assert data_sync.postgresql_username == default_database["USER"]
assert data_sync.postgresql_password == default_database["PASSWORD"]
assert data_sync.postgresql_database == default_database["NAME"]
assert data_sync.postgresql_port == default_database["PORT"]
assert data_sync.postgresql_schema == "public"
assert data_sync.postgresql_table == create_postgresql_test_table
assert data_sync.postgresql_sslmode == default_database["OPTIONS"].get(
"sslmode", "prefer"
)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
assert len(fields) == 12
assert fields[0].name == "id"
assert isinstance(fields[0], NumberField)
assert fields[0].primary is True
assert fields[0].read_only is True
assert fields[0].immutable_type is True
assert fields[0].immutable_properties is False
assert fields[0].number_decimal_places == 0
@pytest.mark.django_db(transaction=True)
def test_sync_postgresql_data_sync(data_fixture, create_postgresql_test_table):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=[
"id",
"text_col",
"char_col",
"int_col",
"float_col",
"numeric_col",
"numeric2_col",
"smallint_col",
"bigint_col",
"decimal_col",
"date_col",
"datetime_col",
"boolean_col",
],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table=create_postgresql_test_table,
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
for f in fields:
print(f.name)
id_field = fields[0]
text_field = fields[1]
char_field = fields[2]
int_field = fields[3]
float_field = fields[4]
numeric_field = fields[5]
numeric2_field = fields[6]
smallint_field = fields[7]
bigint_field = fields[8]
decimal_field = fields[9]
date_field = fields[10]
datetime_field = fields[11]
boolean_field = fields[12]
model = data_sync.table.get_model()
rows = list(model.objects.all())
assert len(rows) == 2
filled_row = rows[0]
empty_row = rows[1]
assert getattr(filled_row, f"field_{id_field.id}") == 1
assert (
getattr(filled_row, f"field_{text_field.id}")
== "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas non nunc et sapien ultricies blandit. "
)
assert getattr(filled_row, f"field_{char_field.id}") == "Short char"
assert getattr(filled_row, f"field_{int_field.id}") == 10
# Decimal places are not set if no scale is provided in the column type.
assert getattr(filled_row, f"field_{float_field.id}") == Decimal("10")
assert getattr(filled_row, f"field_{numeric_field.id}") == 100
# Decimal places should be set because it's defined in column type.
assert getattr(filled_row, f"field_{numeric2_field.id}") == Decimal("10.4444")
assert getattr(filled_row, f"field_{smallint_field.id}") == 200
assert getattr(filled_row, f"field_{bigint_field.id}") == 99999999
# Decimal places are not set if no scale is provided in the column type.
assert getattr(filled_row, f"field_{decimal_field.id}") == Decimal("99999999")
assert getattr(filled_row, f"field_{date_field.id}") == date(2023, 1, 17)
assert getattr(filled_row, f"field_{datetime_field.id}") == datetime(
2022, 2, 28, 12, 0, tzinfo=timezone.utc
)
assert getattr(filled_row, f"field_{boolean_field.id}") is True
assert getattr(empty_row, f"field_{id_field.id}") == 2
assert getattr(empty_row, f"field_{text_field.id}") is None
assert getattr(empty_row, f"field_{char_field.id}") is None
assert getattr(empty_row, f"field_{int_field.id}") is None
assert getattr(empty_row, f"field_{float_field.id}") is None
assert getattr(empty_row, f"field_{numeric_field.id}") is None
assert getattr(empty_row, f"field_{numeric2_field.id}") is None
assert getattr(empty_row, f"field_{smallint_field.id}") is None
assert getattr(empty_row, f"field_{bigint_field.id}") is None
assert getattr(empty_row, f"field_{decimal_field.id}") is None
assert getattr(empty_row, f"field_{date_field.id}") is None
assert getattr(empty_row, f"field_{datetime_field.id}") is None
assert getattr(empty_row, f"field_{boolean_field.id}") is False
@pytest.mark.django_db(transaction=True)
def test_sync_postgresql_data_sync_nothing_changed(
data_fixture, create_postgresql_test_table
):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=[
"id",
"text_col",
"char_col",
"int_col",
"float_col",
"numeric_col",
"numeric2_col",
"smallint_col",
"bigint_col",
"decimal_col",
"date_col",
"datetime_col",
"boolean_col",
],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table=create_postgresql_test_table,
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
with transaction.atomic():
handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
rows = list(model.objects.all())
row_1 = rows[0]
row_2 = rows[1]
row_1_last_modified = row_1.updated_on
row_2_last_modified = row_2.updated_on
with transaction.atomic():
handler.sync_data_sync_table(user=user, data_sync=data_sync)
row_1.refresh_from_db()
row_2.refresh_from_db()
# Because none of the values have changed in the source (interesting) table,
# we don't expect the rows to have been updated. If they have been updated,
# it means that the `is_equal` method of `BaserowFieldDataSyncProperty` is not
# working as expected.
assert row_1.updated_on == row_1_last_modified
assert row_2.updated_on == row_2_last_modified
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_get_properties(
data_fixture, api_client, create_postgresql_test_table
):
default_database = settings.DATABASES["default"]
user, token = data_fixture.create_user_and_token()
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "postgresql",
"postgresql_host": default_database["HOST"],
"postgresql_username": default_database["USER"],
"postgresql_password": default_database["PASSWORD"],
"postgresql_port": default_database["PORT"],
"postgresql_database": default_database["NAME"],
"postgresql_table": create_postgresql_test_table,
"postgresql_sslmode": default_database["OPTIONS"].get("sslmode", "prefer"),
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{"unique_primary": True, "key": "id", "name": "id", "field_type": "number"},
{
"unique_primary": False,
"key": "text_col",
"name": "text_col",
"field_type": "long_text",
},
{
"unique_primary": False,
"key": "char_col",
"name": "char_col",
"field_type": "text",
},
{
"unique_primary": False,
"key": "int_col",
"name": "int_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "float_col",
"name": "float_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "numeric_col",
"name": "numeric_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "numeric2_col",
"name": "numeric2_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "smallint_col",
"name": "smallint_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "bigint_col",
"name": "bigint_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "decimal_col",
"name": "decimal_col",
"field_type": "number",
},
{
"unique_primary": False,
"key": "date_col",
"name": "date_col",
"field_type": "date",
},
{
"unique_primary": False,
"key": "datetime_col",
"name": "datetime_col",
"field_type": "date",
},
{
"unique_primary": False,
"key": "boolean_col",
"name": "boolean_col",
"field_type": "boolean",
},
]
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_get_properties_unsupported_column_types(
data_fixture, api_client, create_postgresql_test_table
):
default_database = settings.DATABASES["default"]
user, token = data_fixture.create_user_and_token()
with patch(
"baserow.contrib.database.data_sync.postgresql_data_sync_type.column_type_to_baserow_field_type",
new={
"char": TextPostgreSQLSyncProperty,
},
):
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "postgresql",
"postgresql_host": default_database["HOST"],
"postgresql_username": default_database["USER"],
"postgresql_password": default_database["PASSWORD"],
"postgresql_port": default_database["PORT"],
"postgresql_database": default_database["NAME"],
"postgresql_table": create_postgresql_test_table,
"postgresql_sslmode": default_database["OPTIONS"].get(
"sslmode", "prefer"
),
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{
"unique_primary": False,
"key": "char_col",
"name": "char_col",
"field_type": "text",
}
]
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_table_connect_to_same_database(data_fixture):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
with override_settings(
BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE=True
):
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=["id"],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table="test_table",
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
# This is expected to fail because `postgresql_host` is equal to the
# default_database["HOST"] and that's not allowed if
# BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE=True.
assert str(e.value) == "It's not allowed to connect to this hostname."
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_table_connect_to_blacklist(data_fixture):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
with override_settings(
BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST=["localhost", "baserow.io"]
):
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=["id"],
postgresql_host="localhost",
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table="test_table",
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
# This is expected to fail because `postgresql_host` is equal to the to one of the
# hostnames in BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST.
assert str(e.value) == "It's not allowed to connect to this hostname."
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_table_does_not_exist(data_fixture):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=["id"],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table="test_table_DOES_NOT_EXIST",
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
assert str(e.value) == "The table test_table_DOES_NOT_EXIST does not exist."
@pytest.mark.django_db
def test_postgresql_data_sync_connection_error(data_fixture):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=["id"],
postgresql_host=default_database["HOST"],
postgresql_username="NOT_EXISTING_USER",
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table="test_table_DOES_NOT_EXIST",
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
assert "failed" in str(e.value)
@pytest.mark.django_db(transaction=True)
def test_postgresql_data_sync_initial_table_limit(
data_fixture, create_postgresql_test_table
):
default_database = settings.DATABASES["default"]
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="postgresql",
synced_properties=["id"],
postgresql_host=default_database["HOST"],
postgresql_username=default_database["USER"],
postgresql_password=default_database["PASSWORD"],
postgresql_port=default_database["PORT"],
postgresql_database=default_database["NAME"],
postgresql_table=create_postgresql_test_table,
postgresql_sslmode=default_database["OPTIONS"].get("sslmode", "prefer"),
)
with override_settings(INITIAL_TABLE_DATA_LIMIT=1):
handler.sync_data_sync_table(user=user, data_sync=data_sync)
assert data_sync.last_sync is None
assert data_sync.last_error == "The table can't contain more than 1 records."

11
backend/tests/baserow/core/test_core_utils.py
View file

@ -15,12 +15,14 @@ from baserow.core.utils import (
ChildProgressBuilder,
MirrorDict,
Progress,
are_hostnames_same,
atomic_if_not_already,
dict_to_object,
escape_csv_cell,
extract_allowed,
find_intermediate_order,
find_unused_name,
get_all_ips,
get_baserow_saas_base_url,
get_value_at_path,
grouper,
@ -650,3 +652,12 @@ def test_get_baserow_saas_base_url_with_debug():
def test_remove_duplicates():
assert remove_duplicates([1, 2, 2, 3]) == [1, 2, 3]
def test_get_all_ips():
assert get_all_ips("localhost") == {"127.0.0.1", "::1"}
def test_are_hostnames_same():
assert are_hostnames_same("localhost", "localhost") is True
assert are_hostnames_same("baserow.io", "localhost") is False

7
changelog/entries/unreleased/feature/3079_postgresql_data_sync.json Normal file
View file

@ -0,0 +1,7 @@
{
"type": "feature",
"message": "PostgreSQL data sync.",
"issue_number": 3079,
"bullet_points": [],
"created_at": "2024-11-04"
}

2
docker-compose.yml
View file

@ -184,6 +184,8 @@ x-backend-variables: &backend-variables
BASEROW_ICAL_VIEW_MAX_EVENTS: ${BASEROW_ICAL_VIEW_MAX_EVENTS:-}
BASEROW_ACCESS_TOKEN_LIFETIME_MINUTES:
BASEROW_REFRESH_TOKEN_LIFETIME_HOURS:
BASEROW_PREVENT_POSTGRESQL_DATA_SYNC_CONNECTION_TO_DATABASE:
BASEROW_POSTGRESQL_DATA_SYNC_BLACKLIST:
BASEROW_ASGI_HTTP_MAX_CONCURRENCY: ${BASEROW_ASGI_HTTP_MAX_CONCURRENCY:-}

38
docs/installation/configuration.md
View file

@ -63,24 +63,26 @@ The installation methods referred to in the variable descriptions are:
| BASEROW\_ASGI\_HTTP\_MAX\_CONCURRENCY | Specifies a limit for concurrent requests handled by a single gunicorn worker. The default is: no limit. | |
### Backend Database Configuration
| Name | Description | Defaults |
|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| DATABASE\_HOST | The hostname of the postgres database Baserow will use to store its data in. | Defaults to db in the standalone and compose installs. If not provided in the \`baserow/baserow\` install then the embedded Postgres will be setup and used. |
| DATABASE\_USER | The username of the database user Baserow will use to connect to the database at DATABASE\_HOST | baserow |
| | | |
| DATABASE\_PORT | The port Baserow will use when trying to connect to the postgres database at DATABASE\_HOST | 5432 |
| DATABASE\_NAME | The database name Baserow will use to store data in. | baserow |
| DATABASE\_PASSWORD | The password of DATABASE\_USER on the postgres server at DATABASE\_HOST | Required to be set by you in the docker-compose and standalone installs. Automatically generated by the baserow/baserow image if not provided and stored in /baserow/data/.pgpass. |
| DATABASE\_PASSWORD\_FILE | **Only supported by the `baserow/baserow` image** If set Baserow will attempt to read the above DATABASE\_PASSWORD from this file location instead. | |
| DATABASE\_OPTIONS | Optional extra options as a JSON formatted string to use when connecting to the database, see [this documentation](https://docs.djangoproject.com/en/3.2/ref/settings/#std-setting-OPTIONS) for more details. | |
| DATABASE\_URL | Alternatively to setting the individual DATABASE\_ parameters above instead you can provide one standard postgres connection string in the format of: postgresql://\[user\[:password\]@\]\[netloc\]\[:port\]\[/dbname\]\[?param1=value1&…\]. Please note this will completely override all other DATABASE_* settings and ignore them. | |
| | | |
| MIGRATE\_ON\_STARTUP | If set to “true” when the Baserow backend service starts up it will automatically apply database migrations. Set to any other value to disable. If you disable this then you must remember to manually apply the database migrations when upgrading Baserow to a new version. | true |
| BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION | If set to “true” when after a migration Baserow will automatically sync all builtin Baserow templates in the background. If you are using a postgres database which is constrained to fewer than 10000 rows then we recommend you disable this as the Baserow templates will go over that row limit. To disable this set to any other value than “true” | true |
| BASEROW\_SYNC\_TEMPLATES\_TIME\_LIMIT | The number of seconds before the background sync templates job will timeout if not yet completed. | 1800 |
| SYNC\_TEMPLATES\_ON\_STARTUP | **Deprecated please use BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION** If provided has the same effect of BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION for backwards compatibility reasons. If BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION is set it will override this value. | true |
| DONT\_UPDATE\_FORMULAS\_AFTER\_MIGRATION | Baserow’s formulas have an internal version number. When upgrading Baserow if the formula language has also changed then after the database migration has run Baserow will also automatically recalculate all formulas if they have a different version. Set this to any non empty value to disable this automatic update if you would prefer to run the update\_formulas management command manually yourself. Formulas might break if you forget to do so after an upgrade of Baserow until and so it is recommended to leave this empty. | |
| POSTGRES\_STARTUP\_CHECK\_ATTEMPTS | When Baserow's Backend service starts up it first checks to see if the postgres database is available. It checks 5 times by default, after which if it still has not connected it will crash. | 5 |
| Name | Description | Defaults |
|---------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| DATABASE\_HOST | The hostname of the postgres database Baserow will use to store its data in. | Defaults to db in the standalone and compose installs. If not provided in the \`baserow/baserow\` install then the embedded Postgres will be setup and used. |
| DATABASE\_USER | The username of the database user Baserow will use to connect to the database at DATABASE\_HOST | baserow |
| | | |
| DATABASE\_PORT | The port Baserow will use when trying to connect to the postgres database at DATABASE\_HOST | 5432 |
| DATABASE\_NAME | The database name Baserow will use to store data in. | baserow |
| DATABASE\_PASSWORD | The password of DATABASE\_USER on the postgres server at DATABASE\_HOST | Required to be set by you in the docker-compose and standalone installs. Automatically generated by the baserow/baserow image if not provided and stored in /baserow/data/.pgpass. |
| DATABASE\_PASSWORD\_FILE | **Only supported by the `baserow/baserow` image** If set Baserow will attempt to read the above DATABASE\_PASSWORD from this file location instead. | |
| DATABASE\_OPTIONS | Optional extra options as a JSON formatted string to use when connecting to the database, see [this documentation](https://docs.djangoproject.com/en/3.2/ref/settings/#std-setting-OPTIONS) for more details. | |
| DATABASE\_URL | Alternatively to setting the individual DATABASE\_ parameters above instead you can provide one standard postgres connection string in the format of: postgresql://\[user\[:password\]@\]\[netloc\]\[:port\]\[/dbname\]\[?param1=value1&…\]. Please note this will completely override all other DATABASE_* settings and ignore them. | |
| | | |
| MIGRATE\_ON\_STARTUP | If set to “true” when the Baserow backend service starts up it will automatically apply database migrations. Set to any other value to disable. If you disable this then you must remember to manually apply the database migrations when upgrading Baserow to a new version. | true |
| BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION | If set to “true” when after a migration Baserow will automatically sync all builtin Baserow templates in the background. If you are using a postgres database which is constrained to fewer than 10000 rows then we recommend you disable this as the Baserow templates will go over that row limit. To disable this set to any other value than “true” | true |
| BASEROW\_SYNC\_TEMPLATES\_TIME\_LIMIT | The number of seconds before the background sync templates job will timeout if not yet completed. | 1800 |
| SYNC\_TEMPLATES\_ON\_STARTUP | **Deprecated please use BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION** If provided has the same effect of BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION for backwards compatibility reasons. If BASEROW\_TRIGGER\_SYNC\_TEMPLATES\_AFTER\_MIGRATION is set it will override this value. | true |
| DONT\_UPDATE\_FORMULAS\_AFTER\_MIGRATION | Baserow’s formulas have an internal version number. When upgrading Baserow if the formula language has also changed then after the database migration has run Baserow will also automatically recalculate all formulas if they have a different version. Set this to any non empty value to disable this automatic update if you would prefer to run the update\_formulas management command manually yourself. Formulas might break if you forget to do so after an upgrade of Baserow until and so it is recommended to leave this empty. | |
| POSTGRES\_STARTUP\_CHECK\_ATTEMPTS | When Baserow's Backend service starts up it first checks to see if the postgres database is available. It checks 5 times by default, after which if it still has not connected it will crash. | 5 |
| BASEROW\_PREVENT\_POSTGRESQL\_DATA\_SYNC\_CONNECTION\_\TO\_DATABASE | If true, then it's impossible to connect to the Baserow PostgreSQL database using the PostgreSQL data sync. | true |
| BASEROW\_POSTGRESQL\_DATA\_SYNC\_BLACKLIST | Optionally provide a comma separated list of hostnames that the Baserow PostgreSQL data sync can't connect to. (e.g. "localhost,baserow.io") | |
### Redis Configuration
| Name | Description | Defaults |

2
enterprise/backend/src/baserow_enterprise/data_sync/baserow_table_data_sync.py
View file

@ -6,10 +6,10 @@ from baserow_premium.fields.field_types import AIFieldType
from baserow_premium.license.handler import LicenseHandler
from rest_framework import serializers
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.data_sync.utils import compare_date
from baserow.contrib.database.fields.field_types import (
AutonumberFieldType,
BooleanFieldType,

2
enterprise/backend/src/baserow_enterprise/data_sync/github_issues_data_sync.py
View file

@ -5,9 +5,9 @@ import requests
from baserow_premium.license.handler import LicenseHandler
from requests.exceptions import JSONDecodeError, RequestException
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.data_sync.utils import compare_date
from baserow.contrib.database.fields.models import (
DateField,
LongTextField,

2
enterprise/backend/src/baserow_enterprise/data_sync/jira_issues_data_sync.py
View file

@ -9,9 +9,9 @@ from jira2markdown import convert
from requests.auth import HTTPBasicAuth
from requests.exceptions import JSONDecodeError, RequestException
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.data_sync.utils import compare_date
from baserow.contrib.database.fields.models import (
DateField,
LongTextField,

10
web-frontend/modules/core/assets/icons/postgresql.svg Normal file
View file

File diff suppressed because one or more lines are too long
Side by side

After

(image error) Size: 10 KiB

3
web-frontend/modules/core/assets/scss/variables.scss
View file

@ -79,7 +79,8 @@ $baserow-icons: 'circle-empty', 'circle-checked', 'check-square', 'formula',
'kanban', 'file-word', 'file-archive', 'gallery', 'file-powerpoint',
'calendar', 'smile', 'smartphone', 'plus', 'heading-1', 'heading-2',
'heading-3', 'paragraph', 'ordered-list', 'enlarge', 'share', 'settings',
'up-down-arrows', 'application', 'groups', 'timeline', 'dashboard', 'jira';
'up-down-arrows', 'application', 'groups', 'timeline', 'dashboard', 'jira',
'postgresql';
$grid-view-row-height-small: 33px;
$grid-view-row-height-medium: 55px;

162
web-frontend/modules/database/components/dataSync/PostgreSQLDataSync.vue Normal file
View file

@ -0,0 +1,162 @@
<template>
<form @submit.prevent="submit">
<p class="margin-bottom-2 margin-top-3">
{{ $t('postgreSQLDataSync.description') }}
</p>
<FormGroup
v-for="field in [
{ name: 'postgresql_host', translationPrefix: 'host', type: 'text' },
{
name: 'postgresql_username',
translationPrefix: 'username',
type: 'text',
},
{
name: 'postgresql_password',
translationPrefix: 'password',
type: 'password',
},
{
name: 'postgresql_database',
translationPrefix: 'database',
type: 'text',
},
{
name: 'postgresql_schema',
translationPrefix: 'schema',
type: 'text',
},
{ name: 'postgresql_table', translationPrefix: 'table', type: 'text' },
]"
:key="field.name"
:error="fieldHasErrors(field.name)"
required
small-label
class="margin-bottom-2"
>
<template #label>{{
$t(`postgreSQLDataSync.${field.translationPrefix}`)
}}</template>
<FormInput
v-model="values[field.name]"
size="large"
:type="field.type"
:error="fieldHasErrors(field.name)"
@blur="$v.values[field.name].$touch()"
>
</FormInput>
<template #error>
<div
v-if="$v.values[field.name].$dirty && !$v.values[field.name].required"
>
{{ $t('error.requiredField') }}
</div>
</template>
</FormGroup>
<div class="row">
<div class="col col-5">
<FormGroup
:error="fieldHasErrors('postgresql_port')"
required
small-label
class="margin-bottom-2"
>
<template #label>{{ $t('postgreSQLDataSync.port') }}</template>
<FormInput
v-model="values.postgresql_port"
size="large"
:error="fieldHasErrors('postgresql_port')"
@blur="$v.values.postgresql_port.$touch()"
>
</FormInput>
<template #error>
<div
v-if="
$v.values.postgresql_port.$dirty &&
!$v.values.postgresql_port.required
"
>
{{ $t('error.requiredField') }}
</div>
<div
v-else-if="
$v.values.postgresql_port.$dirty &&
!$v.values.postgresql_port.numeric
"
>
{{ $t('error.invalidNumber') }}
</div>
</template>
</FormGroup>
</div>
<div class="col col-7">
<FormGroup required small-label class="margin-bottom-2">
<template #label>{{ $t('postgreSQLDataSync.sslMode') }}</template>
<Dropdown v-model="values.postgresql_sslmode" size="large">
<DropdownItem
v-for="option in sslModeOptions"
:key="option"
:name="option"
:value="option"
></DropdownItem>
</Dropdown>
</FormGroup>
</div>
</div>
</form>
</template>
<script>
import { required, numeric } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
export default {
name: 'PostgreSQLDataSync',
mixins: [form],
data() {
return {
allowedValues: [
'postgresql_host',
'postgresql_username',
'postgresql_password',
'postgresql_port',
'postgresql_database',
'postgresql_schema',
'postgresql_table',
'postgresql_sslmode',
],
values: {
postgresql_host: '',
postgresql_username: '',
postgresql_password: '',
postgresql_port: '5432',
postgresql_database: '',
postgresql_schema: 'public',
postgresql_table: '',
postgresql_sslmode: 'prefer',
},
sslModeOptions: [
'disable',
'allow',
'prefer',
'require',
'verify-ca',
'verify-full',
],
}
},
validations: {
values: {
postgresql_host: { required },
postgresql_username: { required },
postgresql_password: { required },
postgresql_database: { required },
postgresql_schema: { required },
postgresql_table: { required },
postgresql_sslmode: { required },
postgresql_port: { required, numeric },
},
},
}
</script>

8
web-frontend/modules/database/components/table/CreateDataSync.vue
View file

@ -144,6 +144,14 @@ export default {
this.properties = data
this.syncedProperties = data.map((p) => p.key)
} catch (error) {
if (error.handler && error.handler.code === 'ERROR_SYNC_ERROR') {
this.showError(
this.$t('dataSyncType.syncError'),
error.handler.detail
)
error.handler.handled()
return
}
this.handleError(error)
} finally {
this.loadingProperties = false

20
web-frontend/modules/database/dataSyncTypes.js
View file

@ -1,6 +1,7 @@
import { Registerable } from '@baserow/modules/core/registry'
import ICalCalendarDataSync from '@baserow/modules/database/components/dataSync/ICalCalendarDataSync'
import PostgreSQLDataSync from '@baserow/modules/database/components/dataSync/PostgreSQLDataSync'
export class DataSyncType extends Registerable {
/**
@ -77,3 +78,22 @@ export class ICalCalendarDataSyncType extends DataSyncType {
return ICalCalendarDataSync
}
}
export class PostgreSQLDataSyncType extends DataSyncType {
static getType() {
return 'postgresql'
}
getIconClass() {
return 'baserow-icon-postgresql'
}
getName() {
const { i18n } = this.app
return i18n.t('dataSyncType.postgresql')
}
getFormComponent() {
return PostgreSQLDataSync
}
}

15
web-frontend/modules/database/locales/en.json
View file

@ -969,12 +969,25 @@
"migrateButtonTooltip": "New filter available. Click to upgrade. It works the same."
},
"dataSyncType": {
"icalCalendar": "Sync iCal feed"
"syncError": "Sync error",
"icalCalendar": "Sync iCal feed",
"postgresql": "Sync PostgreSQL table"
},
"iCalCalendarDataSync": {
"name": "iCal URL",
"description": "The iCal calendar sync, synchronizes automatically with the entries in the calendar file of the URL. It only supports the ICS (Internet Calendar and Scheduling) file type."
},
"postgreSQLDataSync": {
"description": "Synchronizes a PostgreSQL table with a Baserow table matching the provided details below. Note that when the synchronization starts, it will select all the rows in the provided table. Even though Baserow only selects data, we strongly recommend limiting the user to a read-only connection.",
"host": "Host",
"username": "Username",
"password": "Password",
"database": "Database",
"schema": "Schema",
"table": "Table",
"port": "Port",
"sslMode": "SSL Mode"
},
"createDataSync": {
"next": "Next",
"fields": "Select the fields you would like to sync",

6
web-frontend/modules/database/plugin.js
View file

@ -112,7 +112,10 @@ import {
XMLImporterType,
JSONImporterType,
} from '@baserow/modules/database/importerTypes'
import { ICalCalendarDataSyncType } from '@baserow/modules/database/dataSyncTypes'
import {
ICalCalendarDataSyncType,
PostgreSQLDataSyncType,
} from '@baserow/modules/database/dataSyncTypes'
import {
RowsCreatedWebhookEventType,
RowsUpdatedWebhookEventType,
@ -598,6 +601,7 @@ export default (context) => {
app.$registry.register('importer', new XMLImporterType(context))
app.$registry.register('importer', new JSONImporterType(context))
app.$registry.register('dataSync', new ICalCalendarDataSyncType(context))
app.$registry.register('dataSync', new PostgreSQLDataSyncType(context))
app.$registry.register('settings', new APITokenSettingsType(context))
app.$registry.register('exporter', new CSVTableExporterType(context))
app.$registry.register(