archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-03 04:35:31 +00:00
Code Issues Projects Releases Wiki Activity

HubSpot contacts data sync

Browse source
This commit is contained in:
Bram Wiepjes 2024-12-11 22:54:44 +00:00
parent 8e3d878324
commit 7b0d889271
28 changed files with 1523 additions and 67 deletions

1
backend/src/baserow/contrib/database/api/data_sync/serializers.py
View file

@ -74,6 +74,7 @@ class ListDataSyncPropertySerializer(serializers.Serializer):
key = serializers.CharField()
name = serializers.CharField()
field_type = serializers.SerializerMethodField()
initially_selected = serializers.BooleanField()
def get_field_type(self, instance):
field_type = field_type_registry.get_by_model(instance.to_baserow_field())

3
backend/src/baserow/contrib/database/data_sync/handler.py
View file

@ -512,7 +512,8 @@ class DataSyncHandler:
synced_properties.insert(0, data_sync_property.key)
enabled_properties = DataSyncSyncedProperty.objects.filter(
data_sync=data_sync
data_sync=data_sync,
field__trashed=False,
).prefetch_related(
Prefetch("field", queryset=specific_queryset(Field.objects.all())),
"field__select_options",

12
backend/src/baserow/contrib/database/data_sync/registries.py
View file

@ -42,14 +42,24 @@ class DataSyncProperty(ABC):
formatted, but `False` for select options because those should be a fixed set.
"""
def __init__(self, key, name):
initially_selected = True
"""
Indicates whether the property must automatically be toggled on before the users
creates the data sync. This can be used if there are many properties, the user must
be automatically use them all.
"""
def __init__(self, key, name, initially_selected=True):
"""
:param key: A unique key that must never be changed.
:param name: Human-readable name of the property.
:param initially_selected: If true, then the property is suggested to be
enabled.
"""
self.key = key
self.name = name
self.initially_selected = initially_selected
@abstractmethod
def to_baserow_field(self) -> Field:

88
backend/src/baserow/contrib/database/data_sync/utils.py
View file

@ -1,4 +1,8 @@
from dataclasses import dataclass
from datetime import date, datetime, timezone
from typing import List
from baserow.contrib.database.fields.models import Field, SelectOption
def normalize_datetime(d):
@ -25,3 +29,87 @@ def compare_date(date1, date2):
date1 = normalize_date(date1)
date2 = normalize_date(date2)
return date1 == date2
@dataclass
class SourceOption:
id: int
value: str
color: str
order: int
def update_baserow_field_select_options(
source_options: List[SourceOption],
baserow_field: Field,
existing_mapping: dict,
) -> dict:
"""
Creates, updates or deletes the select options based on the provided
`source_options`. This function is made to be used in the `get_metadata`
method of a DataSyncType if it should contain select options.
:param source_options: A list of the options that must be created for the
`baserow_field`.
:param baserow_field: The Baserow field where the select options must be created
for.
:param existing_mapping: A key value dict mapping the source option id with the
created target option. Must be provided if the field already has select
options. It's used to correctly create, update, or delete the options.
:return: The key value dict mapping with the source option id as key and created
target option id as value. Must be passed into this function the next time the
options must be updated.
"""
select_options_mapping = {}
# Collect existing select options and prepare new field options. By storing
# them all in a list, we can loop over them and decide if they should be
# created, updated, or deleted.
target_field_options = [
SelectOption(
value=field_option.value,
color=field_option.color,
order=field_option.order,
field=baserow_field,
)
for field_option in source_options
]
# Prepare lists to track which options need to be created, updated, or deleted.
to_create = []
to_update = []
to_delete_ids = set(existing_mapping.values())
# Loop through the new options to decide on create or update actions.
for existing_option, new_option in zip(source_options, target_field_options):
target_id = existing_mapping.get(str(existing_option.id))
# If a target_id exists in the mapping, we update, otherwise, we create new.
if target_id:
new_option.id = target_id
to_update.append((new_option, existing_option.id))
to_delete_ids.discard(target_id)
else:
to_create.append((new_option, existing_option.id))
if to_create:
created_select_options = SelectOption.objects.bulk_create(
[r[0] for r in to_create]
)
for created_option, existing_option_id in zip(
created_select_options, [r[1] for r in to_create]
):
select_options_mapping[str(existing_option_id)] = created_option.id
if to_update:
SelectOption.objects.bulk_update(
[r[0] for r in to_update], fields=["value", "color", "order", "field"]
)
for updated_option, existing_option_id in to_update:
select_options_mapping[str(existing_option_id)] = updated_option.id
if to_delete_ids:
SelectOption.objects.filter(id__in=to_delete_ids).delete()
return select_options_mapping

8
backend/tests/baserow/contrib/database/api/data_sync/test_data_sync_views.py
View file

@ -959,24 +959,28 @@ def test_get_data_sync_properties(data_fixture, api_client):
"key": "uid",
"name": "Unique ID",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "dtstart",
"name": "Start date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "dtend",
"name": "End date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "summary",
"name": "Summary",
"field_type": "text",
"initially_selected": True,
},
]
@ -1113,24 +1117,28 @@ def test_get_data_sync_properties_of_data_sync(data_fixture, api_client):
"key": "uid",
"name": "Unique ID",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "dtstart",
"name": "Start date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "dtend",
"name": "End date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "summary",
"name": "Summary",
"field_type": "text",
"initially_selected": True,
},
]

31
backend/tests/baserow/contrib/database/data_sync/test_data_sync_handler.py
View file

@ -1671,6 +1671,37 @@ def test_delete_non_unique_primary_data_sync_field(data_fixture):
assert data_sync.table.field_set.all().count() == 3
@pytest.mark.django_db
@responses.activate
def test_delete_field_and_then_sync(data_fixture):
responses.add(
responses.GET,
"https://baserow.io/ical.ics",
status=200,
body=ICAL_FEED_WITH_ONE_ITEMS,
)
user = data_fixture.create_user()
database = data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="ical_calendar",
synced_properties=["uid", "dtstart"],
ical_url="https://baserow.io/ical.ics",
)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
FieldHandler().delete_field(user, fields[1])
data_sync = handler.sync_data_sync_table(user=user, data_sync=data_sync)
assert data_sync.last_error is None
@pytest.mark.field_duration
@pytest.mark.django_db
@responses.activate

21
backend/tests/baserow/contrib/database/data_sync/test_data_sync_postgresql.py
View file

@ -349,78 +349,96 @@ def test_postgresql_data_sync_get_properties(
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{"unique_primary": True, "key": "id", "name": "id", "field_type": "number"},
{
"unique_primary": True,
"key": "id",
"name": "id",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "text_col",
"name": "text_col",
"field_type": "long_text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "char_col",
"name": "char_col",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "int_col",
"name": "int_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "float_col",
"name": "float_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "numeric_col",
"name": "numeric_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "numeric2_col",
"name": "numeric2_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "smallint_col",
"name": "smallint_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "bigint_col",
"name": "bigint_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "decimal_col",
"name": "decimal_col",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "date_col",
"name": "date_col",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "datetime_col",
"name": "datetime_col",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "boolean_col",
"name": "boolean_col",
"field_type": "boolean",
"initially_selected": True,
},
]
@ -464,6 +482,7 @@ def test_postgresql_data_sync_get_properties_unsupported_column_types(
"key": "char_col",
"name": "char_col",
"field_type": "text",
"initially_selected": True,
}
]

7
changelog/entries/unreleased/feature/3119_hubspot_contacts_data_sync.json Normal file
View file

@ -0,0 +1,7 @@
{
"type": "feature",
"message": "HubSpot contacts data sync.",
"issue_number": 3119,
"bullet_points": [],
"created_at": "2024-11-26"
}

2
enterprise/backend/src/baserow_enterprise/apps.py
View file

@ -193,6 +193,7 @@ class BaserowEnterpriseConfig(AppConfig):
from baserow_enterprise.data_sync.data_sync_types import (
GitHubIssuesDataSyncType,
GitLabIssuesDataSyncType,
HubspotContactsDataSyncType,
JiraIssuesDataSyncType,
LocalBaserowTableDataSyncType,
)
@ -201,6 +202,7 @@ class BaserowEnterpriseConfig(AppConfig):
data_sync_type_registry.register(JiraIssuesDataSyncType())
data_sync_type_registry.register(GitHubIssuesDataSyncType())
data_sync_type_registry.register(GitLabIssuesDataSyncType())
data_sync_type_registry.register(HubspotContactsDataSyncType())
# Create default roles
post_migrate.connect(sync_default_roles_after_migrate, sender=self)

68
enterprise/backend/src/baserow_enterprise/data_sync/baserow_table_data_sync.py
View file

@ -12,7 +12,10 @@ from rest_framework import serializers
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.data_sync.utils import compare_date
from baserow.contrib.database.data_sync.utils import (
compare_date,
update_baserow_field_select_options,
)
from baserow.contrib.database.fields.field_types import (
AutonumberFieldType,
BooleanFieldType,
@ -35,7 +38,6 @@ from baserow.contrib.database.fields.models import (
DateField,
Field,
NumberField,
SelectOption,
TextField,
)
from baserow.contrib.database.fields.registries import field_type_registry
@ -139,7 +141,6 @@ class BaserowFieldDataSyncProperty(DataSyncProperty):
if new_metadata is None:
new_metadata = {}
new_metadata["select_options_mapping"] = {}
# Based on the existing mapping, we can figure out which select options must
# be created, updated, and deleted in the synced field.
@ -147,61 +148,12 @@ class BaserowFieldDataSyncProperty(DataSyncProperty):
if existing_metadata:
existing_mapping = existing_metadata.get("select_options_mapping", {})
# Collect existing select options and prepare new field options. By storing
# them all in a list, we can loop over them and decide if they should be
# created, updated, or deleted.
source_field_options = self.field.select_options.all()
target_field_options = [
SelectOption(
value=field_option.value,
color=field_option.color,
order=field_option.order,
field=baserow_field,
)
for field_option in source_field_options
]
# Prepare lists to track which options need to be created, updated, or deleted.
to_create = []
to_update = []
to_delete_ids = set(existing_mapping.values())
# Loop through the new options to decide on create or update actions.
for existing_option, new_option in zip(
source_field_options, target_field_options
):
target_id = existing_mapping.get(str(existing_option.id))
# If a target_id exists in the mapping, we update, otherwise, we create new.
if target_id:
new_option.id = target_id
to_update.append((new_option, existing_option.id))
to_delete_ids.discard(target_id)
else:
to_create.append((new_option, existing_option.id))
if to_create:
created_select_options = SelectOption.objects.bulk_create(
[r[0] for r in to_create]
)
for created_option, existing_option_id in zip(
created_select_options, [r[1] for r in to_create]
):
new_metadata["select_options_mapping"][
str(existing_option_id)
] = created_option.id
if to_update:
SelectOption.objects.bulk_update(
[r[0] for r in to_update], fields=["value", "color", "order", "field"]
)
for updated_option, existing_option_id in to_update:
new_metadata["select_options_mapping"][
str(existing_option_id)
] = updated_option.id
if to_delete_ids:
SelectOption.objects.filter(id__in=to_delete_ids).delete()
select_options_mapping = update_baserow_field_select_options(
self.field.select_options.all(),
baserow_field,
existing_mapping,
)
new_metadata["select_options_mapping"] = select_options_mapping
return new_metadata

1
enterprise/backend/src/baserow_enterprise/data_sync/data_sync_types.py
View file

@ -1,4 +1,5 @@
from .baserow_table_data_sync import LocalBaserowTableDataSyncType # noqa: F401
from .github_issues_data_sync import GitHubIssuesDataSyncType # noqa: F401
from .gitlab_issues_data_sync import GitLabIssuesDataSyncType # noqa: F401
from .hubspot_contacts_data_sync import HubspotContactsDataSyncType # noqa: F401
from .jira_issues_data_sync import JiraIssuesDataSyncType # noqa: F401

313
enterprise/backend/src/baserow_enterprise/data_sync/hubspot_contacts_data_sync.py Normal file
View file

@ -0,0 +1,313 @@
import math
from datetime import date, datetime
from decimal import Decimal
import requests
from baserow_premium.license.handler import LicenseHandler
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.data_sync.utils import (
SourceOption,
compare_date,
update_baserow_field_select_options,
)
from baserow.contrib.database.fields.models import (
DateField,
Field,
LongTextField,
NumberField,
PhoneNumberField,
SingleSelectField,
TextField,
)
from baserow.core.utils import ChildProgressBuilder, get_value_at_path
from baserow_enterprise.data_sync.models import HubSpotContactsDataSync
from baserow_enterprise.features import DATA_SYNC
class HubspotIDProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> Field:
return NumberField(name=self.name)
class BaseHubspotProperty(DataSyncProperty):
immutable_properties = True
def __init__(self, hubspot_object):
name = hubspot_object["name"]
self.description = hubspot_object["description"]
self.field_type = hubspot_object["fieldType"]
self.options = hubspot_object["options"]
return super().__init__(
key=hubspot_object["name"],
name=hubspot_object["label"],
# Because there are so many properties, we only want to initially select
# the ones about contactinformation and the non hubspot specific values.
# This gives a good initial selection of relevant data. Everything else
# can optionally be enabled by the user.
initially_selected=(
hubspot_object["groupName"] == "contactinformation"
and "hs_" not in name
),
)
def prepare_value(self, value, metadata):
return value
class HubspotStringProperty(BaseHubspotProperty):
def to_baserow_field(self) -> Field:
if self.field_type == "textarea":
return LongTextField(name=self.name, description=self.description)
if self.field_type == "phonenumber":
return PhoneNumberField(name=self.name, description=self.description)
else:
return TextField(name=self.name, description=self.description)
class HubspotNumberProperty(BaseHubspotProperty):
def to_baserow_field(self) -> Field:
return NumberField(name=self.name, description=self.description)
def is_equal(self, baserow_row_value, data_sync_row_value) -> bool:
data_sync_row_value = (
Decimal(data_sync_row_value) if data_sync_row_value else None
)
return super().is_equal(baserow_row_value, data_sync_row_value)
class HubspotDateProperty(BaseHubspotProperty):
def to_baserow_field(self) -> Field:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=False,
description=self.description,
)
def is_equal(self, baserow_row_value, data_sync_row_value) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
def prepare_value(self, value, metadata):
try:
return date.fromisoformat(value)
except TypeError:
return None
class HubspotDateTimeProperty(BaseHubspotProperty):
def to_baserow_field(self) -> Field:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
description=self.description,
)
def is_equal(self, baserow_row_value, data_sync_row_value) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
def prepare_value(self, value, metadata):
try:
return datetime.fromisoformat(value)
except TypeError:
return None
class HubspotEnumerationProperty(BaseHubspotProperty):
def to_baserow_field(self) -> Field:
return SingleSelectField(name=self.name, description=self.description)
def prepare_value(self, value, metadata):
if not value:
return None
mapping = metadata.get("select_options_mapping", {})
return mapping.get(value, None)
def get_metadata(self, baserow_field, existing_metadata=None):
new_metadata = super().get_metadata(baserow_field, existing_metadata)
if new_metadata is None:
new_metadata = {}
# Based on the existing mapping, we can figure out which select options must
# be created, updated, and deleted in the synced field.
existing_mapping = {}
if existing_metadata:
existing_mapping = existing_metadata.get("select_options_mapping", {})
options = [
SourceOption(
id=option["value"],
value=option["label"],
color="light-blue",
order=option["displayOrder"],
)
for option in self.options
]
select_options_mapping = update_baserow_field_select_options(
options,
baserow_field,
existing_mapping,
)
new_metadata["select_options_mapping"] = select_options_mapping
return new_metadata
def is_equal(self, baserow_row_value, data_sync_row_value) -> bool:
return super().is_equal(baserow_row_value, data_sync_row_value)
hubspot_property_type_mapping = {
"string": HubspotStringProperty,
"phone_number": HubspotStringProperty,
"number": HubspotNumberProperty,
"date": HubspotDateProperty,
"datetime": HubspotDateTimeProperty,
"enumeration": HubspotEnumerationProperty,
}
class HubspotContactsDataSyncType(DataSyncType):
type = "hubspot_contacts"
model_class = HubSpotContactsDataSync
allowed_fields = [
"hubspot_access_token",
]
request_serializer_field_names = [
"hubspot_access_token",
]
serializer_field_names = []
base_url = "https://api.hubapi.com"
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def get_properties(self, instance):
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
try:
# This endpoint responds with all the available contact properties and
# their types. They can all be included in the response when fetching the
# contacts.
response = requests.get(
f"{self.base_url}/crm/v3/properties/contacts?archived=false",
headers={"Authorization": f"Bearer {instance.hubspot_access_token}"},
timeout=10,
)
response.raise_for_status()
except requests.exceptions.RequestException as e:
raise SyncError(f"Error fetching HubSpot properties: {str(e)}")
fetched_properties = response.json()["results"]
properties = [HubspotIDProperty(key="id", name="Contact ID")]
properties = properties + [
hubspot_property_type_mapping.get(property_object["type"])(
hubspot_object=property_object
)
for property_object in fetched_properties
if property_object["type"] in hubspot_property_type_mapping.keys()
]
return properties
def get_contact_count(self, instance, headers):
try:
response = requests.post(
f"{self.base_url}/crm/v3/objects/contacts/search",
headers=headers,
timeout=10,
json={"filterGroups": [], "limit": 0},
)
response.raise_for_status()
except requests.exceptions.RequestException as e:
raise SyncError(f"Error fetching HubSpot contacts: {str(e)}")
return response.json()["total"]
def get_all_rows(self, instance, progress_builder=None):
properties = self.get_properties(instance)
synced_properties = DataSyncSyncedProperty.objects.filter(data_sync=instance)
synced_property_keys = [p.key for p in synced_properties]
headers = {"Authorization": f"Bearer {instance.hubspot_access_token}"}
page_limit = 50
contact_count = self.get_contact_count(instance, headers)
page_count = math.ceil(contact_count / page_limit)
progress = ChildProgressBuilder.build(
progress_builder,
child_total=page_count + 1,
)
progress.increment(by=1)
all_contacts = []
query_params = {
"limit": page_limit,
"archived": "false",
"properties": synced_property_keys,
}
while True:
url = f"{self.base_url}/crm/v3/objects/contacts"
try:
response = requests.get(
url, headers=headers, params=query_params, timeout=10
)
response.raise_for_status()
except requests.exceptions.RequestException as e:
raise SyncError(f"Error fetching HubSpot contacts: {str(e)}")
data = response.json()
all_contacts.extend(data.get("results", []))
progress.increment(by=1)
# If `after` is not in the response, or if it's `None`, then there is no
# consecutive page, and we can stop the loop.
after = get_value_at_path(data, "paging.next.after", None)
if after:
query_params["after"] = after
else:
break
rows = []
for contact in all_contacts:
row = {"id": Decimal(contact["id"])}
for enabled_property in synced_properties:
if enabled_property.key == "id":
continue
property_instance = next(
p
for p in properties
if p.key != "id" and p.key == enabled_property.key
)
# The property type instance sometimes has to modify the value,
# like with the `enumeration` type, it must be mapped to a select
# option.
row[enabled_property.key] = property_instance.prepare_value(
contact["properties"][enabled_property.key],
enabled_property.metadata,
)
rows.append(row)
return rows

8
enterprise/backend/src/baserow_enterprise/data_sync/models.py
View file

@ -71,3 +71,11 @@ class GitLabIssuesDataSync(DataSync):
max_length=255,
help_text="The API access token used to authenticate requests to GitLab.",
)
class HubSpotContactsDataSync(DataSync):
hubspot_access_token = models.CharField(
max_length=255,
help_text="The private app access token used to authenticate requests to "
"HubSpot.",
)

41
enterprise/backend/src/baserow_enterprise/migrations/0035_hubspotcontactsdatasync.py Normal file
View file

@ -0,0 +1,41 @@
# Generated by Django 5.0.9 on 2024-11-26 19:49
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("baserow_enterprise", "0034_samlappauthprovidermodel_and_more"),
("database", "0173_datasyncsyncedproperty_metadata"),
]
operations = [
migrations.CreateModel(
name="HubSpotContactsDataSync",
fields=[
(
"datasync_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="database.datasync",
),
),
(
"hubspot_access_token",
models.CharField(
help_text="The private app access token used to authenticate requests to HubSpot.",
max_length=255,
),
),
],
options={
"abstract": False,
},
bases=("database.datasync",),
),
]

87
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_github_issues_data_sync.py
View file

@ -479,6 +479,72 @@ def test_create_data_sync_table_pagination(enterprise_data_fixture):
assert model.objects.all().count() == 2
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_is_equal(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=1&per_page=50&state=all",
status=200,
json=SINGLE_ISSUE_RESPONSE,
)
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=2&per_page=50&state=all",
status=200,
json=NO_ISSUES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=[
"id",
"title",
"body",
"user",
"assignee",
"assignees",
"labels",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"milestone",
"url",
],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
rows = model.objects.all()
row_1 = rows[0]
row_1_last_modified = row_1.updated_on
handler.sync_data_sync_table(user=user, data_sync=data_sync)
rows = model.objects.all()
row_1 = rows[0]
# Because none of the values have changed, we don't expect the rows to have been
# updated.
assert row_1.updated_on == row_1_last_modified
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
@ -539,79 +605,98 @@ def test_get_data_sync_properties(enterprise_data_fixture, api_client):
"key": "id",
"name": "GitHub Issue ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "title",
"name": "Title",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "body",
"name": "Body",
"field_type": "long_text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "user",
"name": "User",
"field_type": "text",
"initially_selected": True,
},
{"unique_primary": False, "key": "user", "name": "User", "field_type": "text"},
{
"unique_primary": False,
"key": "assignee",
"name": "Assignee",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "assignees",
"name": "Assignees",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "labels",
"name": "Labels",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "state",
"name": "State",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "created_at",
"name": "Created At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "updated_at",
"name": "Updated At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "closed_at",
"name": "Closed At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "closed_by",
"name": "Closed By",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "milestone",
"name": "Milestone",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "url",
"name": "URL to Issue",
"field_type": "url",
"initially_selected": True,
},
]

88
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_gitlab_issues_data_sync.py
View file

@ -525,6 +525,76 @@ def test_create_data_sync_table_pagination(enterprise_data_fixture):
assert model.objects.all().count() == 2
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_is_equal(enterprise_data_fixture):
responses.add(
responses.GET,
"https://gitlab.com/api/v4/projects/1/issues?page=1&per_page=50&state=all",
status=200,
json=SINGLE_ISSUE_RESPONSE,
)
responses.add(
responses.GET,
"https://gitlab.com/api/v4/projects/1/issues?page=2&per_page=50&state=all",
status=200,
json=NO_ISSUES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="gitlab_issues",
synced_properties=[
"id",
"iid",
"project_id",
"title",
"description",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"labels",
"assignees",
"author",
"upvotes",
"downvotes",
"due_date",
"milestone",
"url",
],
gitlab_url="https://gitlab.com",
gitlab_project_id="1",
gitlab_access_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
rows = model.objects.all()
row_1 = rows[0]
row_1_last_modified = row_1.updated_on
handler.sync_data_sync_table(user=user, data_sync=data_sync)
rows = model.objects.all()
row_1 = rows[0]
# Because none of the values have changed, we don't expect the rows to have been
# updated.
assert row_1.updated_on == row_1_last_modified
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
@ -581,108 +651,126 @@ def test_get_data_sync_properties(enterprise_data_fixture, api_client):
"key": "id",
"name": "Internal unique ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "iid",
"name": "Issue ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "project_id",
"name": "Project ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "title",
"name": "Title",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "description",
"name": "Description",
"field_type": "long_text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "state",
"name": "State",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "created_at",
"name": "Created At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "updated_at",
"name": "Updated At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "closed_at",
"name": "Closed At",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "closed_by",
"name": "Closed By",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "labels",
"name": "Labels",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "assignees",
"name": "Assignees",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "author",
"name": "Author",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "upvotes",
"name": "Upvotes",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "downvotes",
"name": "Downvotes",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "due_date",
"name": "Due date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "milestone",
"name": "Milestone",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "url",
"name": "URL to Issue",
"field_type": "url",
"initially_selected": True,
},
]

661
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_hubspot_contacts_data_sync.py Normal file
View file

@ -0,0 +1,661 @@
import datetime
from copy import deepcopy
from decimal import Decimal
from django.test.utils import override_settings
from django.urls import reverse
import pytest
import responses
from baserow_premium.license.exceptions import FeaturesNotAvailableError
from baserow_premium.license.models import License
from rest_framework.status import (
HTTP_200_OK,
HTTP_400_BAD_REQUEST,
HTTP_402_PAYMENT_REQUIRED,
)
from baserow.contrib.database.data_sync.handler import DataSyncHandler
from baserow.contrib.database.fields.models import NumberField
from baserow.core.db import specific_iterator
from baserow_enterprise.data_sync.models import HubSpotContactsDataSync
ALL_PROPERTIES_RESPONSE = {
"results": [
{
"updatedAt": "2024-09-12T14:22:56.844Z",
"createdAt": "2020-06-30T15:57:37.277Z",
"name": "address",
"label": "Street Address",
"type": "string",
"fieldType": "text",
"description": "Contact's street address, including apartment or unit number.",
"groupName": "contactinformation",
"options": [],
"displayOrder": 6,
"calculated": False,
"externalOptions": False,
"hasUniqueValue": False,
"hidden": False,
"hubspotDefined": True,
"modificationMetadata": {
"archivable": True,
"readOnlyDefinition": True,
"readOnlyValue": False,
},
"formField": True,
"dataSensitivity": "non_sensitive",
},
{
"updatedAt": "2024-09-05T17:14:04.747Z",
"createdAt": "2019-08-06T02:41:09.377Z",
"name": "associatedcompanyid",
"label": "Primary Associated Company ID",
"type": "number",
"fieldType": "number",
"description": "HubSpot defined ID of a contact's primary associated company in HubSpot.",
"groupName": "contactinformation",
"options": [],
"referencedObjectType": "COMPANY",
"displayOrder": 24,
"calculated": False,
"externalOptions": True,
"hasUniqueValue": False,
"hidden": True,
"hubspotDefined": True,
"modificationMetadata": {
"archivable": True,
"readOnlyDefinition": True,
"readOnlyValue": False,
},
"formField": False,
"dataSensitivity": "non_sensitive",
},
{
"updatedAt": "2024-09-05T17:14:04.747Z",
"createdAt": "2019-08-06T02:41:09.148Z",
"name": "createdate",
"label": "Create Date",
"type": "datetime",
"fieldType": "date",
"description": "The date that a contact entered the system",
"groupName": "contactinformation",
"options": [],
"displayOrder": 17,
"calculated": False,
"externalOptions": False,
"hasUniqueValue": False,
"hidden": False,
"hubspotDefined": True,
"modificationMetadata": {
"archivable": True,
"readOnlyDefinition": True,
"readOnlyValue": True,
},
"formField": False,
"dataSensitivity": "non_sensitive",
},
{
"updatedAt": "2024-09-05T17:14:04.747Z",
"createdAt": "2019-08-06T02:41:09.148Z",
"name": "date",
"label": "Create Date",
"type": "date",
"fieldType": "date",
"description": "Just a date",
"groupName": "contactinformation",
"options": [],
"displayOrder": 17,
"calculated": False,
"externalOptions": False,
"hasUniqueValue": False,
"hidden": False,
"hubspotDefined": True,
"modificationMetadata": {
"archivable": True,
"readOnlyDefinition": True,
"readOnlyValue": True,
},
"formField": False,
"dataSensitivity": "non_sensitive",
},
{
"updatedAt": "2024-09-06T20:46:08.884Z",
"createdAt": "2020-06-30T15:57:37.247Z",
"name": "currentlyinworkflow",
"label": "Currently in workflow",
"type": "enumeration",
"fieldType": "booleancheckbox",
"description": "True when contact is enrolled in a workflow.",
"groupName": "contact_activity",
"options": [
{
"label": "True",
"value": "True",
"description": "",
"displayOrder": 0,
"hidden": False,
},
{
"label": "False",
"value": "False",
"description": "",
"displayOrder": 1,
"hidden": False,
},
],
"displayOrder": 1,
"calculated": False,
"externalOptions": False,
"hasUniqueValue": False,
"hidden": False,
"hubspotDefined": True,
"modificationMetadata": {
"archivable": True,
"readOnlyDefinition": True,
"readOnlyValue": True,
},
"formField": False,
"dataSensitivity": "non_sensitive",
},
]
}
COUNT_RESPONSE = {"total": 2, "results": [], "paging": {"next": {"after": "0"}}}
CONTACT_1 = {
"id": "1",
"properties": {
"address": "Test address",
"associatedcompanyid": "1",
"createdate": "2020-06-30T15:57:37.247Z",
"date": "2020-06-30",
"currentlyinworkflow": "True",
},
"createdAt": "2024-11-26T19:19:01.258Z",
"updatedAt": "2024-11-26T19:19:09.765Z",
"archived": False,
}
CONTACT_2 = {
"id": "2",
"properties": {
"address": "Some address",
"associatedcompanyid": "2",
"createdate": "2020-06-29T15:57:37.247Z",
"date": "2020-06-28",
"currentlyinworkflow": "False",
},
"createdAt": "2024-11-26T19:19:01.258Z",
"updatedAt": "2024-11-26T19:19:09.765Z",
"archived": False,
}
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_create_data_sync_table(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="hubspot_contacts",
synced_properties=[
"id",
"address",
"associatedcompanyid",
"createdate",
"date",
"currentlyinworkflow",
],
hubspot_access_token="test",
)
assert isinstance(data_sync, HubSpotContactsDataSync)
assert data_sync.hubspot_access_token == "test"
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
assert len(fields) == 6
assert fields[0].name == "Contact ID"
assert isinstance(fields[0], NumberField)
assert fields[0].primary is True
assert fields[0].read_only is True
assert fields[0].immutable_type is True
assert fields[0].immutable_properties is True
assert fields[0].number_decimal_places == 0
assert fields[1].name == "Street Address"
assert fields[1].primary is False
assert fields[1].read_only is True
assert fields[1].immutable_type is True
assert fields[1].immutable_properties is True
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
responses.add(
responses.POST,
"https://api.hubapi.com/crm/v3/objects/contacts/search",
status=200,
json=COUNT_RESPONSE,
)
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/objects/contacts",
status=200,
json={"results": [CONTACT_1, CONTACT_2]},
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="hubspot_contacts",
synced_properties=[
"id",
"address",
"associatedcompanyid",
"createdate",
"date",
"currentlyinworkflow",
],
hubspot_access_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
data_sync.refresh_from_db()
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
contact_id_field = fields[0]
address_field = fields[1]
associatedcompanyid_field = fields[2]
createdate_field = fields[3]
date_field = fields[4]
currentlyinworkflow_field = fields[5]
model = data_sync.table.get_model()
assert model.objects.all().count() == 2
row = model.objects.all().first()
assert getattr(row, f"field_{contact_id_field.id}") == Decimal("1")
assert getattr(row, f"field_{address_field.id}") == "Test address"
assert getattr(row, f"field_{associatedcompanyid_field.id}") == Decimal("1")
assert getattr(row, f"field_{createdate_field.id}") == datetime.datetime(
2020, 6, 30, 15, 57, 37, 247000, tzinfo=datetime.timezone.utc
)
assert getattr(row, f"field_{date_field.id}") == datetime.date(2020, 6, 30)
assert getattr(row, f"field_{currentlyinworkflow_field.id}").value == "True"
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_pagination(enterprise_data_fixture):
count_response = deepcopy(COUNT_RESPONSE)
count_response["total"] = 51
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
responses.add(
responses.POST,
"https://api.hubapi.com/crm/v3/objects/contacts/search",
status=200,
json=count_response,
)
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/objects/contacts?limit=50&archived=false&properties=id",
status=200,
json={"results": [CONTACT_1], "paging": {"next": {"after": "1"}}},
)
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/objects/contacts?limit=50&archived=false&properties=id&after=1",
status=200,
json={"results": [CONTACT_2]},
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="hubspot_contacts",
synced_properties=[
"id",
],
hubspot_access_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
assert model.objects.all().count() == 2
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
contact_id_field = fields[0]
rows = model.objects.all()
assert rows[0]
assert getattr(rows[0], f"field_{contact_id_field.id}") == Decimal("1")
assert getattr(rows[1], f"field_{contact_id_field.id}") == Decimal("2")
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_is_equal(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
responses.add(
responses.POST,
"https://api.hubapi.com/crm/v3/objects/contacts/search",
status=200,
json=COUNT_RESPONSE,
)
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/objects/contacts",
status=200,
json={"results": [CONTACT_1, CONTACT_2]},
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="hubspot_contacts",
synced_properties=[
"id",
"address",
"associatedcompanyid",
"createdate",
"date",
"currentlyinworkflow",
],
hubspot_access_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
rows = model.objects.all()
row_1 = rows[0]
row_2 = rows[1]
row_1_last_modified = row_1.updated_on
row_2_last_modified = row_2.updated_on
handler.sync_data_sync_table(user=user, data_sync=data_sync)
rows = model.objects.all()
row_1 = rows[0]
row_2 = rows[1]
# Because none of the values have changed, we don't expect the rows to have been
# updated.
assert row_1.updated_on == row_1_last_modified
assert row_2.updated_on == row_2_last_modified
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_create_data_sync_table_invalid_access_token(
enterprise_data_fixture, api_client
):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=401,
json={
"status": "error",
"message": "Authentication credentials not found. This API supports OAuth 2.0 authentication and you can find more details at https://developers.hubspot.com/docs/methods/auth/oauth-overview",
"correlationId": "ID",
"category": "INVALID_AUTHENTICATION",
},
)
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "hubspot_contacts",
"hubspot_access_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
response_json = response.json()
assert response.status_code == HTTP_400_BAD_REQUEST
assert response_json["error"] == "ERROR_SYNC_ERROR"
assert (
response_json["detail"]
== "Error fetching HubSpot properties: 401 Client Error: Unauthorized for url: https://api.hubapi.com/crm/v3/properties/contacts?archived=false"
)
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_get_data_sync_properties(enterprise_data_fixture, api_client):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "hubspot_contacts",
"hubspot_access_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{
"unique_primary": True,
"key": "id",
"name": "Contact ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "address",
"name": "Street Address",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "associatedcompanyid",
"name": "Primary Associated Company ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "createdate",
"name": "Create Date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "date",
"name": "Create Date",
"field_type": "date",
"initially_selected": True,
},
{
"unique_primary": False,
"key": "currentlyinworkflow",
"name": "Currently in workflow",
"field_type": "single_select",
"initially_selected": False,
},
]
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_create_data_sync_without_license(enterprise_data_fixture, api_client):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
user, token = enterprise_data_fixture.create_user_and_token()
database = enterprise_data_fixture.create_database_application(user=user)
url = reverse("api:database:data_sync:list", kwargs={"database_id": database.id})
response = api_client.post(
url,
{
"table_name": "Test 1",
"type": "hubspot_contacts",
"synced_properties": ["id"],
"hubspot_access_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_402_PAYMENT_REQUIRED
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_without_license(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="hubspot_contacts",
synced_properties=["id"],
hubspot_access_token="test",
)
License.objects.all().delete()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)
@pytest.mark.django_db(transaction=True)
@override_settings(DEBUG=True)
@responses.activate
def test_async_sync_data_sync_table_without_license(
api_client, enterprise_data_fixture
):
responses.add(
responses.GET,
"https://api.hubapi.com/crm/v3/properties/contacts?archived=false",
status=200,
json=ALL_PROPERTIES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
database = enterprise_data_fixture.create_database_application(user=user)
url = reverse("api:database:data_sync:list", kwargs={"database_id": database.id})
response = api_client.post(
url,
{
"table_name": "Test 1",
"type": "hubspot_contacts",
"synced_properties": ["id"],
"hubspot_access_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
data_sync_id = response.json()["data_sync"]["id"]
License.objects.all().delete()
response = api_client.post(
reverse(
"api:database:data_sync:sync_table", kwargs={"data_sync_id": data_sync_id}
),
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_402_PAYMENT_REQUIRED

13
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_jira_issues_data_sync.py
View file

@ -927,78 +927,91 @@ def test_get_data_sync_properties(enterprise_data_fixture, api_client):
"key": "jira_id",
"name": "Jira Issue ID",
"field_type": "text",
"initially_selected": True,
},
{
"field_type": "text",
"key": "summary",
"name": "Summary",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "long_text",
"key": "description",
"name": "Description",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "text",
"key": "assignee",
"name": "Assignee",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "text",
"key": "reporter",
"name": "Reporter",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "text",
"key": "labels",
"name": "Labels",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "date",
"key": "created",
"name": "Created Date",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "date",
"key": "updated",
"name": "Updated Date",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "date",
"key": "resolved",
"name": "Resolved Date",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "date",
"key": "due",
"name": "Due Date",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "text",
"key": "status",
"name": "State",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "text",
"key": "project",
"name": "Project",
"unique_primary": False,
"initially_selected": True,
},
{
"field_type": "url",
"key": "url",
"name": "Issue URL",
"unique_primary": False,
"initially_selected": True,
},
]

3
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_local_baserow_table_data_sync_type.py
View file

@ -606,18 +606,21 @@ def test_get_data_sync_properties(enterprise_data_fixture, api_client):
"key": "id",
"name": "Row ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": f"field_{field_1.id}",
"name": "Text",
"field_type": "text",
"initially_selected": True,
},
{
"unique_primary": False,
"key": f"field_{field_2.id}",
"name": "Number",
"field_type": "number",
"initially_selected": True,
},
]

66
enterprise/web-frontend/modules/baserow_enterprise/components/dataSync/HubspotContactsDataSyncForm.vue Normal file
View file

@ -0,0 +1,66 @@
<template>
<form @submit.prevent="submit">
<FormGroup
:error="fieldHasErrors('hubspot_access_token')"
:label="$t('hubspotContactsDataSync.accessToken')"
required
class="margin-bottom-2"
:helper-text="$t('hubspotContactsDataSync.accessTokenHelper')"
:protected-edit="update"
small-label
>
<FormInput
v-model="values.hubspot_access_token"
:error="fieldHasErrors('hubspot_access_token')"
:disabled="disabled"
size="large"
@blur="$v.values.hubspot_access_token.$touch()"
/>
<template #error>
<span
v-if="
$v.values.hubspot_access_token.$dirty &&
!$v.values.hubspot_access_token.required
"
>
{{ $t('error.requiredField') }}
</span>
</template>
</FormGroup>
</form>
</template>
<script>
import { required } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
export default {
name: 'HubspotContactsDataSyncForm',
mixins: [form],
props: {
update: {
type: Boolean,
required: false,
default: false,
},
disabled: {
type: Boolean,
required: false,
default: false,
},
},
data() {
return {
allowedValues: ['hubspot_access_token'],
values: {
hubspot_access_token: '',
},
}
},
validations: {
values: {
hubspot_access_token: { required },
},
},
}
</script>

28
enterprise/web-frontend/modules/baserow_enterprise/dataSyncTypes.js
View file

@ -6,6 +6,7 @@ import EnterpriseModal from '@baserow_enterprise/components/EnterpriseModal'
import JiraIssuesDataSyncForm from '@baserow_enterprise/components/dataSync/JiraIssuesDataSyncForm'
import GitHubIssuesDataSyncForm from '@baserow_enterprise/components/dataSync/GitHubIssuesDataSyncForm'
import GitLabIssuesDataSyncForm from '@baserow_enterprise/components/dataSync/GitLabIssuesDataSyncForm'
import HubspotContactsDataSyncForm from '@baserow_enterprise/components/dataSync/HubspotContactsDataSyncForm'
export class LocalBaserowTableDataSyncType extends DataSyncType {
static getType() {
@ -114,3 +115,30 @@ export class GitLabIssuesDataSyncType extends DataSyncType {
return EnterpriseModal
}
}
export class HubspotContactsDataSyncType extends DataSyncType {
static getType() {
return 'hubspot_contacts'
}
getIconClass() {
return 'baserow-icon-hubspot'
}
getName() {
const { i18n } = this.app
return i18n.t('enterpriseDataSyncType.hubspotContacts')
}
getFormComponent() {
return HubspotContactsDataSyncForm
}
isDeactivated(workspaceId) {
return !this.app.$hasFeature(EnterpriseFeatures.DATA_SYNC, workspaceId)
}
getDeactivatedClickModal() {
return EnterpriseModal
}
}

7
enterprise/web-frontend/modules/baserow_enterprise/locales/en.json
View file

@ -362,7 +362,8 @@
"localBaserowTable": "Sync Baserow table",
"jiraIssues": "Sync Jira issues",
"githubIssues": "Sync GitHub issues",
"gitlabIssues": "Sync GitLab issues"
"gitlabIssues": "Sync GitLab issues",
"hubspotContacts": "Sync HubSpot contacts"
},
"localBaserowTableDataSync": {
"name": "Source table ID",
@ -397,6 +398,10 @@
"accessToken": "Access token",
"accessTokenHelper": "Can be generated here https://gitlab.com/-/user_settings/personal_access_tokens by clicking on `Add new token`, select `read_api`."
},
"hubspotContactsDataSync": {
"accessToken": "Private app access token",
"accessTokenHelper": "To generate a private app access token in HubSpot, click on Settings in the top bar, navigate to Integrations > Private Apps, and create a new private app. Assign the following scopes: crm.objects.contacts.read, crm.schemas.contacts.read, and crm.objects.custom.read. Finally, click Create app to generate the token."
},
"samlAuthLink": {
"loginWithSaml": "Login with SAML",
"placeholderWithSaml": "{login} with SAML",

2
enterprise/web-frontend/modules/baserow_enterprise/plugin.js
View file

@ -47,6 +47,7 @@ import {
JiraIssuesDataSyncType,
GitHubIssuesDataSyncType,
GitLabIssuesDataSyncType,
HubspotContactsDataSyncType,
} from '@baserow_enterprise/dataSyncTypes'
import { FF_AB_SSO } from '@baserow/modules/core/plugins/featureFlags'
@ -142,4 +143,5 @@ export default (context) => {
app.$registry.register('dataSync', new JiraIssuesDataSyncType(context))
app.$registry.register('dataSync', new GitHubIssuesDataSyncType(context))
app.$registry.register('dataSync', new GitLabIssuesDataSyncType(context))
app.$registry.register('dataSync', new HubspotContactsDataSyncType(context))
}

3
web-frontend/modules/core/assets/icons/hubspot.svg Normal file
View file

@ -0,0 +1,3 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M16.5761 9.94739V7.81274C16.8555 7.68054 17.0922 7.47098 17.2585 7.20838C17.4248 6.94578 17.5141 6.64091 17.5158 6.32913V6.27888C17.5147 5.84313 17.3429 5.42555 17.038 5.11736C16.7331 4.80916 16.32 4.6354 15.8887 4.63401H15.839C15.4077 4.6352 14.9945 4.80877 14.6895 5.11683C14.3845 5.42488 14.2125 5.84238 14.2111 6.27813V6.32838C14.2125 6.63843 14.3004 6.94179 14.4648 7.20358C14.6292 7.46537 14.8634 7.67496 15.1405 7.80824L15.1501 7.81274V9.95189C14.3424 10.0766 13.5819 10.4157 12.9463 10.9345L12.9552 10.9277L7.14447 6.35538C7.25498 5.93654 7.2183 5.49194 7.04066 5.09737C6.86303 4.70281 6.55545 4.38272 6.17037 4.19169C5.78529 4.00066 5.34655 3.95051 4.92897 4.04979C4.51139 4.14908 4.14082 4.39165 3.88044 4.73615C3.62007 5.08065 3.48602 5.50574 3.50115 5.93894C3.51628 6.37214 3.67966 6.78663 3.96341 7.11172C4.24717 7.43682 4.63374 7.65239 5.05722 7.72168C5.48069 7.79098 5.91483 7.7097 6.28563 7.49171L6.27672 7.49621L11.9894 11.9898C11.4844 12.755 11.2161 13.6547 11.2189 14.5745C11.2189 15.5818 11.5344 16.5156 12.0703 17.2792L12.0607 17.2649L10.3222 19.0215C10.1831 18.9761 10.0379 18.9521 9.89171 18.9503H9.89022C9.59175 18.9503 9.29999 19.0397 9.05182 19.2073C8.80365 19.3748 8.61022 19.613 8.49601 19.8916C8.38179 20.1702 8.3519 20.4768 8.41013 20.7726C8.46836 21.0684 8.61208 21.3401 8.82313 21.5534C9.03418 21.7666 9.30308 21.9119 9.59581 21.9707C9.88855 22.0295 10.192 21.9993 10.4677 21.8839C10.7435 21.7685 10.9792 21.5731 11.145 21.3223C11.3108 21.0715 11.3993 20.7767 11.3993 20.4751C11.3975 20.3235 11.3728 20.1729 11.3258 20.0289L11.3288 20.0394L13.0487 18.3015C13.6092 18.734 14.26 19.0316 14.9515 19.1716C15.643 19.3117 16.357 19.2905 17.0392 19.1097C17.7214 18.9289 18.3538 18.5933 18.8883 18.1284C19.4228 17.6634 19.8452 17.0814 20.1236 16.4267C20.4019 15.7719 20.5288 15.0616 20.4945 14.3498C20.4603 13.638 20.2658 12.9435 19.9259 12.3191C19.5859 11.6947 19.1096 11.1569 18.5329 10.7465C17.9563 10.3362 17.2947 10.0642 16.5984 9.95114L16.5716 9.94739H16.5761ZM15.8605 16.9806C15.3901 16.9795 14.9306 16.8374 14.54 16.5725C14.1494 16.3075 13.8453 15.9315 13.6661 15.492C13.4869 15.0525 13.4406 14.5692 13.5331 14.1032C13.6256 13.6371 13.8527 13.2092 14.1858 12.8735C14.5188 12.5378 14.9429 12.3094 15.4044 12.2171C15.8658 12.1248 16.344 12.1727 16.7785 12.3549C17.213 12.5371 17.5844 12.8453 17.8456 13.2406C18.1068 13.6359 18.2463 14.1006 18.2463 14.576V14.5775C18.2463 15.2152 17.9955 15.8269 17.5492 16.2778C17.1029 16.7288 16.4976 16.9821 15.8665 16.9821L15.8605 16.9806Z" fill="black"/>
</svg>
Side by side

After

(image error) Size: 2.6 KiB

2
web-frontend/modules/core/assets/scss/variables.scss
View file

@ -80,7 +80,7 @@ $baserow-icons: 'circle-empty', 'circle-checked', 'check-square', 'formula',
'calendar', 'smile', 'smartphone', 'plus', 'heading-1', 'heading-2',
'heading-3', 'paragraph', 'ordered-list', 'enlarge', 'share', 'settings',
'up-down-arrows', 'application', 'groups', 'timeline', 'dashboard', 'jira',
'postgresql';
'postgresql', 'hubspot';
$grid-view-row-height-small: 33px;
$grid-view-row-height-medium: 55px;

2
web-frontend/modules/database/components/dataSync/ConfigureDataSyncVisibleFields.vue
View file

@ -13,7 +13,7 @@
{{ $t('configureDataSyncVisibleFields.fields') }}</template
>
<SwitchInput
v-for="property in properties"
v-for="property in orderedProperties"
:key="property.key"
class="margin-top-2"
small

2
web-frontend/modules/database/components/table/CreateDataSync.vue
View file

@ -25,7 +25,7 @@
<FormGroup small-label class="margin-top-3">
<template #label> {{ $t('createDataSync.fields') }}</template>
<SwitchInput
v-for="property in properties"
v-for="property in orderedProperties"
:key="property.key"
class="margin-top-2"
small

22
web-frontend/modules/database/mixins/dataSync.js
View file

@ -17,6 +17,24 @@ export default {
beforeDestroy() {
this.stopPollIfRunning()
},
computed: {
orderedProperties() {
if (!this.properties) {
return []
}
// Show the properties where `initially_selected == True` first.
return this.properties
.slice()
.sort((a, b) =>
a.initially_selected === b.initially_selected
? 0
: a.initially_selected
? -1
: 1
)
},
},
methods: {
toggleVisibleField(key) {
const index = this.syncedProperties.findIndex((f) => key === f)
@ -64,7 +82,9 @@ export default {
)
this.loadedProperties = true
this.properties = data
this.syncedProperties = data.map((p) => p.key)
this.syncedProperties = data
.filter((p) => p.initially_selected)
.map((p) => p.key)
} catch (error) {
if (error.handler && error.handler.code === 'ERROR_SYNC_ERROR') {
this.showError(