archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-15 09:34:13 +00:00
Code Issues Projects Releases Wiki Activity

Github issues data sync

Browse source
This commit is contained in:
Bram Wiepjes 2024-10-24 12:45:30 +00:00
parent b319d1f990
commit 5e228fa846
14 changed files with 1707 additions and 471 deletions

7
changelog/entries/unreleased/feature/3077_github_issues_data_sync.json Normal file
View file

@ -0,0 +1,7 @@
{
"type": "feature",
"message": "GitHub issues data sync.",
"issue_number": 3077,
"bullet_points": [],
"created_at": "2024-10-21"
}

2
enterprise/backend/src/baserow_enterprise/apps.py
View file

@ -185,12 +185,14 @@ class BaserowEnterpriseConfig(AppConfig):
data_sync_type_registry,
)
from baserow_enterprise.data_sync.data_sync_types import (
GitHubIssuesDataSyncType,
JiraIssuesDataSyncType,
LocalBaserowTableDataSyncType,
)
data_sync_type_registry.register(LocalBaserowTableDataSyncType())
data_sync_type_registry.register(JiraIssuesDataSyncType())
data_sync_type_registry.register(GitHubIssuesDataSyncType())
# Create default roles
post_migrate.connect(sync_default_roles_after_migrate, sender=self)

192
enterprise/backend/src/baserow_enterprise/data_sync/baserow_table_data_sync.py Normal file
View file

@ -0,0 +1,192 @@
from datetime import date, datetime
from typing import Any, Dict, List
from uuid import UUID
from baserow_premium.fields.field_types import AIFieldType
from baserow_premium.license.handler import LicenseHandler
from rest_framework import serializers
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.fields.field_types import (
AutonumberFieldType,
BooleanFieldType,
CreatedOnFieldType,
DateFieldType,
DurationFieldType,
EmailFieldType,
FileFieldType,
LastModifiedFieldType,
LongTextFieldType,
NumberFieldType,
PhoneNumberFieldType,
RatingFieldType,
TextFieldType,
URLFieldType,
UUIDFieldType,
)
from baserow.contrib.database.fields.models import (
DateField,
Field,
LongTextField,
NumberField,
TextField,
)
from baserow.contrib.database.fields.registries import field_type_registry
from baserow.contrib.database.rows.operations import ReadDatabaseRowOperationType
from baserow.contrib.database.table.exceptions import TableDoesNotExist
from baserow.contrib.database.table.handler import TableHandler
from baserow.core.db import specific_iterator
from baserow.core.handler import CoreHandler
from baserow_enterprise.features import DATA_SYNC
from .models import LocalBaserowTableDataSync
class RowIDDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> NumberField:
return NumberField(
name=self.name, number_decimal_places=0, number_negative=False
)
class BaserowFieldDataSyncProperty(DataSyncProperty):
supported_field_types = [
TextFieldType.type,
LongTextFieldType.type,
URLFieldType.type,
EmailFieldType.type,
NumberFieldType.type,
RatingFieldType.type,
BooleanFieldType.type,
DateFieldType.type,
DurationFieldType.type,
FileFieldType.type,
PhoneNumberFieldType.type,
CreatedOnFieldType.type,
LastModifiedFieldType.type,
UUIDFieldType.type,
AutonumberFieldType.type,
AIFieldType.type,
]
field_types_override = {
CreatedOnFieldType.type: DateField,
LastModifiedFieldType.type: DateField,
UUIDFieldType.type: TextField,
AutonumberFieldType.type: NumberField,
AIFieldType.type: LongTextField,
}
def __init__(self, field, immutable_properties, **kwargs):
self.field = field
self.immutable_properties = immutable_properties
super().__init__(**kwargs)
def to_baserow_field(self) -> Field:
field_type = field_type_registry.get_by_model(self.field)
allowed_fields = ["name"] + field_type.allowed_fields
model_class = self.field_types_override.get(
field_type.type, field_type.model_class
)
return model_class(
**{
allowed_field: getattr(self.field, allowed_field)
for allowed_field in allowed_fields
if hasattr(self.field, allowed_field)
and hasattr(model_class, allowed_field)
}
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
# The CreatedOn and LastModified fields are always stored as datetime in the
# source table, but not always in the data sync table, so if that happens we'll
# compare loosely.
if isinstance(baserow_row_value, date) and isinstance(
data_sync_row_value, datetime
):
return compare_date(baserow_row_value, data_sync_row_value)
# The baserow row value is converted to a string, so we would need to convert
# the uuid object to a string to do a good comparison.
if isinstance(data_sync_row_value, UUID):
data_sync_row_value = str(data_sync_row_value)
return super().is_equal(baserow_row_value, data_sync_row_value)
class LocalBaserowTableDataSyncType(DataSyncType):
type = "local_baserow_table"
model_class = LocalBaserowTableDataSync
allowed_fields = ["source_table_id", "authorized_user_id"]
serializer_field_names = ["source_table_id"]
serializer_field_overrides = {
"source_table_id": serializers.IntegerField(
help_text="The ID of the source table that must be synced.",
required=True,
allow_null=False,
),
}
def prepare_values(self, user, values):
# The user that creates the data sync is automatically the one on whose
# behalf the data is synced in the future.
values["authorized_user_id"] = user.id
return values
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def _get_table(self, instance):
try:
table = TableHandler().get_table(instance.source_table_id)
except TableDoesNotExist:
raise SyncError("The source table doesn't exist.")
if not CoreHandler().check_permissions(
instance.authorized_user,
ReadDatabaseRowOperationType.type,
workspace=table.database.workspace,
context=table,
raise_permission_exceptions=False,
):
raise SyncError("The authorized user doesn't have access to the table.")
return table
def get_properties(self, instance) -> List[DataSyncProperty]:
table = self._get_table(instance)
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
fields = specific_iterator(table.field_set.all())
properties = [RowIDDataSyncProperty("id", "Row ID")]
return properties + [
BaserowFieldDataSyncProperty(
field=field,
immutable_properties=True,
key=f"field_{field.id}",
name=field.name,
)
for field in fields
if field_type_registry.get_by_model(field).type
in BaserowFieldDataSyncProperty.supported_field_types
]
def get_all_rows(self, instance) -> List[Dict]:
table = self._get_table(instance)
enabled_properties = DataSyncSyncedProperty.objects.filter(data_sync=instance)
enabled_property_field_ids = [p.key for p in enabled_properties]
model = table.get_model()
rows_queryset = model.objects.all().values(*["id"] + enabled_property_field_ids)
return rows_queryset

472
enterprise/backend/src/baserow_enterprise/data_sync/data_sync_types.py
View file

@ -1,469 +1,3 @@
from datetime import date, datetime
from typing import Any, Dict, List
from uuid import UUID
import advocate
from advocate import UnacceptableAddressException
from baserow_premium.fields.field_types import AIFieldType
from baserow_premium.license.handler import LicenseHandler
from jira2markdown import convert
from requests.auth import HTTPBasicAuth
from requests.exceptions import JSONDecodeError, RequestException
from rest_framework import serializers
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.models import DataSyncSyncedProperty
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.fields.field_types import (
AutonumberFieldType,
BooleanFieldType,
CreatedOnFieldType,
DateFieldType,
DurationFieldType,
EmailFieldType,
FileFieldType,
LastModifiedFieldType,
LongTextFieldType,
NumberFieldType,
PhoneNumberFieldType,
RatingFieldType,
TextFieldType,
URLFieldType,
UUIDFieldType,
)
from baserow.contrib.database.fields.models import (
DateField,
Field,
LongTextField,
NumberField,
TextField,
URLField,
)
from baserow.contrib.database.fields.registries import field_type_registry
from baserow.contrib.database.rows.operations import ReadDatabaseRowOperationType
from baserow.contrib.database.table.exceptions import TableDoesNotExist
from baserow.contrib.database.table.handler import TableHandler
from baserow.core.db import specific_iterator
from baserow.core.handler import CoreHandler
from baserow.core.utils import get_value_at_path
from baserow_enterprise.features import DATA_SYNC
from .models import JiraIssuesDataSync, LocalBaserowTableDataSync
class RowIDDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> NumberField:
return NumberField(
name=self.name, number_decimal_places=0, number_negative=False
)
class BaserowFieldDataSyncProperty(DataSyncProperty):
supported_field_types = [
TextFieldType.type,
LongTextFieldType.type,
URLFieldType.type,
EmailFieldType.type,
NumberFieldType.type,
RatingFieldType.type,
BooleanFieldType.type,
DateFieldType.type,
DurationFieldType.type,
FileFieldType.type,
PhoneNumberFieldType.type,
CreatedOnFieldType.type,
LastModifiedFieldType.type,
UUIDFieldType.type,
AutonumberFieldType.type,
AIFieldType.type,
]
field_types_override = {
CreatedOnFieldType.type: DateField,
LastModifiedFieldType.type: DateField,
UUIDFieldType.type: TextField,
AutonumberFieldType.type: NumberField,
AIFieldType.type: LongTextField,
}
def __init__(self, field, immutable_properties, **kwargs):
self.field = field
self.immutable_properties = immutable_properties
super().__init__(**kwargs)
def to_baserow_field(self) -> Field:
field_type = field_type_registry.get_by_model(self.field)
allowed_fields = ["name"] + field_type.allowed_fields
model_class = self.field_types_override.get(
field_type.type, field_type.model_class
)
return model_class(
**{
allowed_field: getattr(self.field, allowed_field)
for allowed_field in allowed_fields
if hasattr(self.field, allowed_field)
and hasattr(model_class, allowed_field)
}
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
# The CreatedOn and LastModified fields are always stored as datetime in the
# source table, but not always in the data sync table, so if that happens we'll
# compare loosely.
if isinstance(baserow_row_value, date) and isinstance(
data_sync_row_value, datetime
):
return compare_date(baserow_row_value, data_sync_row_value)
# The baserow row value is converted to a string, so we would need to convert
# the uuid object to a string to do a good comparison.
if isinstance(data_sync_row_value, UUID):
data_sync_row_value = str(data_sync_row_value)
return super().is_equal(baserow_row_value, data_sync_row_value)
class LocalBaserowTableDataSyncType(DataSyncType):
type = "local_baserow_table"
model_class = LocalBaserowTableDataSync
allowed_fields = ["source_table_id", "authorized_user_id"]
serializer_field_names = ["source_table_id"]
serializer_field_overrides = {
"source_table_id": serializers.IntegerField(
help_text="The ID of the source table that must be synced.",
required=True,
allow_null=False,
),
}
def prepare_values(self, user, values):
# The user that creates the data sync is automatically the one on whose
# behalf the data is synced in the future.
values["authorized_user_id"] = user.id
return values
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def _get_table(self, instance):
try:
table = TableHandler().get_table(instance.source_table_id)
except TableDoesNotExist:
raise SyncError("The source table doesn't exist.")
if not CoreHandler().check_permissions(
instance.authorized_user,
ReadDatabaseRowOperationType.type,
workspace=table.database.workspace,
context=table,
raise_permission_exceptions=False,
):
raise SyncError("The authorized user doesn't have access to the table.")
return table
def get_properties(self, instance) -> List[DataSyncProperty]:
table = self._get_table(instance)
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
fields = specific_iterator(table.field_set.all())
properties = [RowIDDataSyncProperty("id", "Row ID")]
return properties + [
BaserowFieldDataSyncProperty(
field=field,
immutable_properties=True,
key=f"field_{field.id}",
name=field.name,
)
for field in fields
if field_type_registry.get_by_model(field).type
in BaserowFieldDataSyncProperty.supported_field_types
]
def get_all_rows(self, instance) -> List[Dict]:
table = self._get_table(instance)
enabled_properties = DataSyncSyncedProperty.objects.filter(data_sync=instance)
enabled_property_field_ids = [p.key for p in enabled_properties]
model = table.get_model()
rows_queryset = model.objects.all().values(*["id"] + enabled_property_field_ids)
return rows_queryset
class JiraIDDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraSummaryDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraDescriptionDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> LongTextField:
return LongTextField(name=self.name, long_text_enable_rich_text=True)
class JiraAssigneeDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraReporterDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraLabelsDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraCreatedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraUpdatedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraResolvedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraDueDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraStateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraProjectDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraURLDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> URLField:
return URLField(name=self.name)
class JiraIssuesDataSyncType(DataSyncType):
type = "jira_issues"
model_class = JiraIssuesDataSync
allowed_fields = ["jira_url", "jira_project_key", "jira_username", "jira_api_token"]
serializer_field_names = [
"jira_url",
"jira_project_key",
"jira_username",
"jira_api_token",
]
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def get_properties(self, instance) -> List[DataSyncProperty]:
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
return [
JiraIDDataSyncProperty("jira_id", "Jira Issue ID"),
JiraSummaryDataSyncProperty("summary", "Summary"),
JiraDescriptionDataSyncProperty("description", "Description"),
JiraAssigneeDataSyncProperty("assignee", "Assignee"),
JiraReporterDataSyncProperty("reporter", "Reporter"),
JiraLabelsDataSyncProperty("labels", "Labels"),
JiraCreatedDateDataSyncProperty("created", "Created Date"),
JiraUpdatedDateDataSyncProperty("updated", "Updated Date"),
JiraResolvedDateDataSyncProperty("resolved", "Resolved Date"),
JiraDueDateDataSyncProperty("due", "Due Date"),
JiraStateDataSyncProperty("status", "State"),
JiraStateDataSyncProperty("project", "Project"),
JiraURLDataSyncProperty("url", "Issue URL"),
]
def _parse_datetime(self, value):
if not value:
return None
try:
return datetime.fromisoformat(value)
except ValueError:
raise SyncError(f"The date {value} could not be parsed.")
def _fetch_issues(self, instance):
headers = {"Content-Type": "application/json"}
issues = []
start_at = 0
max_results = 50
try:
while True:
url = (
f"{instance.jira_url}"
+ f"/rest/api/2/search"
+ f"?startAt={start_at}"
+ f"&maxResults={max_results}"
)
if instance.jira_project_key:
url += f"&jql=project={instance.jira_project_key}"
response = advocate.get(
url,
auth=HTTPBasicAuth(instance.jira_username, instance.jira_api_token),
headers=headers,
timeout=10,
)
if not response.ok:
try:
json = response.json()
if "errorMessages" in json and len(json["errorMessages"]) > 0:
raise SyncError(json["errorMessages"][0])
except JSONDecodeError:
pass
raise SyncError(
"The request to Jira did not return an OK response."
)
data = response.json()
if len(data["issues"]) == 0 and start_at == 0:
raise SyncError(
"No issues found. This is usually because the authentication "
"details are wrong."
)
issues.extend(data["issues"])
start_at += max_results
if data["total"] <= start_at:
break
except (RequestException, UnacceptableAddressException, ConnectionError):
raise SyncError("Error fetching issues from Jira.")
return issues
def get_all_rows(self, instance) -> List[Dict]:
issue_list = []
for issue in self._fetch_issues(instance):
try:
jira_id = issue["id"]
issue_url = f"{instance.jira_url}/browse/{issue['key']}"
except KeyError:
raise SyncError(
"The `id` and `key` are not found in the issue. This is likely the "
"result of an invalid response from Jira."
)
summary = get_value_at_path(issue, "fields.summary", "")
description = get_value_at_path(issue, "fields.description", "") or ""
assignee = get_value_at_path(issue, "fields.assignee.displayName", "")
reporter = get_value_at_path(issue, "fields.reporter.displayName", "")
project = get_value_at_path(issue, "fields.project.name", "")
status = get_value_at_path(issue, "fields.status.name", "")
labels = ", ".join(issue["fields"].get("labels", []))
created = self._parse_datetime(issue["fields"].get("created"))
updated = self._parse_datetime(issue["fields"].get("updated"))
resolved = self._parse_datetime(issue["fields"].get("resolutiondate"))
due = self._parse_datetime(issue["fields"].get("duedate"))
issue_dict = {
"jira_id": jira_id,
"summary": summary,
"description": convert(description),
"assignee": assignee,
"reporter": reporter,
"labels": labels,
"created": created,
"updated": updated,
"resolved": resolved,
"due": due,
"status": status,
"project": project,
"url": issue_url,
}
issue_list.append(issue_dict)
return issue_list
from .baserow_table_data_sync import LocalBaserowTableDataSyncType # noqa: F401
from .github_issues_data_sync import GitHubIssuesDataSyncType # noqa: F401
from .jira_issues_data_sync import JiraIssuesDataSyncType # noqa: F401

284
enterprise/backend/src/baserow_enterprise/data_sync/github_issues_data_sync.py Normal file
View file

@ -0,0 +1,284 @@
from datetime import datetime
from typing import Any, Dict, List
import requests
from baserow_premium.license.handler import LicenseHandler
from requests.exceptions import JSONDecodeError, RequestException
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.fields.models import (
DateField,
LongTextField,
NumberField,
TextField,
URLField,
)
from baserow.core.utils import get_value_at_path
from baserow_enterprise.features import DATA_SYNC
from .models import GitHubIssuesDataSync
class GitHubIDDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> NumberField:
return NumberField(
name=self.name, number_decimal_places=0, number_negative=False
)
class GitHubTitleDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubBodyDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> LongTextField:
return LongTextField(name=self.name, long_text_enable_rich_text=True)
class GitHubUserDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubAssigneeDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubAssigneesDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubLabelsDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubStateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubCreatedAtDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class GitHubUpdatedAtDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class GitHubClosedAtDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class GitHubClosedByDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubMilestoneDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class GitHubUrlDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> URLField:
return URLField(name=self.name)
class GitHubIssuesDataSyncType(DataSyncType):
type = "github_issues"
model_class = GitHubIssuesDataSync
allowed_fields = [
"github_issues_owner",
"github_issues_repo",
"github_issues_api_token",
]
serializer_field_names = [
"github_issues_owner",
"github_issues_repo",
"github_issues_api_token",
]
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def get_properties(self, instance) -> List[DataSyncProperty]:
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
return [
GitHubIDDataSyncProperty("id", "GitHub Issue ID"),
GitHubTitleDataSyncProperty("title", "Title"),
GitHubBodyDataSyncProperty("body", "Body"),
GitHubUserDataSyncProperty("user", "User"),
GitHubAssigneeDataSyncProperty("assignee", "Assignee"),
GitHubAssigneesDataSyncProperty("assignees", "Assignees"),
GitHubLabelsDataSyncProperty("labels", "Labels"),
GitHubStateDataSyncProperty("state", "State"),
GitHubCreatedAtDataSyncProperty("created_at", "Created At"),
GitHubUpdatedAtDataSyncProperty("updated_at", "Updated At"),
GitHubClosedAtDataSyncProperty("closed_at", "Closed At"),
GitHubClosedByDataSyncProperty("closed_by", "Closed By"),
GitHubMilestoneDataSyncProperty("milestone", "Milestone"),
GitHubUrlDataSyncProperty("url", "URL to Issue"),
]
def _parse_datetime(self, value):
if not value:
return None
try:
return datetime.fromisoformat(value)
except ValueError:
raise SyncError(f"The date {value} could not be parsed.")
def _fetch_issues(self, instance):
url = f"https://api.github.com/repos/{instance.github_issues_owner}/{instance.github_issues_repo}/issues"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {instance.github_issues_api_token}",
"X-GitHub-Api-Version": "2022-11-28",
}
page, per_page = 1, 50
issues = []
try:
while True:
response = requests.get(
url,
headers=headers,
params={"page": page, "per_page": per_page, "state": "all"},
timeout=20,
)
if not response.ok:
try:
json = response.json()
if "message" in json:
raise SyncError(json["message"])
except JSONDecodeError:
pass
raise SyncError(
"The request to GitHub did not return an OK response."
)
data = response.json()
if not data:
break
issues.extend(data)
page += 1
except RequestException as e:
raise SyncError(f"Error fetching GitHub Issues: {str(e)}")
return issues
def get_all_rows(self, instance) -> List[Dict]:
issues = []
for issue in self._fetch_issues(instance):
issue_id = get_value_at_path(issue, "number")
created_at = self._parse_datetime(get_value_at_path(issue, "created_at"))
updated_at = self._parse_datetime(get_value_at_path(issue, "updated_at"))
closed_at = self._parse_datetime(get_value_at_path(issue, "closed_at"))
assignees = ", ".join(
[a["login"] for a in get_value_at_path(issue, "assignees", [])]
)
labels = ", ".join(
[label["name"] for label in get_value_at_path(issue, "labels", [])]
)
url = (
f"https://github.com/{instance.github_issues_owner}/"
f"{instance.github_issues_repo}/issues/"
f"{issue_id}"
)
issues.append(
{
"id": issue_id,
"title": get_value_at_path(issue, "title", ""),
"body": get_value_at_path(issue, "body", ""),
"user": get_value_at_path(issue, "user.login", ""),
"assignee": get_value_at_path(issue, "assignee.login", ""),
"assignees": assignees,
"labels": labels,
"state": get_value_at_path(issue, "state", ""),
"created_at": created_at,
"updated_at": updated_at,
"closed_at": closed_at,
"closed_by": get_value_at_path(issue, "closed_by.login", ""),
"milestone": get_value_at_path(issue, "milestone.title", ""),
"url": url,
}
)
return issues

293
enterprise/backend/src/baserow_enterprise/data_sync/jira_issues_data_sync.py Normal file
View file

@ -0,0 +1,293 @@
from datetime import datetime
from typing import Any, Dict, List
import advocate
from advocate import UnacceptableAddressException
from baserow_premium.license.handler import LicenseHandler
from jira2markdown import convert
from requests.auth import HTTPBasicAuth
from requests.exceptions import JSONDecodeError, RequestException
from baserow.contrib.database.data_sync.data_sync_types import compare_date
from baserow.contrib.database.data_sync.exceptions import SyncError
from baserow.contrib.database.data_sync.registries import DataSyncProperty, DataSyncType
from baserow.contrib.database.fields.models import (
DateField,
LongTextField,
TextField,
URLField,
)
from baserow.core.utils import get_value_at_path
from baserow_enterprise.features import DATA_SYNC
from .models import JiraIssuesDataSync
class JiraIDDataSyncProperty(DataSyncProperty):
unique_primary = True
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraSummaryDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraDescriptionDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> LongTextField:
return LongTextField(name=self.name, long_text_enable_rich_text=True)
class JiraAssigneeDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraReporterDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraLabelsDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraCreatedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraUpdatedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraResolvedDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraDueDateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> DateField:
return DateField(
name=self.name,
date_format="ISO",
date_include_time=True,
date_time_format="24",
date_show_tzinfo=True,
)
def is_equal(self, baserow_row_value: Any, data_sync_row_value: Any) -> bool:
return compare_date(baserow_row_value, data_sync_row_value)
class JiraStateDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraProjectDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> TextField:
return TextField(name=self.name)
class JiraURLDataSyncProperty(DataSyncProperty):
immutable_properties = True
def to_baserow_field(self) -> URLField:
return URLField(name=self.name)
class JiraIssuesDataSyncType(DataSyncType):
type = "jira_issues"
model_class = JiraIssuesDataSync
allowed_fields = ["jira_url", "jira_project_key", "jira_username", "jira_api_token"]
serializer_field_names = [
"jira_url",
"jira_project_key",
"jira_username",
"jira_api_token",
]
def prepare_sync_job_values(self, instance):
# Raise the error so that the job doesn't start and the user is informed with
# the correct error.
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
def get_properties(self, instance) -> List[DataSyncProperty]:
# The `table_id` is not set if when just listing the properties using the
# `DataSyncPropertiesView` endpoint, but it will be set when creating the view.
if instance.table_id:
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
return [
JiraIDDataSyncProperty("jira_id", "Jira Issue ID"),
JiraSummaryDataSyncProperty("summary", "Summary"),
JiraDescriptionDataSyncProperty("description", "Description"),
JiraAssigneeDataSyncProperty("assignee", "Assignee"),
JiraReporterDataSyncProperty("reporter", "Reporter"),
JiraLabelsDataSyncProperty("labels", "Labels"),
JiraCreatedDateDataSyncProperty("created", "Created Date"),
JiraUpdatedDateDataSyncProperty("updated", "Updated Date"),
JiraResolvedDateDataSyncProperty("resolved", "Resolved Date"),
JiraDueDateDataSyncProperty("due", "Due Date"),
JiraStateDataSyncProperty("status", "State"),
JiraStateDataSyncProperty("project", "Project"),
JiraURLDataSyncProperty("url", "Issue URL"),
]
def _parse_datetime(self, value):
if not value:
return None
try:
return datetime.fromisoformat(value)
except ValueError:
raise SyncError(f"The date {value} could not be parsed.")
def _fetch_issues(self, instance):
headers = {"Content-Type": "application/json"}
issues = []
start_at = 0
max_results = 50
try:
while True:
url = (
f"{instance.jira_url}"
+ f"/rest/api/2/search"
+ f"?startAt={start_at}"
+ f"&maxResults={max_results}"
)
if instance.jira_project_key:
url += f"&jql=project={instance.jira_project_key}"
response = advocate.get(
url,
auth=HTTPBasicAuth(instance.jira_username, instance.jira_api_token),
headers=headers,
timeout=10,
)
if not response.ok:
try:
json = response.json()
if "errorMessages" in json and len(json["errorMessages"]) > 0:
raise SyncError(json["errorMessages"][0])
except JSONDecodeError:
pass
raise SyncError(
"The request to Jira did not return an OK response."
)
data = response.json()
if len(data["issues"]) == 0 and start_at == 0:
raise SyncError(
"No issues found. This is usually because the authentication "
"details are wrong."
)
issues.extend(data["issues"])
start_at += max_results
if data["total"] <= start_at:
break
except (RequestException, UnacceptableAddressException, ConnectionError):
raise SyncError("Error fetching issues from Jira.")
return issues
def get_all_rows(self, instance) -> List[Dict]:
issue_list = []
for issue in self._fetch_issues(instance):
try:
jira_id = issue["id"]
issue_url = f"{instance.jira_url}/browse/{issue['key']}"
except KeyError:
raise SyncError(
"The `id` and `key` are not found in the issue. This is likely the "
"result of an invalid response from Jira."
)
summary = get_value_at_path(issue, "fields.summary", "")
description = get_value_at_path(issue, "fields.description", "") or ""
assignee = get_value_at_path(issue, "fields.assignee.displayName", "")
reporter = get_value_at_path(issue, "fields.reporter.displayName", "")
project = get_value_at_path(issue, "fields.project.name", "")
status = get_value_at_path(issue, "fields.status.name", "")
labels = ", ".join(issue["fields"].get("labels", []))
created = self._parse_datetime(issue["fields"].get("created"))
updated = self._parse_datetime(issue["fields"].get("updated"))
resolved = self._parse_datetime(issue["fields"].get("resolutiondate"))
due = self._parse_datetime(issue["fields"].get("duedate"))
issue_dict = {
"jira_id": jira_id,
"summary": summary,
"description": convert(description),
"assignee": assignee,
"reporter": reporter,
"labels": labels,
"created": created,
"updated": updated,
"resolved": resolved,
"due": due,
"status": status,
"project": project,
"url": issue_url,
}
issue_list.append(issue_dict)
return issue_list

13
enterprise/backend/src/baserow_enterprise/data_sync/models.py
View file

@ -42,3 +42,16 @@ class JiraIssuesDataSync(DataSync):
max_length=255,
help_text="The API token of the Jira account used for authentication.",
)
class GitHubIssuesDataSync(DataSync):
github_issues_owner = models.CharField(
max_length=255, help_text="The owner of the repository on GitHub."
)
github_issues_repo = models.CharField(
max_length=255, help_text="The name of the repository on GitHub."
)
github_issues_api_token = models.CharField(
max_length=255,
help_text="The API token used to authenticate requests to GitHub.",
)

55
enterprise/backend/src/baserow_enterprise/migrations/0031_githubissuesdatasync.py Normal file
View file

@ -0,0 +1,55 @@
# Generated by Django 5.0.9 on 2024-10-21 20:26
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("baserow_enterprise", "0030_jiraissuesdatasync"),
("database", "0170_update_password_tsv_fields"),
]
operations = [
migrations.CreateModel(
name="GitHubIssuesDataSync",
fields=[
(
"datasync_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="database.datasync",
),
),
(
"github_issues_owner",
models.CharField(
help_text="The owner of the repository on GitHub.",
max_length=255,
),
),
(
"github_issues_repo",
models.CharField(
help_text="The name of the repository on GitHub.",
max_length=255,
),
),
(
"github_issues_api_token",
models.CharField(
help_text="The API token used to authenticate requests to GitHub.",
max_length=255,
),
),
],
options={
"abstract": False,
},
bases=("database.datasync",),
),
]

704
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_github_issues_data_sync.py Normal file
View file

@ -0,0 +1,704 @@
import datetime
from copy import deepcopy
from decimal import Decimal
from django.test.utils import override_settings
from django.urls import reverse
import pytest
import responses
from baserow_premium.license.exceptions import FeaturesNotAvailableError
from baserow_premium.license.models import License
from rest_framework.status import HTTP_200_OK, HTTP_402_PAYMENT_REQUIRED
from baserow.contrib.database.data_sync.handler import DataSyncHandler
from baserow.contrib.database.fields.models import NumberField
from baserow.core.db import specific_iterator
from baserow_enterprise.data_sync.models import GitHubIssuesDataSync
SINGLE_ISSUE = {
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1",
"repository_url": "https://api.github.com/repos/baserow_owner/baserow_repo",
"labels_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/labels{/name}",
"comments_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/comments",
"events_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/events",
"html_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1",
"id": 1,
"node_id": "MDU6SXNzdWUx",
"number": 1,
"title": "Found a bug",
"user": {
"login": "octocat",
"id": 1,
"node_id": "U_MDU6SXNzdWUx",
"gravatar_id": "",
"type": "User",
"user_view_type": "public",
"site_admin": False,
},
"labels": [
{
"id": 1,
"node_id": "LA_MDU6SXNzdWUx",
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/labels/bug",
"name": "bug",
"color": "f29513",
"default": False,
"description": "A bug",
},
{
"id": 2,
"node_id": "LA_MDU6SXNzdWUx",
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/labels/bug",
"name": "feature",
"color": "f29513",
"default": False,
"description": "A feature",
},
],
"state": "open",
"locked": False,
"assignee": {"login": "octocat", "id": 1},
"assignees": [{"login": "octocat", "id": 1}, {"login": "bram", "id": 2}],
"milestone": {
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/milestones/1",
"html_url": "https://github.com/baserow_owner/baserow_repo/milestones/v1.0",
"labels_url": "https://api.github.com/repos/baserow_owner/baserow_repo/milestones/1/labels",
"id": 1,
"node_id": "MDk6TWlsZXN0b25lMTAwMjYwNA==",
"number": 1,
"state": "open",
"title": "v1.0",
"description": "Tracking milestone for version 1.0",
"creator": {
"login": "octocat",
"id": 1,
"node_id": "MDQ6VXNlcjE=",
"avatar_url": "https://github.com/images/error/octocat_happy.gif",
"gravatar_id": "",
"url": "https://api.github.com/users/octocat",
"html_url": "https://github.com/octocat",
"followers_url": "https://api.github.com/users/octocat/followers",
"following_url": "https://api.github.com/users/octocat/following{/other_user}",
"gists_url": "https://api.github.com/users/octocat/gists{/gist_id}",
"starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/octocat/subscriptions",
"organizations_url": "https://api.github.com/users/octocat/orgs",
"repos_url": "https://api.github.com/users/octocat/repos",
"events_url": "https://api.github.com/users/octocat/events{/privacy}",
"received_events_url": "https://api.github.com/users/octocat/received_events",
"type": "User",
"site_admin": False,
},
"open_issues": 4,
"closed_issues": 8,
"created_at": "2011-04-10T20:09:31Z",
"updated_at": "2014-03-03T18:58:10Z",
"closed_at": "2013-02-12T13:22:01Z",
"due_on": "2012-10-09T23:39:01Z",
},
"comments": 0,
"created_at": "2024-10-12T20:04:08Z",
"updated_at": "2024-10-12T20:22:23Z",
"closed_at": "2024-10-14T20:10:23Z",
"author_association": "NONE",
"active_lock_reason": None,
"body": "### Bug Description",
"closed_by": {
"login": "octocat",
"id": 1,
},
"reactions": {
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/reactions",
"total_count": 0,
"+1": 0,
"-1": 0,
"laugh": 0,
"hooray": 0,
"confused": 0,
"heart": 0,
"rocket": 0,
"eyes": 0,
},
"timeline_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/timeline",
"performed_via_github_app": None,
"state_reason": None,
}
SECOND_ISSUE = deepcopy(SINGLE_ISSUE)
SECOND_ISSUE["id"] = 2
SECOND_ISSUE["number"] = 2
SECOND_ISSUE["title"] = "Another bug"
SINGLE_ISSUE_RESPONSE = [SINGLE_ISSUE]
NO_ISSUES_RESPONSE = []
EMPTY_ISSUE = {
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1",
"repository_url": "https://api.github.com/repos/baserow_owner/baserow_repo",
"labels_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/labels{/name}",
"comments_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/comments",
"events_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/events",
"html_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1",
"id": 1,
"node_id": "MDU6SXNzdWUx",
"number": 1,
"title": "",
"user": None,
"labels": [],
"state": "open",
"locked": False,
"assignee": None,
"assignees": [],
"milestone": None,
"comments": 0,
"created_at": None,
"updated_at": None,
"closed_at": None,
"author_association": "NONE",
"active_lock_reason": None,
"body": "",
"closed_by": None,
"reactions": {
"url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/reactions",
"total_count": 0,
"+1": 0,
"-1": 0,
"laugh": 0,
"hooray": 0,
"confused": 0,
"heart": 0,
"rocket": 0,
"eyes": 0,
},
"timeline_url": "https://api.github.com/repos/baserow_owner/baserow_repo/issues/1/timeline",
"performed_via_github_app": None,
"state_reason": None,
}
EMPTY_ISSUE_RESPONSE = [EMPTY_ISSUE]
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_create_data_sync_table(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=[
"id",
"title",
"body",
"user",
"assignee",
"assignees",
"labels",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"milestone",
"url",
],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
assert isinstance(data_sync, GitHubIssuesDataSync)
assert data_sync.github_issues_owner == "baserow_owner"
assert data_sync.github_issues_repo == "baserow_repo"
assert data_sync.github_issues_api_token == "test"
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
assert len(fields) == 14
assert fields[0].name == "GitHub Issue ID"
assert isinstance(fields[0], NumberField)
assert fields[0].primary is True
assert fields[0].read_only is True
assert fields[0].immutable_type is True
assert fields[0].immutable_properties is True
assert fields[0].number_decimal_places == 0
assert fields[1].name == "Title"
assert fields[1].primary is False
assert fields[1].read_only is True
assert fields[1].immutable_type is True
assert fields[1].immutable_properties is True
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=1&per_page=50&state=all",
status=200,
json=SINGLE_ISSUE_RESPONSE,
)
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=2&per_page=50&state=all",
status=200,
json=NO_ISSUES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=[
"id",
"title",
"body",
"user",
"assignee",
"assignees",
"labels",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"milestone",
"url",
],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
github_id_field = fields[0]
title_field = fields[1]
body_field = fields[2]
user_field = fields[3]
assignee_field = fields[4]
assignees_field = fields[5]
labels_field = fields[6]
state_field = fields[7]
created_field = fields[8]
updated_field = fields[9]
closed_field = fields[10]
closed_by = fields[11]
milestone_field = fields[12]
url_field = fields[13]
model = data_sync.table.get_model()
assert model.objects.all().count() == 1
row = model.objects.all().first()
assert getattr(row, f"field_{github_id_field.id}") == Decimal("1")
assert getattr(row, f"field_{title_field.id}") == "Found a bug"
assert getattr(row, f"field_{body_field.id}") == "### Bug Description"
assert getattr(row, f"field_{user_field.id}") == "octocat"
assert getattr(row, f"field_{assignee_field.id}") == "octocat"
assert getattr(row, f"field_{assignees_field.id}") == "octocat, bram"
assert getattr(row, f"field_{labels_field.id}") == "bug, feature"
assert getattr(row, f"field_{state_field.id}") == "open"
assert getattr(row, f"field_{created_field.id}") == datetime.datetime(
2024, 10, 12, 20, 4, 8, tzinfo=datetime.timezone.utc
)
assert getattr(row, f"field_{updated_field.id}") == datetime.datetime(
2024, 10, 12, 20, 22, 23, tzinfo=datetime.timezone.utc
)
assert getattr(row, f"field_{closed_field.id}") == datetime.datetime(
2024, 10, 14, 20, 10, 23, tzinfo=datetime.timezone.utc
)
assert getattr(row, f"field_{closed_by.id}") == "octocat"
assert getattr(row, f"field_{milestone_field.id}") == "v1.0"
assert (
getattr(row, f"field_{url_field.id}")
== "https://github.com/baserow_owner/baserow_repo/issues/1"
)
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_sync_data_sync_table_empty_issue(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=1&per_page=50&state=all",
status=200,
json=EMPTY_ISSUE_RESPONSE,
)
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=2&per_page=50&state=all",
status=200,
json=NO_ISSUES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=[
"id",
"title",
"body",
"user",
"assignee",
"assignees",
"labels",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"milestone",
"url",
],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
github_id_field = fields[0]
title_field = fields[1]
body_field = fields[2]
user_field = fields[3]
assignee_field = fields[4]
assignees_field = fields[5]
labels_field = fields[6]
state_field = fields[7]
created_field = fields[8]
updated_field = fields[9]
closed_field = fields[10]
closed_by = fields[11]
milestone_field = fields[12]
url_field = fields[13]
model = data_sync.table.get_model()
assert model.objects.all().count() == 1
row = model.objects.all().first()
assert getattr(row, f"field_{github_id_field.id}") == Decimal("1")
assert getattr(row, f"field_{title_field.id}") == ""
assert getattr(row, f"field_{body_field.id}") == ""
assert getattr(row, f"field_{user_field.id}") == ""
assert getattr(row, f"field_{assignee_field.id}") == ""
assert getattr(row, f"field_{assignees_field.id}") == ""
assert getattr(row, f"field_{labels_field.id}") == ""
assert getattr(row, f"field_{state_field.id}") == "open"
assert getattr(row, f"field_{created_field.id}") is None
assert getattr(row, f"field_{updated_field.id}") is None
assert getattr(row, f"field_{closed_field.id}") is None
assert getattr(row, f"field_{closed_by.id}") == ""
assert getattr(row, f"field_{milestone_field.id}") == ""
assert (
getattr(row, f"field_{url_field.id}")
== "https://github.com/baserow_owner/baserow_repo/issues/1"
)
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_create_data_sync_table_pagination(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=1&per_page=50&state=all",
status=200,
json=SINGLE_ISSUE_RESPONSE,
)
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=2&per_page=50&state=all",
status=200,
json=[SECOND_ISSUE],
)
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues?page=3&per_page=50&state=all",
status=200,
json=NO_ISSUES_RESPONSE,
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=[
"id",
"title",
"body",
"user",
"assignee",
"assignees",
"labels",
"state",
"created_at",
"updated_at",
"closed_at",
"closed_by",
"milestone",
"url",
],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
data_sync = handler.sync_data_sync_table(user=user, data_sync=data_sync)
model = data_sync.table.get_model()
assert model.objects.all().count() == 2
@pytest.mark.django_db
@override_settings(DEBUG=True)
@responses.activate
def test_create_data_sync_table_invalid_auth(enterprise_data_fixture):
responses.add(
responses.GET,
"https://api.github.com/repos/baserow_owner/baserow_repo/issues",
status=401,
json={
"message": "Bad credentials",
"documentation_url": "https://docs.github.com/rest",
"status": "401",
},
)
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=["id"],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
data_sync = handler.sync_data_sync_table(user=user, data_sync=data_sync)
assert data_sync.last_error == "Bad credentials"
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_get_data_sync_properties(enterprise_data_fixture, api_client):
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "github_issues",
"github_issues_owner": "baserow_owner",
"github_issues_repo": "baserow_repo",
"github_issues_api_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{
"unique_primary": True,
"key": "id",
"name": "GitHub Issue ID",
"field_type": "number",
},
{
"unique_primary": False,
"key": "title",
"name": "Title",
"field_type": "text",
},
{
"unique_primary": False,
"key": "body",
"name": "Body",
"field_type": "long_text",
},
{"unique_primary": False, "key": "user", "name": "User", "field_type": "text"},
{
"unique_primary": False,
"key": "assignee",
"name": "Assignee",
"field_type": "text",
},
{
"unique_primary": False,
"key": "assignees",
"name": "Assignees",
"field_type": "text",
},
{
"unique_primary": False,
"key": "labels",
"name": "Labels",
"field_type": "text",
},
{
"unique_primary": False,
"key": "state",
"name": "State",
"field_type": "text",
},
{
"unique_primary": False,
"key": "created_at",
"name": "Created At",
"field_type": "date",
},
{
"unique_primary": False,
"key": "updated_at",
"name": "Updated At",
"field_type": "date",
},
{
"unique_primary": False,
"key": "closed_at",
"name": "Closed At",
"field_type": "date",
},
{
"unique_primary": False,
"key": "closed_by",
"name": "Closed By",
"field_type": "text",
},
{
"unique_primary": False,
"key": "milestone",
"name": "Milestone",
"field_type": "text",
},
{
"unique_primary": False,
"key": "url",
"name": "URL to Issue",
"field_type": "url",
},
]
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_create_data_sync_without_license(enterprise_data_fixture, api_client):
user, token = enterprise_data_fixture.create_user_and_token()
database = enterprise_data_fixture.create_database_application(user=user)
url = reverse("api:database:data_sync:list", kwargs={"database_id": database.id})
response = api_client.post(
url,
{
"table_name": "Test 1",
"type": "github_issues",
"synced_properties": ["github_id"],
"github_issues_owner": "baserow_owner",
"github_issues_repo": "baserow_repo",
"github_issues_api_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_402_PAYMENT_REQUIRED
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_sync_data_sync_table_without_license(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="github_issues",
synced_properties=["id"],
github_issues_owner="baserow_owner",
github_issues_repo="baserow_repo",
github_issues_api_token="test",
)
License.objects.all().delete()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)
@pytest.mark.django_db(transaction=True)
@override_settings(DEBUG=True)
def test_async_sync_data_sync_table_without_license(
api_client, enterprise_data_fixture
):
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
database = enterprise_data_fixture.create_database_application(user=user)
url = reverse("api:database:data_sync:list", kwargs={"database_id": database.id})
response = api_client.post(
url,
{
"table_name": "Test 1",
"type": "github_issues",
"synced_properties": ["id"],
"github_issues_owner": "baserow_owner",
"github_issues_repo": "baserow_repo",
"github_issues_api_token": "test",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
data_sync_id = response.json()["data_sync"]["id"]
License.objects.all().delete()
response = api_client.post(
reverse(
"api:database:data_sync:sync_table", kwargs={"data_sync_id": data_sync_id}
),
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_402_PAYMENT_REQUIRED

4
enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_local_baserow_table_data_sync_type.py
View file

@ -16,7 +16,9 @@ from baserow.contrib.database.fields.models import NumberField
from baserow.contrib.database.fields.registries import field_type_registry
from baserow.core.db import specific_iterator
from baserow.test_utils.helpers import setup_interesting_test_table
from baserow_enterprise.data_sync.data_sync_types import BaserowFieldDataSyncProperty
from baserow_enterprise.data_sync.baserow_table_data_sync import (
BaserowFieldDataSyncProperty,
)
from baserow_enterprise.data_sync.models import LocalBaserowTableDataSync

111
enterprise/web-frontend/modules/baserow_enterprise/components/dataSync/GitHubIssuesDataSyncForm.vue Normal file
View file

@ -0,0 +1,111 @@
<template>
<form @submit.prevent="submit">
<FormGroup
:error="fieldHasErrors('github_issues_owner')"
:label="$t('githubIssuesDataSync.owner')"
required
class="margin-bottom-2"
:helper-text="$t('githubIssuesDataSync.ownerHelper')"
small-label
>
<FormInput
v-model="values.github_issues_owner"
:error="fieldHasErrors('github_issues_owner')"
size="large"
@blur="$v.values.github_issues_owner.$touch()"
/>
<template #error>
<span
v-if="
$v.values.github_issues_owner.$dirty &&
!$v.values.github_issues_owner.required
"
>
{{ $t('error.requiredField') }}
</span>
</template>
</FormGroup>
<FormGroup
:error="fieldHasErrors('github_issues_repo')"
:label="$t('githubIssuesDataSync.repo')"
required
class="margin-bottom-2"
:helper-text="$t('githubIssuesDataSync.repoHelper')"
small-label
>
<FormInput
v-model="values.github_issues_repo"
:error="fieldHasErrors('github_issues_repo')"
size="large"
@blur="$v.values.github_issues_repo.$touch()"
/>
<template #error>
<span
v-if="
$v.values.github_issues_owner.$dirty &&
!$v.values.github_issues_owner.required
"
>
{{ $t('error.requiredField') }}
</span>
</template>
</FormGroup>
<FormGroup
:error="fieldHasErrors('github_issues_api_token')"
:label="$t('githubIssuesDataSync.apiToken')"
required
:helper-text="$t('githubIssuesDataSync.apiTokenHelper')"
small-label
>
<FormInput
v-model="values.github_issues_api_token"
:error="fieldHasErrors('github_issues_api_token')"
size="large"
@blur="$v.values.github_issues_api_token.$touch()"
/>
<template #error>
<span
v-if="
$v.values.github_issues_api_token.$dirty &&
!$v.values.github_issues_api_token.required
"
>
{{ $t('error.requiredField') }}
</span>
</template>
</FormGroup>
</form>
</template>
<script>
import { required } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
export default {
name: 'GitHubIssuesDataSyncForm',
mixins: [form],
data() {
return {
allowedValues: [
'github_issues_owner',
'github_issues_repo',
'github_issues_api_token',
],
values: {
github_issues_owner: '',
github_issues_repo: '',
github_issues_api_token: '',
},
}
},
validations: {
values: {
github_issues_owner: { required },
github_issues_repo: { required },
github_issues_api_token: { required },
},
},
}
</script>

28
enterprise/web-frontend/modules/baserow_enterprise/dataSyncTypes.js
View file

@ -4,6 +4,7 @@ import LocalBaserowTableDataSync from '@baserow_enterprise/components/dataSync/L
import EnterpriseFeatures from '@baserow_enterprise/features'
import EnterpriseModal from '@baserow_enterprise/components/EnterpriseModal'
import JiraIssuesDataSyncForm from '@baserow_enterprise/components/dataSync/JiraIssuesDataSyncForm'
import GitHubIssuesDataSyncForm from '@baserow_enterprise/components/dataSync/GitHubIssuesDataSyncForm'
export class LocalBaserowTableDataSyncType extends DataSyncType {
static getType() {
@ -58,3 +59,30 @@ export class JiraIssuesDataSyncType extends DataSyncType {
return EnterpriseModal
}
}
export class GitHubIssuesDataSyncType extends DataSyncType {
static getType() {
return 'github_issues'
}
getIconClass() {
return 'iconoir-github'
}
getName() {
const { i18n } = this.app
return i18n.t('enterpriseDataSyncType.githubIssues')
}
getFormComponent() {
return GitHubIssuesDataSyncForm
}
isDeactivated(workspaceId) {
return !this.app.$hasFeature(EnterpriseFeatures.DATA_SYNC, workspaceId)
}
getDeactivatedClickModal() {
return EnterpriseModal
}
}

11
enterprise/web-frontend/modules/baserow_enterprise/locales/en.json
View file

@ -338,7 +338,8 @@
},
"enterpriseDataSyncType": {
"localBaserowTable": "Sync Baserow table",
"jiraIssues": "Sync Jira issues"
"jiraIssues": "Sync Jira issues",
"githubIssues": "Sync GitHub issues"
},
"localBaserowTableDataSync": {
"name": "Source table ID",
@ -356,5 +357,13 @@
"usernameHelper": "The username of the Jira account used to authenticate. This is the email that you use to sign in.",
"apiToken": "Jira API token",
"apiTokenHelper": "The API token of the Jira account used for authentication. Can be created at https://id.atlassian.com/manage-profile/security/api-tokens."
},
"githubIssuesDataSync": {
"owner": "Owner",
"ownerHelper": "Username or owner of the repository. Typically in the top left corner `owner / repo`",
"repo": "Repository",
"repoHelper": "The name of the repository. Typically in the top left corner `owner / repo`",
"apiToken": "API token",
"apiTokenHelper": "Can be generated here https://github.com/settings/tokens by clicking on `Generate new token`, select `All repositories`, then under `Repository permissions` set `Issues` to `read-only`."
}
}

2
enterprise/web-frontend/modules/baserow_enterprise/plugin.js
View file

@ -41,6 +41,7 @@ import {
import {
LocalBaserowTableDataSyncType,
JiraIssuesDataSyncType,
GitHubIssuesDataSyncType,
} from '@baserow_enterprise/dataSyncTypes'
export default (context) => {
@ -124,4 +125,5 @@ export default (context) => {
app.$registry.register('dataSync', new LocalBaserowTableDataSyncType(context))
app.$registry.register('dataSync', new JiraIssuesDataSyncType(context))
app.$registry.register('dataSync', new GitHubIssuesDataSyncType(context))
}