Add XML import based on DOMParser

changelog.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+* Added support for importing tables from XML files.
 * Added support for different character encodings when importing CSV files.
 * Prevent websocket reconnect loop when the authentication fails.
 * Refactored the GridView component and improved interface speed.

web-frontend/modules/database/components/table/TableXMLImporter.vue

@@ -0,0 +1,164 @@
+<template>
+  <div>
+    <div class="control">
+      <label class="control__label">Choose XML file</label>
+      <div class="control__description">
+        You can import an existing XML by uploading the .XML file with tabular
+        data, i.e.:
+        <pre>
+&lt;notes&gt;
+  &lt;note&gt;
+    &lt;to&gt;Tove&lt;/to&gt;
+    &lt;from&gt;Jani&lt;/from&gt;
+    &lt;heading&gt;Reminder&lt;/heading&gt;
+    &lt;body&gt;Don't forget me this weekend!&lt;/body&gt;
+  &lt;/note&gt;
+  &lt;note&gt;
+    &lt;heading&gt;Reminder&lt;/heading&gt;
+    &lt;heading2&gt;Reminder2&lt;/heading2&gt;
+    &lt;to&gt;Tove&lt;/to&gt;
+    &lt;from&gt;Jani&lt;/from&gt;
+    &lt;body&gt;Don't forget me this weekend!&lt;/body&gt;
+  &lt;/note&gt;
+&lt;/notes&gt;</pre
+        >
+      </div>
+      <div class="control__elements">
+        <div class="file-upload">
+          <input
+            v-show="false"
+            ref="file"
+            type="file"
+            accept=".xml"
+            @change="select($event)"
+          />
+          <a
+            class="button button--large button--ghost file-upload__button"
+            @click.prevent="$refs.file.click($event)"
+          >
+            <i class="fas fa-cloud-upload-alt"></i>
+            Choose XML file
+          </a>
+          <div class="file-upload__file">{{ filename }}</div>
+        </div>
+        <div v-if="$v.filename.$error" class="error">
+          This field is required.
+        </div>
+      </div>
+    </div>
+    <div v-if="error !== ''" class="alert alert--error alert--has-icon">
+      <div class="alert__icon">
+        <i class="fas fa-exclamation"></i>
+      </div>
+      <div class="alert__title">Something went wrong</div>
+      <p class="alert__content">
+        {{ error }}
+      </p>
+    </div>
+    <TableImporterPreview
+      v-if="error === '' && Object.keys(preview).length !== 0"
+      :preview="preview"
+    ></TableImporterPreview>
+  </div>
+</template>
+
+<script>
+import { required } from 'vuelidate/lib/validators'
+
+import form from '@baserow/modules/core/mixins/form'
+import importer from '@baserow/modules/database/mixins/importer'
+import TableImporterPreview from '@baserow/modules/database/components/table/TableImporterPreview'
+import { parseXML } from '@baserow/modules/database/utils/xml'
+
+export default {
+  name: 'TableXMLImporter',
+  components: { TableImporterPreview },
+  mixins: [form, importer],
+  data() {
+    return {
+      values: {
+        data: '',
+        firstRowHeader: true,
+      },
+      filename: '',
+      error: '',
+      rawData: null,
+      preview: {},
+    }
+  },
+  validations: {
+    values: {
+      data: { required },
+    },
+    filename: { required },
+  },
+  methods: {
+    /**
+     * Method that is called when a file has been chosen. It will check if the file is
+     * not larger than 15MB. Otherwise it will take a long time and possibly a crash
+     * if so many entries have to be loaded into memory. If the file is valid, the
+     * contents will be loaded into memory and the reload method will be called which
+     * parses the content.
+     */
+    select(event) {
+      if (event.target.files.length === 0) {
+        return
+      }
+
+      const file = event.target.files[0]
+      const maxSize = 1024 * 1024 * 15
+
+      if (file.size > maxSize) {
+        this.filename = ''
+        this.values.data = ''
+        this.error = 'The maximum file size is 15MB.'
+        this.preview = {}
+        this.$emit('input', this.value)
+      } else {
+        this.filename = file.name
+        const reader = new FileReader()
+        reader.addEventListener('load', (event) => {
+          this.rawData = event.target.result
+          this.reload()
+        })
+        reader.readAsBinaryString(event.target.files[0])
+      }
+    },
+    reload() {
+      const [header, xmlData, errors] = parseXML(this.rawData)
+      if (errors.length > 0) {
+        this.values.data = ''
+        this.error = `Error occured while processing XML: ${errors.join('\n')}`
+        this.preview = {}
+      } else if (xmlData.length > 0) {
+        let hasHeader = false
+        if (header.length > 0) {
+          xmlData.unshift(header)
+          hasHeader = true
+        }
+
+        const limit = this.$env.INITIAL_TABLE_DATA_LIMIT
+        if (limit !== null) {
+          const count = xmlData.length
+          if (count > limit) {
+            this.values.data = ''
+            this.error = `It is not possible to import more than ${limit} rows.`
+            this.preview = {}
+            this.$emit('input', this.value)
+            return
+          }
+        }
+
+        this.values.data = JSON.stringify(xmlData)
+        this.error = ''
+        this.preview = this.getPreview(xmlData, hasHeader)
+      } else {
+        this.values.data = ''
+        this.error = 'This XML file is empty.'
+        this.preview = {}
+      }
+      this.$emit('input', this.value)
+    },
+  },
+}
+</script>

web-frontend/modules/database/importerTypes.js

@@ -2,6 +2,7 @@ import { Registerable } from '@baserow/modules/core/registry'
 
 import TableCSVImporter from '@baserow/modules/database/components/table/TableCSVImporter'
 import TablePasteImporter from '@baserow/modules/database/components/table/TablePasteImporter'
+import TableXMLImporter from '@baserow/modules/database/components/table/TableXMLImporter'
 
 export class ImporterType extends Registerable {
   /**
@@ -84,3 +85,21 @@ export class PasteImporterType extends ImporterType {
     return TablePasteImporter
   }
 }
+
+export class XMLImporterType extends ImporterType {
+  getType() {
+    return 'xml'
+  }
+
+  getIconClass() {
+    return 'file-code'
+  }
+
+  getName() {
+    return 'Import an XML file'
+  }
+
+  getFormComponent() {
+    return TableXMLImporter
+  }
+}

web-frontend/modules/database/plugin.js

@@ -32,6 +32,7 @@ import {
 import {
   CSVImporterType,
   PasteImporterType,
+  XMLImporterType,
 } from '@baserow/modules/database/importerTypes'
 import { APITokenSettingsType } from '@baserow/modules/database/settingsTypes'
 
@@ -77,6 +78,7 @@ export default ({ store, app }) => {
   app.$registry.register('field', new PhoneNumberFieldType())
   app.$registry.register('importer', new CSVImporterType())
   app.$registry.register('importer', new PasteImporterType())
+  app.$registry.register('importer', new XMLImporterType())
   app.$registry.register('settings', new APITokenSettingsType())
 
   registerRealtimeEvents(app.$realtime)

web-frontend/modules/database/utils/xml.js

@@ -0,0 +1,34 @@
+/**
+ * Parses a rawXML string and extracts tabular data from it.
+ */
+export const parseXML = (rawXML) => {
+  let xmlData = []
+  const header = []
+  const xmlDoc = new window.DOMParser().parseFromString(rawXML, 'text/xml')
+  const parseErrors = xmlDoc.getElementsByTagName('parsererror')
+  const errors = []
+  if (parseErrors.length > 0) {
+    Array.from(parseErrors).forEach((parseError) =>
+      errors.push(parseError.textContent)
+    )
+  }
+  if (xmlDoc && xmlDoc.documentElement && xmlDoc.documentElement.children) {
+    xmlData = Array.from(xmlDoc.documentElement.children).map((row) => {
+      const vals = Array.from(row.children).map((rowChild) => {
+        const rowTag = rowChild.tagName
+        if (!header.includes(rowTag)) {
+          header.push(rowTag)
+        }
+        return { tag: rowTag, value: rowChild.innerHTML }
+      })
+      return vals
+    })
+  }
+  xmlData = xmlData.map((line) => {
+    return header.map((h) => {
+      const lineValue = line.filter((lv) => lv.tag === h)
+      return lineValue.length > 0 ? lineValue[0].value : ''
+    })
+  })
+  return [header, xmlData, errors]
+}

web-frontend/test/database/utils/xml.spec.js

@@ -0,0 +1,47 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import { parseXML } from '@/modules/database/utils/xml'
+
+describe('test xml utils', () => {
+  test('test xml parser', () => {
+    const [header, xmlData, errors] = parseXML(`
+<notes>
+  <note>
+      <to>Tove</to>
+      <from>Jani</from>
+      <heading>Reminder</heading>
+      <body>Don't forget me this weekend!</body>
+  </note>
+  <note>
+      <heading>Reminder</heading>
+      <heading2>Reminder2</heading2>
+      <to>Tove</to>
+      <from>Jani</from>
+      <body>Don't forget me this weekend!</body>
+  </note>
+</notes>
+    `)
+    expect(errors.length).toBe(0)
+    expect(header.length).toBe(5)
+    expect(header[0]).toBe('to')
+    expect(header[1]).toBe('from')
+    expect(header[2]).toBe('heading')
+    expect(header[3]).toBe('body')
+    expect(header[4]).toBe('heading2')
+    expect(xmlData.length).toBe(2)
+    expect(xmlData[0].length).toBe(5)
+    expect(xmlData[1].length).toBe(5)
+    expect(xmlData[0][0]).toBe('Tove')
+    expect(xmlData[0][1]).toBe('Jani')
+    expect(xmlData[0][2]).toBe('Reminder')
+    expect(xmlData[0][3]).toBe("Don't forget me this weekend!")
+    expect(xmlData[0][4]).toBe('')
+    expect(xmlData[1][0]).toBe('Tove')
+    expect(xmlData[1][1]).toBe('Jani')
+    expect(xmlData[1][2]).toBe('Reminder')
+    expect(xmlData[1][3]).toBe("Don't forget me this weekend!")
+    expect(xmlData[1][4]).toBe('Reminder2')
+  })
+})