archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-02 20:48:06 +00:00
Code Issues Projects Releases Wiki Activity

Add integrations JSON file for website usage. ()

Browse source 
* Add generation of JSON integrations data.

* Clean up generated data for JSON file.

* Properly clean up deploy entry info.

* Fix argument order for regex substitutions.
This commit is contained in:
Austin S. Hemmelgarn 2023-10-18 11:30:30 -04:00 • committed by GitHub
parent a27aed521f
commit 8934a18ce4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 117 additions and 39 deletions

21
integrations/deploy.yaml
View file

@ -35,6 +35,8 @@
--stable-channel{% if $showClaimingOptions %} --claim-token {% claim_token %} --claim-rooms {% $claim_rooms %} --claim-url {% claim_url %}{% /if %}
additional_info: &ref_containers >
Did you know you can also deploy Netdata on your OS using {% goToCategory navigateToSettings=$navigateToSettings categoryId="deploy.docker-kubernetes" %}Kubernetes{% /goToCategory %} or {% goToCategory categoryId="deploy.docker-kubernetes" %}Docker{% /goToCategory %}?
clean_additional_info: &ref_clean_containers >
Did you know you can also deploy Netdata on your OS using Kubernetes or Docker?
related_resources: {}
platform_info:
group: ''
@ -85,7 +87,7 @@
icon_filename: 'rhel.png'
most_popular: false
platform_info:
group: 'include'
group: 'no_include'
distro: 'rhel'
quick_start: -1
- <<: *linux
@ -148,6 +150,18 @@
group: 'include'
distro: 'centos'
quick_start: -1
- <<: *linux
id: deploy-centos-stream
meta:
<<: *linux_meta
name: CentOS Stream
link: https://www.centos.org/centos-stream
icon_filename: 'centos.png'
most_popular: false
platform_info:
group: 'include'
distro: 'centos-stream'
quick_start: -1
- <<: *linux
id: deploy-manjarolinux
meta:
@ -200,9 +214,10 @@
methods:
- *ks_curl
additional_info: *ref_containers
clean_additional_info: *ref_clean_containers
related_resources: {}
platform_info:
group: 'include'
group: 'no_include'
distro: 'macos'
quick_start: 5
- id: deploy-docker
@ -573,6 +588,6 @@
Netdata can also be installed via [FreeBSD ports](https://www.freshports.org/net-mgmt/netdata).
related_resources: {}
platform_info:
group: 'include'
group: 'no_include'
distro: 'freebsd'
quick_start: 6

119
integrations/gen_integrations.py
View file

@ -2,8 +2,10 @@
import json
import os
import re
import sys
from copy import deepcopy
from pathlib import Path
from jsonschema import Draft7Validator, ValidationError
@ -17,6 +19,7 @@ GO_REPO = 'netdata/go.d.plugin'
INTEGRATIONS_PATH = Path(__file__).parent
TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates'
OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js'
JSON_PATH = INTEGRATIONS_PATH / 'integrations.json'
CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml'
REPO_PATH = INTEGRATIONS_PATH.parent
SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
@ -65,6 +68,9 @@ NOTIFICATION_RENDER_KEYS = [
'troubleshooting',
]
CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False)
DEBUG = os.environ.get('DEBUG', False)
@ -438,12 +444,17 @@ def render_collectors(categories, collectors, ids):
debug('Removing duplicate collectors.')
collectors, ids = dedupe_integrations(collectors, ids)
clean_collectors = []
idmap = {i['id']: i for i in collectors}
for item in collectors:
debug(f'Processing { item["id"] }.')
item['edit_link'] = make_edit_link(item)
clean_item = deepcopy(item)
related = []
for res in item['meta']['related_resources']['integrations']['list']:
@ -487,23 +498,27 @@ def render_collectors(categories, collectors, ids):
for key in COLLECTOR_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
data = template.render(entry=item, related=related)
data = template.render(entry=item, related=related, clean=False)
clean_data = template.render(entry=item, related=related, clean=True)
if 'variables' in item['meta']['monitored_instance']:
template = get_jinja_env().from_string(data)
data = template.render(variables=item['meta']['monitored_instance']['variables'])
template = get_jinja_env().from_string(clean_data)
clean_data = template.render(variables=item['meta']['monitored_instance']['variables'])
else:
data = ''
clean_data = ''
item[key] = data
clean_item[key] = clean_data
item['edit_link'] = make_edit_link(item)
for k in ['_src_path', '_repo', '_index']:
del item[k], clean_item[k]
del item['_src_path']
del item['_repo']
del item['_index']
clean_collectors.append(clean_item)
return collectors, ids
return collectors, clean_collectors, ids
def render_deploy(distros, categories, deploy, ids):
@ -514,11 +529,14 @@ def render_deploy(distros, categories, deploy, ids):
debug('Checking deployment ids.')
deploy, ids = dedupe_integrations(deploy, ids)
clean_deploy = []
template = get_jinja_env().get_template('platform_info.md')
for item in deploy:
debug(f'Processing { item["id"] }.')
item['edit_link'] = make_edit_link(item)
clean_item = deepcopy(item)
if item['platform_info']['group']:
entries = [
@ -532,16 +550,27 @@ def render_deploy(distros, categories, deploy, ids):
else:
entries = []
data = template.render(entries=entries)
data = template.render(entries=entries, clean=False)
clean_data = template.render(entries=entries, clean=True)
for method in clean_item['methods']:
for command in method['commands']:
command['command'] = CUSTOM_TAG_PATTERN.sub('', command['command'])
command['command'] = FIXUP_BLANK_PATTERN.sub('', command['command'])
item['platform_info'] = data
item['edit_link'] = make_edit_link(item)
clean_item['platform_info'] = clean_data
del item['_src_path']
del item['_repo']
del item['_index']
if 'clean_additional_info' in item:
clean_item['additional_info'] = item['clean_additional_info']
del item['clean_additional_info'], clean_item['clean_additional_info']
return deploy, ids
for k in ['_src_path', '_repo', '_index']:
del item[k], clean_item[k]
clean_deploy.append(clean_item)
return deploy, clean_deploy, ids
def render_exporters(categories, exporters, ids):
@ -553,27 +582,37 @@ def render_exporters(categories, exporters, ids):
exporters, ids = dedupe_integrations(exporters, ids)
clean_exporters = []
for item in exporters:
item['edit_link'] = make_edit_link(item)
clean_item = deepcopy(item)
for key in EXPORTER_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
data = template.render(entry=item)
data = template.render(entry=item, clean=False)
clean_data = template.render(entry=item, clean=True)
if 'variables' in item['meta']:
template = get_jinja_env().from_string(data)
data = template.render(variables=item['meta']['variables'])
data = template.render(variables=item['meta']['variables'], clean=False)
template = get_jinja_env().from_string(clean_data)
clean_data = template.render(variables=item['meta']['variables'], clean=True)
else:
data = ''
clean_data = ''
item[key] = data
clean_item[key] = clean_data
item['edit_link'] = make_edit_link(item)
for k in ['_src_path', '_repo', '_index']:
del item[k], clean_item[k]
del item['_src_path']
del item['_repo']
del item['_index']
clean_exporters.append(clean_item)
return exporters, ids
return exporters, clean_exporters, ids
def render_notifications(categories, notifications, ids):
@ -585,27 +624,37 @@ def render_notifications(categories, notifications, ids):
notifications, ids = dedupe_integrations(notifications, ids)
clean_notifications = []
for item in notifications:
item['edit_link'] = make_edit_link(item)
clean_item = deepcopy(item)
for key in NOTIFICATION_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
data = template.render(entry=item)
data = template.render(entry=item, clean=False)
clean_data = template.render(entry=item, clean=True)
if 'variables' in item['meta']:
template = get_jinja_env().from_string(data)
data = template.render(variables=item['meta']['variables'])
data = template.render(variables=item['meta']['variables'], clean=False)
template = get_jinja_env().from_string(clean_data)
clean_data = template.render(variables=item['meta']['variables'], clean=True)
else:
data = ''
clean_data = ''
item[key] = data
clean_item[key] = clean_data
item['edit_link'] = make_edit_link(item)
for k in ['_src_path', '_repo', '_index']:
del item[k], clean_item[k]
del item['_src_path']
del item['_repo']
del item['_index']
clean_notifications.append(clean_item)
return notifications, ids
return notifications, clean_notifications, ids
def render_integrations(categories, integrations):
@ -617,6 +666,13 @@ def render_integrations(categories, integrations):
OUTPUT_PATH.write_text(data)
def render_json(categories, integrations):
JSON_PATH.write_text(json.dumps({
'categories': categories,
'integrations': integrations,
}))
def main():
categories = load_categories()
distros = load_yaml(DISTROS_FILE)
@ -625,14 +681,17 @@ def main():
exporters = load_exporters()
notifications = load_notifications()
collectors, ids = render_collectors(categories, collectors, dict())
deploy, ids = render_deploy(distros, categories, deploy, ids)
exporters, ids = render_exporters(categories, exporters, ids)
notifications, ids = render_notifications(categories, notifications, ids)
collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
notifications, clean_notifications, ids = render_notifications(categories, notifications, ids)
integrations = collectors + deploy + exporters + notifications
render_integrations(categories, integrations)
clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications
render_json(categories, clean_integrations)
if __name__ == '__main__':
sys.exit(main())

4
integrations/schemas/deploy.json
View file

@ -68,6 +68,10 @@
"type": "string",
"description": "Any additional information about this platform."
},
"clean_additional_info": {
"type": "string",
"description": "Any additional information about this platform, without any embedded custom tags."
},
"related_resources": {
"type": "object",
"description": "TBD"

4
integrations/templates/metrics.md
View file

@ -1,7 +1,7 @@
[% if entry.metrics.scopes %]
## Metrics
[% if entry.metrics.folding.enabled %]
[% if entry.metrics.folding.enabled and not clean %]
{% details summary="[[ entry.metrics.folding.title ]]" %}
[% endif %]
Metrics grouped by *scope*.
@ -39,7 +39,7 @@ Metrics:
[% endfor %]
[% endfor %]
[% if entry.metrics.folding.enabled %]
[% if entry.metrics.folding.enabled and not clean %]
{% /details %}
[% endif %]
[% else %]

8
integrations/templates/setup.md
View file

@ -59,7 +59,7 @@ There is no configuration file.
[[ entry.setup.configuration.options.description ]]
[% if entry.setup.configuration.options.list %]
[% if entry.setup.configuration.options.folding.enabled %]
[% if entry.setup.configuration.options.folding.enabled and not clean %]
{% details summary="[[ entry.setup.configuration.options.folding.title ]]" %}
[% endif %]
| Name | Description | Default | Required |
@ -76,7 +76,7 @@ There is no configuration file.
[% endif %]
[% endfor %]
[% if entry.setup.configuration.options.folding.enabled %]
[% if entry.setup.configuration.options.folding.enabled and not clean %]
{% /details %}
[% endif %]
[% elif not entry.setup.configuration.options.description %]
@ -91,13 +91,13 @@ There are no configuration options.
[[ example.description ]]
[% if example.folding.enabled %]
[% if example.folding.enabled and not clean %]
{% details summary="[[ entry.setup.configuration.examples.folding.title ]]" %}
[% endif %]
```yaml
[[ example.config ]]
```
[% if example.folding.enabled %]
[% if example.folding.enabled and not clean %]
{% /details %}
[% endif %]
[% endfor %]