import json import re import shutil from pathlib import Path # Dictionary responsible for making the symbolic links at the end of the script's run. symlink_dict = {} def cleanup(): """ clean directories that are either data collection or exporting integrations """ for element in Path("src/go/plugin/go.d/modules").glob('**/*/'): if "integrations" in str(element): shutil.rmtree(element) for element in Path("src/collectors").glob('**/*/'): # print(element) if "integrations" in str(element): shutil.rmtree(element) for element in Path("src/exporting").glob('**/*/'): if "integrations" in str(element): shutil.rmtree(element) for element in Path("integrations/cloud-notifications").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) for element in Path("integrations/cloud-authentication").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) def generate_category_from_name(category_fragment, category_array): """ Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects. """ category_name = "" i = 0 dummy_id = category_fragment[0] while i < len(category_fragment): for category in category_array: if dummy_id == category['id']: category_name = category_name + "/" + category["name"] try: # print("equals") # print(fragment, category_fragment[i+1]) dummy_id = dummy_id + "." + category_fragment[i + 1] # print(dummy_id) except IndexError: return category_name.split("/", 1)[1] category_array = category['children'] break i += 1 def clean_and_write(md, path): """ This function takes care of the special details element, and converts it to the equivalent that md expects. Then it writes the buffer on the file provided. """ # clean first, replace md = md.replace("{% details summary=\"", "<details><summary>") md = md.replace("{% details open=true summary=\"", "<details open><summary>") md = md.replace("\" %}", "</summary>\n") md = md.replace("{% /details %}", "</details>\n") path.write_text(md) def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'): """ Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field """ output = "" if mode == 'default': path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}' elif mode == 'cloud-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') elif mode == 'agent-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README") elif mode == 'cloud-authentication': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') output = markdown_string.replace( "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"') return output def clean_string(string): """ simple function to get rid of caps, spaces, slashes and parentheses from a given string The string represents an integration name, as it would be displayed in the final text """ return string.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "").replace(":", "") def read_integrations_js(path_to_file): """ Open integrations/integrations.js and extract the dictionaries """ try: data = Path(path_to_file).read_text() categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0] integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1] return json.loads(categories_str), json.loads(integrations_str) except FileNotFoundError as e: print("Exception", e) def create_overview(integration, filename, overview_key_name="overview"): # empty overview_key_name to have only image on overview if not overview_key_name: return f"""# {integration['meta']['name']} <img src="https://netdata.cloud/img/{filename}" width="150"/> """ split = re.split(r'(#.*\n)', integration[overview_key_name], 1) first_overview_part = split[1] rest_overview_part = split[2] if not filename: return f"""{first_overview_part}{rest_overview_part} """ return f"""{first_overview_part} <img src="https://netdata.cloud/img/{filename}" width="150"/> {rest_overview_part} """ def build_readme_from_integration(integration, mode=''): # COLLECTORS if mode == 'collector': try: # initiate the variables for the collector meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['monitored_instance']['name'] learn_rel_path = generate_category_from_name( integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace( "Data Collection", "Collecting Metrics") most_popular = integration['meta']['most_popular'] # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path}" most_popular: {most_popular} message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" endmeta--> {create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}""" if integration['metrics']: md += f""" {integration['metrics']} """ if integration['alerts']: md += f""" {integration['alerts']} """ if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in collector md construction", e, integration['id']) # EXPORTERS elif mode == 'exporter': try: # initiate the variables for the exporter meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "Exporting Metrics" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE EXPORTER'S metadata.yaml FILE" endmeta--> {create_overview(integration, integration['meta']['icon_filename'])}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in exporter md construction", e, integration['id']) # NOTIFICATIONS elif mode == 'agent-notification': try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE" endmeta--> {create_overview(integration, integration['meta']['icon_filename'], "overview")}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in notification md construction", e, integration['id']) elif mode == 'cloud-notification': try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE" endmeta--> {create_overview(integration, integration['meta']['icon_filename'], "")}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ except Exception as e: print("Exception in notification md construction", e, integration['id']) # AUTHENTICATIONS elif mode == 'authentication': if True: # initiate the variables for the authentication method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path.replace("authentication", "Netdata Cloud/Authentication & Authorization/Cloud Authentication & Authorization Integrations")}" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE AUTHENTICATION'S metadata.yaml FILE" endmeta--> {create_overview(integration, integration['meta']['icon_filename'])}""" if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ # except Exception as e: # print("Exception in authentication md construction", e, integration['id']) if "community" in integration['meta'].keys(): community = "<img src=\"https://img.shields.io/badge/maintained%20by-Community-blue\" />" else: community = "<img src=\"https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44\" />" return meta_yaml, sidebar_label, learn_rel_path, md, community def build_path(meta_yaml_link): """ function that takes a metadata yaml file link, and makes it into a path that gets used to write to a file. """ return meta_yaml_link.replace("https://github.com/netdata/", "") \ .split("/", 1)[1] \ .replace("edit/master/", "") \ .replace("/metadata.yaml", "") def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default'): """ takes the arguments needed to write the integration markdown to the proper file. """ upper, lower = md.split("##", 1) md = upper + community + f"\n\n##{lower}" if mode == 'default': # Only if the path exists, this caters for running the same script on both the go and netdata repos. if Path(path).exists(): if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() try: md = add_custom_edit_url(md, meta_yaml, sidebar_label) clean_and_write( md, Path(f'{path}/integrations/{clean_string(sidebar_label)}.md') ) except FileNotFoundError as e: print("Exception in writing to file", e) # If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link if len(list(Path(f'{path}/integrations').iterdir())) == 1: symlink_dict.update( {path: f'integrations/{clean_string(sidebar_label)}.md'}) else: try: symlink_dict.pop(path) except KeyError: # We don't need to print something here. pass elif mode == 'cloud-notification': # for cloud notifications we generate them near their metadata.yaml name = clean_string(integration['meta']['name']) if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() # proper_edit_name = meta_yaml.replace( # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification') finalpath = f'{path}/integrations/{name}.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) elif mode == 'agent-notification': # add custom_edit_url as the md file, so we can have uniqueness in the ingest script # afterwards the ingest will replace this metadata with meta_yaml md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification') finalpath = f'{path}/README.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) elif mode == 'authentication': name = clean_string(integration['meta']['name']) if not Path(f'{path}/integrations').exists(): Path(f'{path}/integrations').mkdir() # proper_edit_name = meta_yaml.replace( # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication') finalpath = f'{path}/integrations/{name}.md' try: clean_and_write( md, Path(finalpath) ) except FileNotFoundError as e: print("Exception in writing to file", e) def make_symlinks(symlink_dict): """ takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder. """ for element in symlink_dict: if not Path(f'{element}/README.md').exists(): Path(f'{element}/README.md').touch() try: # Remove the README to prevent it being a normal file Path(f'{element}/README.md').unlink() except FileNotFoundError: continue # and then make a symlink to the actual markdown Path(f'{element}/README.md').symlink_to(symlink_dict[element]) filepath = Path(f'{element}/{symlink_dict[element]}') md = filepath.read_text() # This preserves the custom_edit_url for most files as it was, # so the existing links don't break, this is vital for link replacement afterwards filepath.write_text(md.replace( f'{element}/{symlink_dict[element]}', f'{element}/README.md')) cleanup() categories, integrations = read_integrations_js('integrations/integrations.js') # Iterate through every integration for integration in integrations: if integration['integration_type'] == "collector": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='collector') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community) else: # kind of specific if clause, so we can avoid running excessive code in the go repo if integration['integration_type'] == "exporter": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='exporter') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community) elif integration['integration_type'] == "agent_notification": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='agent-notification') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification') elif integration['integration_type'] == "cloud_notification": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='cloud-notification') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification') elif integration['integration_type'] == "authentication": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='authentication') path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication') make_symlinks(symlink_dict)