import json import os # Dictionary responsible for making the symbolic links at the end of the script's run. symlink_dict = {} def generate_category_from_name(category_fragment, category_array): """ Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects. """ category_name = "" i = 0 dummy_id = category_fragment[0] while i < len(category_fragment): for category in category_array: if dummy_id == category['id']: category_name = category_name + "/" + category["name"] try: # print("equals") # print(fragment, category_fragment[i+1]) dummy_id = dummy_id + "." + category_fragment[i+1] # print(dummy_id) except IndexError: return category_name.split("/", 1)[1] category_array = category['children'] break i += 1 def clean_and_write(md, txt): """ This function takes care of the special details element, and converts it to the equivalent that md expects. Then it writes the buffer on the file provided. """ # clean first, replace md = md.replace("{% details summary=\"", "<details><summary>").replace( "\" %}", "</summary>\n").replace("{% /details %}", "</details>\n") # print(md) # exit() txt.write(md) # Open integrations/integrations.js and extract the dictionaries with open('integrations/integrations.js') as dataFile: data = dataFile.read() categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0] integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1] categories = json.loads(categories_str) integrations = json.loads(integrations_str) i = 0 # Iterate through every integration for integration in integrations: i += 1 if integration['integration_type'] == "collector": try: # initiate the variables for the collector meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['monitored_instance']['name'] learn_rel_path = generate_category_from_name( integration['meta']['monitored_instance']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path}" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" endmeta--> {integration['overview']} """ if integration['metrics']: md += f""" {integration['metrics']} """ if integration['alerts']: md += f""" {integration['alerts']} """ if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ path = meta_yaml.replace("https://github.com/netdata/", "") \ .split("/", 1)[1] \ .replace("edit/master/", "") \ .replace("/metadata.yaml", "") # Only if the path exists, this caters for running the same script on both the go and netdata repos. if os.path.exists(path): try: if not os.path.exists(f'{path}/integrations'): os.mkdir(f'{path}/integrations') with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md', 'w+') as txt: # add custom_edit_url as the md file, so we can have uniqueness in the ingest script # afterwards the ingest will replace this metadata with meta_yaml md = md.replace( "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"') clean_and_write(md, txt) except Exception as e: print("Error in writing to the collector file", e, integration['id']) # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link if len(os.listdir(f'{path}/integrations')) == 1: symlink_dict.update( {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md'}) else: try: symlink_dict.pop(path) except KeyError: # We don't need to print something here. pass except Exception as e: print("Exception in collector md construction", e, integration['id']) # kind of specific if clause, so we can avoid running excessive code in the go repo elif integration['integration_type'] == "exporter" and "go.d.plugin" not in os.getcwd(): try: # initiate the variables for the exporter meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "Exporting" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE EXPORTER'S metadata.yaml FILE" endmeta--> {integration['overview']} """ if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ path = meta_yaml.replace("https://github.com/netdata/", "") \ .split("/", 1)[1] \ .replace("edit/master/", "") \ .replace("/metadata.yaml", "") if os.path.exists(path): try: if not os.path.exists(f'{path}/integrations'): os.mkdir(f'{path}/integrations') with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md', 'w+') as txt: # add custom_edit_url as the md file, so we can have uniqueness in the ingest script # afterwards the ingest will replace this metadata with meta_yaml md = md.replace( "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"') clean_and_write(md, txt) except Exception as e: print("Error in writing to the file", e, integration['id']) # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link if len(os.listdir(f'{path}/integrations')) == 1: symlink_dict.update( {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md'}) else: try: symlink_dict.pop(path) except KeyError: # We don't need to print something here. pass except Exception as e: print("Exception in exporter md construction", e, integration['id']) # kind of specific if clause, so we can avoid running excessive code in the go repo elif integration['integration_type'] == "notification" and "go.d.plugin" not in os.getcwd(): try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") sidebar_label = integration['meta']['name'] learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) # build the markdown string md = \ f"""<!--startmeta meta_yaml: "{meta_yaml}" sidebar_label: "{sidebar_label}" learn_status: "Published" learn_rel_path: "{learn_rel_path.replace("notifications", "Alerting/Notifications")}" message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE" endmeta--> {integration['overview']} """ if integration['setup']: md += f""" {integration['setup']} """ if integration['troubleshooting']: md += f""" {integration['troubleshooting']} """ path = meta_yaml.replace("https://github.com/netdata/", "") \ .split("/", 1)[1] \ .replace("edit/master/", "") \ .replace("/metadata.yaml", "") if "cloud-notifications" in path: # for cloud notifications we generate them near their metadata.yaml name = integration['meta']['name'].lower().replace(" ", "_") if not os.path.exists(f'{path}/integrations'): os.mkdir(f'{path}/integrations') proper_edit_name = meta_yaml.replace( "metadata.yaml", f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"') md = md.replace("<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{proper_edit_name}') finalpath = f'{path}/integrations/{name}.md' else: # add custom_edit_url as the md file, so we can have uniqueness in the ingest script # afterwards the ingest will replace this metadata with meta_yaml md = md.replace("<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("metadata.yaml", "README.md")}') finalpath = f'{path}/README.md' try: with open(finalpath, 'w') as txt: clean_and_write(md, txt) except Exception as e: print("Exception in notification md construction", e, integration['id']) except Exception as e: print("Exception in for loop", e, "\n", integration) for element in symlink_dict: # Remove the README to prevent it being a normal file os.remove(f'{element}/README.md') # and then make a symlink to the actual markdown os.symlink(symlink_dict[element], f'{element}/README.md') with open(f'{element}/{symlink_dict[element]}', 'r') as txt: md = txt.read() # This preserves the custom_edit_url for most files as it was, # so the existing links don't break, this is vital for link replacement afterwards with open(f'{element}/{symlink_dict[element]}', 'w+') as txt: md = md.replace(f'{element}/{symlink_dict[element]}', f'{element}/README.md') txt.write(md)