Move plugins.d directory outside of collectors (#18637)

* Move plugins.d out of collectors

It's being used by streaming as well.

* Move ndsudo and local_listeners back to collectors.

.github/labeler.yml

@@ -77,12 +77,6 @@ area/collectors:
               - src/collectors/**
               - src/go/**
 
-collectors/plugins.d:
-  - any:
-      - changed-files:
-          - any-glob-to-any-file:
-              - src/collectors/plugins.d/**
-
 collectors/apps:
   - any:
       - changed-files:
@@ -289,3 +283,9 @@ area/web:
       - changed-files:
           - any-glob-to-any-file:
               - src/web/**
+
+area/plugins.d:
+  - any:
+      - changed-files:
+          - any-glob-to-any-file:
+              - src/plugins.d/**

CMakeLists.txt

@@ -1136,18 +1136,18 @@ set(INTERNAL_COLLECTORS_FILES
 )
 
 set(PLUGINSD_PLUGIN_FILES
-        src/collectors/plugins.d/plugins_d.c
-        src/collectors/plugins.d/plugins_d.h
-        src/collectors/plugins.d/pluginsd_dyncfg.c
-        src/collectors/plugins.d/pluginsd_dyncfg.h
-        src/collectors/plugins.d/pluginsd_functions.c
-        src/collectors/plugins.d/pluginsd_functions.h
-        src/collectors/plugins.d/pluginsd_internals.c
-        src/collectors/plugins.d/pluginsd_internals.h
-        src/collectors/plugins.d/pluginsd_parser.c
-        src/collectors/plugins.d/pluginsd_parser.h
-        src/collectors/plugins.d/pluginsd_replication.c
-        src/collectors/plugins.d/pluginsd_replication.h
+        src/plugins.d/plugins_d.c
+        src/plugins.d/plugins_d.h
+        src/plugins.d/pluginsd_dyncfg.c
+        src/plugins.d/pluginsd_dyncfg.h
+        src/plugins.d/pluginsd_functions.c
+        src/plugins.d/pluginsd_functions.h
+        src/plugins.d/pluginsd_internals.c
+        src/plugins.d/pluginsd_internals.h
+        src/plugins.d/pluginsd_parser.c
+        src/plugins.d/pluginsd_parser.h
+        src/plugins.d/pluginsd_replication.c
+        src/plugins.d/pluginsd_replication.h
 )
 
 set(RRD_PLUGIN_FILES
@@ -2068,7 +2068,7 @@ if(ENABLE_PLUGIN_CUPS)
 endif()
 
 if(NEED_NDSUDO)
-    set(NDSUDO_FILES src/collectors/plugins.d/ndsudo.c)
+    set(NDSUDO_FILES src/collectors/utils/ndsudo.c)
 
     add_executable(ndsudo ${NDSUDO_FILES})
 
@@ -2212,7 +2212,7 @@ endif()
 
 if(ENABLE_PLUGIN_LOCAL_LISTENERS)
         set(LOCAL_LISTENERS_FILES
-                src/collectors/plugins.d/local_listeners.c
+                src/collectors/utils/local_listeners.c
                 src/libnetdata/maps/local-sockets.h
         )
 

docs/developer-and-contributor-corner/python-collector.md

@@ -73,7 +73,7 @@ The basic elements of a Netdata collector are:
 - `get_data()`: The basic function of the plugin which will return to Netdata the correct values.
 
 **Note**: All names are better explained in the 
-[External Plugins Documentation](/src/collectors/plugins.d/README.md).
+[External Plugins Documentation](/src/plugins.d/README.md).
 Parameters like `priority` and `update_every` mentioned in that documentation are handled by the `python.d.plugin`,
 not by each collection module. 
 

docs/diagrams/netdata-overview.xml

@@ -78,7 +78,7 @@
             <mxGeometry x="254" y="690" width="770" height="93" as="geometry" />
           </mxCell>
         </UserObject>
-        <UserObject label="&lt;div&gt;&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;plugins.d&lt;/b&gt;&lt;/font&gt;&lt;/div&gt;&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;external plugins API&lt;/b&gt;&lt;/font&gt;&lt;div&gt;&lt;font style=&quot;font-size: 14px&quot;&gt;text format, over pipes&lt;/font&gt;&lt;/div&gt;" link="https://github.com/netdata/netdata/tree/master/src/collectors/plugins.d" id="32">
+        <UserObject label="&lt;div&gt;&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;plugins.d&lt;/b&gt;&lt;/font&gt;&lt;/div&gt;&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;external plugins API&lt;/b&gt;&lt;/font&gt;&lt;div&gt;&lt;font style=&quot;font-size: 14px&quot;&gt;text format, over pipes&lt;/font&gt;&lt;/div&gt;" link="https://github.com/netdata/netdata/tree/master/src/plugins.d" id="32">
           <mxCell style="whiteSpace=wrap;html=1;fontSize=16;fillColor=#FFFFFF;strokeColor=#59DE9C;" parent="1" vertex="1">
             <mxGeometry x="74" y="887" width="580" height="130" as="geometry" />
           </mxCell>

docs/glossary.md

@@ -53,7 +53,7 @@ Use the alphabatized list below to find the answer to your single-term questions
 
 ## E
 
-- [**External Plugins**](/src/collectors/plugins.d/README.md): These gather metrics from external processes, such as a webserver or database, and run as independent processes that communicate with the Netdata daemon via pipes.
+- [**External Plugins**](/src/plugins.d/README.md): These gather metrics from external processes, such as a webserver or database, and run as independent processes that communicate with the Netdata daemon via pipes.
 
 ## F
 

docs/top-monitoring-netdata-functions.md

@@ -7,7 +7,7 @@ executed on the node/host where the function is made available.
 
 Collectors besides the metric collection, storing, and/or streaming work are capable of executing specific routines on request. These routines will bring additional information to help you troubleshoot or even trigger some action to happen on the node itself.
 
-For more details please check out documentation on how we use our internal collector to get this from the first collector that exposes functions - [plugins.d](/src/collectors/plugins.d/README.md#function).
+For more details please check out documentation on how we use our internal collector to get this from the first collector that exposes functions - [plugins.d](/src/plugins.d/README.md#function).
 
 ## Prerequisites
 

packaging/installer/methods/macos.md

@@ -9,7 +9,7 @@ learn_rel_path: "Installation/Install on specific environments"
 # Install Netdata on macOS
 
 Netdata works on macOS, albeit with some limitations. 
-The number of charts displaying system metrics is limited, but you can use any of Netdata's [external plugins](/src/collectors/plugins.d/README.md) to monitor any services you might have installed on your macOS system. 
+The number of charts displaying system metrics is limited, but you can use any of Netdata's [external plugins](/src/plugins.d/README.md) to monitor any services you might have installed on your macOS system. 
 You could also use a macOS system as the parent node in a [streaming configuration](/src/streaming/README.md).
 
 You can install Netdata in one of the three following ways: 

src/collectors/README.md

@@ -45,7 +45,7 @@ specifics of what a given collector does.
     -   **External** plugins organize collectors that gather metrics from external processes, such as a MySQL database or
         Nginx web server. They can be written in any language, and the `netdata` daemon spawns them as long-running
         independent processes. They communicate with the daemon via pipes. All external plugins are managed by
-        [plugins.d](/src/collectors/plugins.d/README.md), which provides additional management options.
+        [plugins.d](/src/plugins.d/README.md), which provides additional management options.
 
 -   **Orchestrators** are external plugins that run and manage one or more modules. They run as independent processes.
     The Go orchestrator is in active development.

src/collectors/charts.d.plugin/README.md

@@ -7,7 +7,7 @@
 3. It communicates with Netdata via a unidirectional pipe (sending data to the `netdata` daemon)
 4. Supports any number of data collection **modules**
 
-To better understand the guidelines and the API behind our External plugins, please have a look at the [Introduction to External plugins](/src/collectors/plugins.d/README.md) prior to reading this page.
+To better understand the guidelines and the API behind our External plugins, please have a look at the [Introduction to External plugins](/src/plugins.d/README.md) prior to reading this page.
 
 
 `charts.d.plugin` has been designed so that the actual script that will do data collection will be permanently in

src/collectors/python.d.plugin/go_expvar/integrations/go_applications_expvar.md

@@ -231,7 +231,7 @@ See [this issue](https://github.com/netdata/netdata/pull/1902#issuecomment-28449
 
 Please see these two links to the official Netdata documentation for more information about the values:
 
--   [External plugins - charts](/src/collectors/plugins.d/README.md#chart)
+-   [External plugins - charts](/src/plugins.d/README.md#chart)
 -   [Chart variables](/src/collectors/python.d.plugin/README.md#global-variables-order-and-chart)
 
 **Line definitions**
@@ -255,7 +255,7 @@ hidden: False
 ```
 
 Please see the following link for more information about the options and their default values:
-[External plugins - dimensions](/src/collectors/plugins.d/README.md#dimension)
+[External plugins - dimensions](/src/plugins.d/README.md#dimension)
 
 Apart from top-level expvars, this plugin can also parse expvars stored in a multi-level map;
 All dicts in the resulting JSON document are then flattened to one level.

src/collectors/python.d.plugin/go_expvar/metadata.yaml

@@ -200,7 +200,7 @@ modules:
                 
                 Please see these two links to the official Netdata documentation for more information about the values:
                 
-                -   [External plugins - charts](/src/collectors/plugins.d/README.md#chart)
+                -   [External plugins - charts](/src/plugins.d/README.md#chart)
                 -   [Chart variables](/src/collectors/python.d.plugin/README.md#global-variables-order-and-chart)
                 
                 **Line definitions**
@@ -224,7 +224,7 @@ modules:
                 ```
                 
                 Please see the following link for more information about the options and their default values:
-                [External plugins - dimensions](/src/collectors/plugins.d/README.md#dimension)
+                [External plugins - dimensions](/src/plugins.d/README.md#dimension)
                 
                 Apart from top-level expvars, this plugin can also parse expvars stored in a multi-level map;
                 All dicts in the resulting JSON document are then flattened to one level.

src/go/pkg/netdataapi/api.go

@@ -11,7 +11,7 @@ import (
 
 type (
 	// API implements Netdata external plugins API.
-	// https://learn.netdata.cloud/docs/agent/collectors/plugins.d#the-output-of-the-plugin
+	// https://learn.netdata.cloud/docs/agent/plugins.d#the-output-of-the-plugin
 	API struct {
 		io.Writer
 	}

src/go/plugin/go.d/agent/discovery/sd/discoverer/netlisteners/netlisteners.go

@@ -294,7 +294,7 @@ func (e *localListenersExec) discover(ctx context.Context) ([]byte, error) {
 	defer cancel()
 
 	// TCPv4/6 and UPDv4 sockets in LISTEN state
-	// https://github.com/netdata/netdata/blob/master/src/collectors/plugins.d/local_listeners.c
+	// https://github.com/netdata/netdata/blob/master/src/collectors/utils/local_listeners.c
 	args := []string{
 		"no-udp6",
 		"no-local",

src/go/plugin/go.d/agent/module/charts.go

@@ -78,7 +78,7 @@ type (
 	}
 
 	// Chart represents a chart.
-	// For the full description please visit https://docs.netdata.cloud/collectors/plugins.d/#chart
+	// For the full description please visit https://docs.netdata.cloud/plugins.d/#chart
 	Chart struct {
 		// typeID is the unique identification of the chart, if not specified,
 		// the orchestrator will use job full name + chart ID as typeID (default behaviour).
@@ -128,7 +128,7 @@ type (
 	}
 
 	// Dim represents a chart dimension.
-	// For detailed description please visit https://docs.netdata.cloud/collectors/plugins.d/#dimension.
+	// For detailed description please visit https://docs.netdata.cloud/plugins.d/#dimension.
 	Dim struct {
 		ID   string
 		Name string
@@ -141,7 +141,7 @@ type (
 	}
 
 	// Var represents a chart variable.
-	// For detailed description please visit https://docs.netdata.cloud/collectors/plugins.d/#variable
+	// For detailed description please visit https://docs.netdata.cloud/plugins.d/#variable
 	Var struct {
 		ID    string
 		Name  string

src/go/plugin/go.d/docs/how-to-write-a-module.md

@@ -48,7 +48,7 @@ The steps are:
   developed collector. It will be placed into the `bin` directory (e.g `go.d.plugin/bin`)
 - Run it in the debug mode `bin/godplugin -d -m <MODULE_NAME>`. This will output the `STDOUT` of the collector, the same
   output that is sent to the Netdata Agent and is transformed into charts. You can read more about this collector API in
-  our [documentation](/src/collectors/plugins.d/README.md#external-plugins-api).
+  our [documentation](/src/plugins.d/README.md#external-plugins-api).
 - If you want to test the collector with the actual Netdata Agent, you need to replace the `go.d.plugin` binary that
   exists in the Netdata Agent installation directory with the one you just compiled. Once
   you restart the Netdata Agent, it will detect and run it, creating all the charts. It is advised not to remove the default `go.d.plugin` binary, but simply rename it to `go.d.plugin.old` so that the Agent doesn't run it, but you can easily rename it back once you are done.
@@ -119,7 +119,7 @@ func (e *Example) Check() bool {
 ### Charts method
 
 :exclamation: Netdata module
-produces [`charts`](/src/collectors/plugins.d/README.md#chart), not
+produces [`charts`](/src/plugins.d/README.md#chart), not
 raw metrics.
 
 Use [`agent/module`](https://github.com/netdata/netdata/blob/master/src/go/plugin/go.d/agent/module/charts.go)
@@ -127,7 +127,7 @@ package to create them,
 it contains charts and dimensions structs.
 
 - `Charts` returns
-  the [charts](/src/collectors/plugins.d/README.md#chart) (`*module.Charts`).
+  the [charts](/src/plugins.d/README.md#chart) (`*module.Charts`).
 - Called after `Check` and only if `Check` returned `true`.
 - If it returns `nil`, the job will be disabled
 - :warning: Make sure not to share returned value between module instances (jobs).

src/plugins.d/README.md

@@ -1,6 +1,6 @@
 <!--
 title: "External plugins"
-custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/collectors/plugins.d/README.md"
+custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/plugins.d/README.md"
 sidebar_label: "External plugins"
 learn_status: "Published"
 learn_topic_type: "References"

src/streaming/protocol/command-claimed_id.c

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "commands.h"
-#include "collectors/plugins.d/pluginsd_internals.h"
+#include "plugins.d/pluginsd_internals.h"
 
 PARSER_RC rrdpush_receiver_pluginsd_claimed_id(char **words, size_t num_words, PARSER *parser) {
     const char *machine_guid_str = get_word(words, num_words, 1);

src/streaming/protocol/command-nodeid.c

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "commands.h"
-#include "collectors/plugins.d/pluginsd_internals.h"
+#include "plugins.d/pluginsd_internals.h"
 
 // the child disconnected from the parent, and it has to clear the parent's claim id
 void rrdpush_sender_clear_parent_claim_id(RRDHOST *host) {

src/streaming/receiver.c

@@ -34,7 +34,7 @@ void receiver_state_free(struct receiver_state *rpt) {
     freez(rpt);
 }
 
-#include "collectors/plugins.d/pluginsd_parser.h"
+#include "plugins.d/pluginsd_parser.h"
 
 // IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly
 #define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1)

src/streaming/stream_path.c

@@ -2,7 +2,7 @@
 
 #include "stream_path.h"
 #include "rrdpush.h"
-#include "collectors/plugins.d/pluginsd_internals.h"
+#include "plugins.d/pluginsd_internals.h"
 
 ENUM_STR_MAP_DEFINE(STREAM_PATH_FLAGS) = {
     { .id = STREAM_PATH_FLAG_ACLK, .name = "aclk" },