1 #include "daemon/collectd.h"
2 #include "daemon/common.h"
3 #include "daemon/plugin.h"
9 #define MAX_DEVNAME_LEN 256
10 #define PLUGIN_NAME "gpu_nvml"
12 static nvmlReturn_t nv_status = NVML_SUCCESS;
13 static char *nv_errline = "";
15 #define TRY_CATCH(f, catch) \
16 if ((nv_status = f) != NVML_SUCCESS) { \
21 #define TRY_CATCH_OPTIONAL(f, catch) \
22 if ((nv_status = f) != NVML_SUCCESS && \
23 nv_status != NVML_ERROR_NOT_SUPPORTED) { \
28 #define TRY(f) TRY_CATCH(f, catch)
29 #define TRYOPT(f) TRY_CATCH_OPTIONAL(f, catch)
31 #define WRAPGAUGE(x) ((value_t){.gauge = (gauge_t)(x)})
33 static const char *config_keys[] = {
37 static const unsigned int n_config_keys = STATIC_ARRAY_SIZE(config_keys);
39 // This is a bitflag, necessitating the (extremely conservative) assumption
40 // that there are no more than 64 GPUs on this system.
41 static uint64_t conf_match_mask = 0;
42 static bool conf_mask_is_exclude = 0;
44 static int nvml_config(const char *key, const char *value) {
46 unsigned long device_ix;
49 if (strcasecmp(key, config_keys[0]) == 0) {
50 device_ix = strtoul(value, &eptr, 10);
57 conf_match_mask |= (1 << device_ix);
58 } else if (strcasecmp(key, config_keys[1])) {
60 IS_TRUE(value) { conf_mask_is_exclude = 1; }
68 static int nvml_init(void) {
72 catch : ERROR("NVML init failed with %d", nv_status);
76 static int nvml_shutdown(void) {
80 catch : ERROR("NVML shutdown failed with %d", nv_status);
84 static void nvml_submit(const char *plugin_instance, const char *type,
85 const char *type_instance, value_t nvml) {
87 value_list_t vl = VALUE_LIST_INIT;
92 sstrncpy(vl.plugin, PLUGIN_NAME, sizeof(vl.plugin));
93 sstrncpy(vl.plugin_instance, plugin_instance, sizeof(vl.plugin_instance));
95 sstrncpy(vl.type, type, sizeof(vl.type));
97 if (type_instance != NULL) {
98 sstrncpy(vl.type_instance, type_instance, sizeof(vl.type_instance));
101 plugin_dispatch_values(&vl);
104 static int nvml_read(void) {
106 unsigned int device_count;
107 TRY_CATCH(nvmlDeviceGetCount(&device_count), catch_nocount);
109 if (device_count > 64) {
113 for (int ix = 0; ix < device_count; ix++) {
115 int is_match = ((1 << ix) & conf_match_mask) || (conf_match_mask == 0);
116 if (conf_mask_is_exclude == !!is_match) {
121 TRY(nvmlDeviceGetHandleByIndex(ix, &dev));
123 char dev_name[MAX_DEVNAME_LEN + 1];
125 TRY(nvmlDeviceGetName(dev, dev_name, MAX_DEVNAME_LEN));
127 // Try to be as lenient as possible with the variety of devices that are
128 // out there, ignoring any NOT_SUPPORTED errors gently.
129 nvmlMemory_t meminfo;
130 TRYOPT(nvmlDeviceGetMemoryInfo(dev, &meminfo))
131 if (nv_status == NVML_SUCCESS) {
132 double pct_mem_used = 100. * (double)meminfo.used / meminfo.total;
133 nvml_submit(dev_name, "percent", "mem_used", WRAPGAUGE(pct_mem_used));
136 nvmlUtilization_t utilization;
137 TRYOPT(nvmlDeviceGetUtilizationRates(dev, &utilization))
138 if (nv_status == NVML_SUCCESS)
139 nvml_submit(dev_name, "percent", "gpu_used", WRAPGAUGE(utilization.gpu));
141 unsigned int fan_speed;
142 TRYOPT(nvmlDeviceGetFanSpeed(dev, &fan_speed))
143 if (nv_status == NVML_SUCCESS)
144 nvml_submit(dev_name, "fanspeed", NULL, WRAPGAUGE(fan_speed));
146 unsigned int core_temp;
147 TRYOPT(nvmlDeviceGetTemperature(dev, NVML_TEMPERATURE_GPU, &core_temp))
148 if (nv_status == NVML_SUCCESS)
149 nvml_submit(dev_name, "temperature", "core", WRAPGAUGE(core_temp));
151 unsigned int sm_clk_mhz;
152 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_SM, &sm_clk_mhz))
153 if (nv_status == NVML_SUCCESS)
154 nvml_submit(dev_name, "frequency", "sm", WRAPGAUGE(1e6 * sm_clk_mhz));
156 unsigned int mem_clk_mhz;
157 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_MEM, &mem_clk_mhz))
158 if (nv_status == NVML_SUCCESS)
159 nvml_submit(dev_name, "frequency", "mem", WRAPGAUGE(1e6 * mem_clk_mhz));
161 unsigned int power_mW;
162 TRYOPT(nvmlDeviceGetPowerUsage(dev, &power_mW))
163 if (nv_status == NVML_SUCCESS)
164 nvml_submit(dev_name, "power", NULL, WRAPGAUGE(1e-3 * power_mW));
168 // Failures here indicate transient errors or removal of GPU. In either
169 // case it will either be resolved or the GPU will no longer be enumerated
170 // the next time round.
171 catch : WARNING("NVML call \"%s\" failed with code %d on dev at index %d!",
172 nv_errline, nv_status, ix);
178 // Failures here indicate serious misconfiguration; we bail out totally.
180 ERROR("Failed to enumerate NVIDIA GPUs (\"%s\" returned %d)", nv_errline,
185 void module_register(void) {
186 plugin_register_init(PLUGIN_NAME, nvml_init);
187 plugin_register_config(PLUGIN_NAME, nvml_config, config_keys, n_config_keys);
188 plugin_register_read(PLUGIN_NAME, nvml_read);
189 plugin_register_shutdown(PLUGIN_NAME, nvml_shutdown);