1 #include "daemon/collectd.h"
2 #include "daemon/common.h"
3 #include "daemon/plugin.h"
9 #define MAX_DEVNAME_LEN 256
10 #define PLUGIN_NAME "gpu_nvml"
12 static nvmlReturn_t nv_status = NVML_SUCCESS;
13 static char *nv_errline = "";
15 #define TRY_CATCH(f, catch) \
16 if ((nv_status = f) != NVML_SUCCESS) { \
21 #define TRY_CATCH_OPTIONAL(f, catch) \
22 if ((nv_status = f) != NVML_SUCCESS && \
23 nv_status != NVML_ERROR_NOT_SUPPORTED) { \
28 #define TRY(f) TRY_CATCH(f, catch)
29 #define TRYOPT(f) TRY_CATCH_OPTIONAL(f, catch)
31 #define WRAPGAUGE(x) ((value_t){.gauge = (gauge_t)(x)})
33 static const char *config_keys[] = {
34 "GPUIndex", "IgnoreSelected",
36 static const unsigned int n_config_keys = STATIC_ARRAY_SIZE(config_keys);
38 // This is a bitflag, necessitating the (extremely conservative) assumption
39 // that there are no more than 64 GPUs on this system.
40 static uint64_t conf_match_mask = 0;
41 static bool conf_mask_is_exclude = 0;
43 static int nvml_config(const char *key, const char *value) {
45 unsigned long device_ix;
48 if (strcasecmp(key, config_keys[0]) == 0) {
49 device_ix = strtoul(value, &eptr, 10);
51 ERROR("Failed to parse GPUIndex value \"%s\"", value);
54 if (device_ix >= 64) {
55 ERROR("At most 64 GPUs (0 <= GPUIndex < 64) are supported!");
58 conf_match_mask |= (1 << device_ix);
59 } else if (strcasecmp(key, config_keys[1])) {
61 IS_TRUE(value) { conf_mask_is_exclude = 1; }
63 ERROR("Unrecognized config option %s", key);
70 static int nvml_init(void) {
74 catch : ERROR("NVML init failed with %d", nv_status);
78 static int nvml_shutdown(void) {
82 catch : ERROR("NVML shutdown failed with %d", nv_status);
86 static void nvml_submit(const char *plugin_instance, const char *type,
87 const char *type_instance, value_t nvml) {
89 value_list_t vl = VALUE_LIST_INIT;
94 sstrncpy(vl.plugin, PLUGIN_NAME, sizeof(vl.plugin));
95 sstrncpy(vl.plugin_instance, plugin_instance, sizeof(vl.plugin_instance));
97 sstrncpy(vl.type, type, sizeof(vl.type));
99 if (type_instance != NULL) {
100 sstrncpy(vl.type_instance, type_instance, sizeof(vl.type_instance));
103 plugin_dispatch_values(&vl);
106 static int nvml_read(void) {
108 unsigned int device_count;
109 TRY_CATCH(nvmlDeviceGetCount(&device_count), catch_nocount);
111 if (device_count > 64) {
115 for (int ix = 0; ix < device_count; ix++) {
117 int is_match = ((1 << ix) & conf_match_mask) || (conf_match_mask == 0);
118 if (conf_mask_is_exclude == !!is_match) {
123 TRY(nvmlDeviceGetHandleByIndex(ix, &dev));
125 char dev_name[MAX_DEVNAME_LEN + 1];
127 TRY(nvmlDeviceGetName(dev, dev_name, MAX_DEVNAME_LEN));
129 // Try to be as lenient as possible with the variety of devices that are
130 // out there, ignoring any NOT_SUPPORTED errors gently.
131 nvmlMemory_t meminfo;
132 TRYOPT(nvmlDeviceGetMemoryInfo(dev, &meminfo))
133 if (nv_status == NVML_SUCCESS) {
134 double pct_mem_used = 100. * (double)meminfo.used / meminfo.total;
135 nvml_submit(dev_name, "percent", "mem_used", WRAPGAUGE(pct_mem_used));
138 nvmlUtilization_t utilization;
139 TRYOPT(nvmlDeviceGetUtilizationRates(dev, &utilization))
140 if (nv_status == NVML_SUCCESS)
141 nvml_submit(dev_name, "percent", "gpu_used", WRAPGAUGE(utilization.gpu));
143 unsigned int fan_speed;
144 TRYOPT(nvmlDeviceGetFanSpeed(dev, &fan_speed))
145 if (nv_status == NVML_SUCCESS)
146 nvml_submit(dev_name, "fanspeed", NULL, WRAPGAUGE(fan_speed));
148 unsigned int core_temp;
149 TRYOPT(nvmlDeviceGetTemperature(dev, NVML_TEMPERATURE_GPU, &core_temp))
150 if (nv_status == NVML_SUCCESS)
151 nvml_submit(dev_name, "temperature", "core", WRAPGAUGE(core_temp));
153 unsigned int sm_clk_mhz;
154 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_SM, &sm_clk_mhz))
155 if (nv_status == NVML_SUCCESS)
156 nvml_submit(dev_name, "frequency", "sm", WRAPGAUGE(1e6 * sm_clk_mhz));
158 unsigned int mem_clk_mhz;
159 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_MEM, &mem_clk_mhz))
160 if (nv_status == NVML_SUCCESS)
161 nvml_submit(dev_name, "frequency", "mem", WRAPGAUGE(1e6 * mem_clk_mhz));
163 unsigned int power_mW;
164 TRYOPT(nvmlDeviceGetPowerUsage(dev, &power_mW))
165 if (nv_status == NVML_SUCCESS)
166 nvml_submit(dev_name, "power", NULL, WRAPGAUGE(1e-3 * power_mW));
170 // Failures here indicate transient errors or removal of GPU. In either
171 // case it will either be resolved or the GPU will no longer be enumerated
172 // the next time round.
173 catch : WARNING("NVML call \"%s\" failed with code %d on dev at index %d!",
174 nv_errline, nv_status, ix);
180 // Failures here indicate serious misconfiguration; we bail out totally.
182 ERROR("Failed to enumerate NVIDIA GPUs (\"%s\" returned %d)", nv_errline,
187 void module_register(void) {
188 plugin_register_init(PLUGIN_NAME, nvml_init);
189 plugin_register_config(PLUGIN_NAME, nvml_config, config_keys, n_config_keys);
190 plugin_register_read(PLUGIN_NAME, nvml_read);
191 plugin_register_shutdown(PLUGIN_NAME, nvml_shutdown);