src/gpu_nvml.c

   1 #include "daemon/collectd.h"
   2 #include "daemon/common.h"
   3 #include "daemon/plugin.h"
   4
   5 #include <nvml.h>
   6 #include <stdint.h>
   7 #include <stdio.h>
   8
   9 #define MAX_DEVNAME_LEN 256
  10 #define PLUGIN_NAME "gpu_nvml"
  11
  12 static nvmlReturn_t nv_status = NVML_SUCCESS;
  13 static char *nv_errline = "";
  14
  15 #define TRY_CATCH(f, catch)                                                    \
  16   if ((nv_status = f) != NVML_SUCCESS) {                                       \
  17     nv_errline = #f;                                                           \
  18     goto catch;                                                                \
  19   }
  20
  21 #define TRY_CATCH_OPTIONAL(f, catch)                                           \
  22   if ((nv_status = f) != NVML_SUCCESS &&                                       \
  23       nv_status != NVML_ERROR_NOT_SUPPORTED) {                                 \
  24     nv_errline = #f;                                                           \
  25     goto catch;                                                                \
  26   }
  27
  28 #define TRY(f) TRY_CATCH(f, catch)
  29 #define TRYOPT(f) TRY_CATCH_OPTIONAL(f, catch)
  30
  31 #define WRAPGAUGE(x) ((value_t){.gauge = (gauge_t)(x)})
  32
  33 static const char *config_keys[] = {
  34     "GPUIndex",
  35     "IgnoreSelected",
  36 };
  37 static const unsigned int n_config_keys = STATIC_ARRAY_SIZE(config_keys);
  38
  39 // This is a bitflag, necessitating the (extremely conservative) assumption
  40 // that there are no more than 64 GPUs on this system.
  41 static uint64_t conf_match_mask = 0;
  42 static bool conf_mask_is_exclude = 0;
  43
  44 static int nvml_config(const char *key, const char *value) {
  45
  46   unsigned long device_ix;
  47   char *eptr;
  48
  49   if (strcasecmp(key, config_keys[0]) == 0) {
  50     device_ix = strtoul(value, &eptr, 10);
  51     if (eptr == value) {
  52       return -1;
  53     }
  54     if (device_ix > 64) {
  55       return -2;
  56     }
  57     conf_match_mask |= (1 << device_ix);
  58   } else if (strcasecmp(key, config_keys[1])) {
  59     if
  60       IS_TRUE(value) { conf_mask_is_exclude = 1; }
  61   } else {
  62     return -10;
  63   }
  64
  65   return 0;
  66 }
  67
  68 static int nvml_init(void) {
  69   TRY(nvmlInit());
  70   return 0;
  71
  72   catch : ERROR("NVML init failed with %d", nv_status);
  73   return -1;
  74 }
  75
  76 static int nvml_shutdown(void) {
  77   TRY(nvmlShutdown())
  78   return 0;
  79
  80   catch : ERROR("NVML shutdown failed with %d", nv_status);
  81   return -1;
  82 }
  83
  84 static void nvml_submit(const char *plugin_instance, const char *type,
  85                         const char *type_instance, value_t nvml) {
  86
  87   value_list_t vl = VALUE_LIST_INIT;
  88
  89   vl.values = &nvml;
  90   vl.values_len = 1;
  91
  92   sstrncpy(vl.plugin, PLUGIN_NAME, sizeof(vl.plugin));
  93   sstrncpy(vl.plugin_instance, plugin_instance, sizeof(vl.plugin_instance));
  94
  95   sstrncpy(vl.type, type, sizeof(vl.type));
  96
  97   if (type_instance != NULL) {
  98     sstrncpy(vl.type_instance, type_instance, sizeof(vl.type_instance));
  99   }
 100
 101   plugin_dispatch_values(&vl);
 102 }
 103
 104 static int nvml_read(void) {
 105
 106   unsigned int device_count;
 107   TRY_CATCH(nvmlDeviceGetCount(&device_count), catch_nocount);
 108
 109   if (device_count > 64) {
 110     device_count = 64;
 111   }
 112
 113   for (int ix = 0; ix < device_count; ix++) {
 114
 115     int is_match = ((1 << ix) & conf_match_mask) || (conf_match_mask == 0);
 116     if (conf_mask_is_exclude == !!is_match) {
 117       continue;
 118     }
 119
 120     nvmlDevice_t dev;
 121     TRY(nvmlDeviceGetHandleByIndex(ix, &dev));
 122
 123     char dev_name[MAX_DEVNAME_LEN + 1];
 124     dev_name[0] = '\0';
 125     TRY(nvmlDeviceGetName(dev, dev_name, MAX_DEVNAME_LEN));
 126
 127     // Try to be as lenient as possible with the variety of devices that are
 128     // out there, ignoring any NOT_SUPPORTED errors gently.
 129     nvmlMemory_t meminfo;
 130     TRYOPT(nvmlDeviceGetMemoryInfo(dev, &meminfo))
 131     if (nv_status == NVML_SUCCESS) {
 132       double pct_mem_used = 100. * (double)meminfo.used / meminfo.total;
 133       nvml_submit(dev_name, "percent", "mem_used", WRAPGAUGE(pct_mem_used));
 134     }
 135
 136     nvmlUtilization_t utilization;
 137     TRYOPT(nvmlDeviceGetUtilizationRates(dev, &utilization))
 138     if (nv_status == NVML_SUCCESS)
 139       nvml_submit(dev_name, "percent", "gpu_used", WRAPGAUGE(utilization.gpu));
 140
 141     unsigned int fan_speed;
 142     TRYOPT(nvmlDeviceGetFanSpeed(dev, &fan_speed))
 143     if (nv_status == NVML_SUCCESS)
 144       nvml_submit(dev_name, "fanspeed", NULL, WRAPGAUGE(fan_speed));
 145
 146     unsigned int core_temp;
 147     TRYOPT(nvmlDeviceGetTemperature(dev, NVML_TEMPERATURE_GPU, &core_temp))
 148     if (nv_status == NVML_SUCCESS)
 149       nvml_submit(dev_name, "temperature", "core", WRAPGAUGE(core_temp));
 150
 151     unsigned int sm_clk_mhz;
 152     TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_SM, &sm_clk_mhz))
 153     if (nv_status == NVML_SUCCESS)
 154       nvml_submit(dev_name, "frequency", "sm", WRAPGAUGE(1e6 * sm_clk_mhz));
 155
 156     unsigned int mem_clk_mhz;
 157     TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_MEM, &mem_clk_mhz))
 158     if (nv_status == NVML_SUCCESS)
 159       nvml_submit(dev_name, "frequency", "mem", WRAPGAUGE(1e6 * mem_clk_mhz));
 160
 161     unsigned int power_mW;
 162     TRYOPT(nvmlDeviceGetPowerUsage(dev, &power_mW))
 163     if (nv_status == NVML_SUCCESS)
 164       nvml_submit(dev_name, "power", NULL, WRAPGAUGE(1e-3 * power_mW));
 165
 166     continue;
 167
 168     // Failures here indicate transient errors or removal of GPU. In either
 169     // case it will either be resolved or the GPU will no longer be enumerated
 170     // the next time round.
 171     catch : WARNING("NVML call \"%s\" failed with code %d on dev at index %d!",
 172                     nv_errline, nv_status, ix);
 173     continue;
 174   }
 175
 176   return 0;
 177
 178   // Failures here indicate serious misconfiguration; we bail out totally.
 179 catch_nocount:
 180   ERROR("Failed to enumerate NVIDIA GPUs (\"%s\" returned %d)", nv_errline,
 181         nv_status);
 182   return -1;
 183 }
 184
 185 void module_register(void) {
 186   plugin_register_init(PLUGIN_NAME, nvml_init);
 187   plugin_register_config(PLUGIN_NAME, nvml_config, config_keys, n_config_keys);
 188   plugin_register_read(PLUGIN_NAME, nvml_read);
 189   plugin_register_shutdown(PLUGIN_NAME, nvml_shutdown);
 190 }