2 * collectd - src/utils_format_stackdriver.c
5 * Copyright (C) 2017 Florian Forster
7 * Permission to use, copy, modify, and/or distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
20 * Florian Forster <octo at collectd.org>
25 #include "utils_format_stackdriver.h"
29 #include "utils_avltree.h"
30 #include "utils_cache.h"
31 #include "utils_time.h"
33 #include <yajl/yajl_gen.h>
34 #include <yajl/yajl_parse.h>
35 #if HAVE_YAJL_YAJL_VERSION_H
36 #include <yajl/yajl_version.h>
43 c_avl_tree_t *metric_descriptors;
50 typedef struct sd_label_s sd_label_t;
52 struct sd_resource_s {
59 static int json_string(yajl_gen gen, char const *s) /* {{{ */
61 yajl_gen_status status =
62 yajl_gen_string(gen, (unsigned char const *)s, strlen(s));
63 if (status != yajl_gen_status_ok)
67 } /* }}} int json_string */
69 static int json_time(yajl_gen gen, cdtime_t t) {
72 size_t status = rfc3339(buffer, sizeof(buffer), t);
77 return json_string(gen, buffer);
78 } /* }}} int json_time */
83 * "type": "library.googleapis.com/book",
85 * "/genre": "fiction",
87 * "/title": "The Old Man and the Sea"
91 static int format_gcm_resource(yajl_gen gen, sd_resource_t *res) /* {{{ */
93 yajl_gen_map_open(gen);
95 int status = json_string(gen, "type") || json_string(gen, res->type);
99 if (res->labels_num != 0) {
100 status = json_string(gen, "labels");
104 yajl_gen_map_open(gen);
105 for (size_t i = 0; i < res->labels_num; i++) {
106 status = json_string(gen, res->labels[i].key) ||
107 json_string(gen, res->labels[i].value);
111 yajl_gen_map_close(gen);
114 yajl_gen_map_close(gen);
116 } /* }}} int format_gcm_resource */
121 * // Union field, only one of the following:
122 * "int64Value": string,
123 * "doubleValue": number,
126 static int format_typed_value(yajl_gen gen, int ds_type, value_t v,
127 int64_t start_value) {
130 yajl_gen_map_open(gen);
133 case DS_TYPE_GAUGE: {
134 int status = json_string(gen, "doubleValue");
138 status = (int)yajl_gen_double(gen, (double)v.gauge);
139 if (status != yajl_gen_status_ok)
142 yajl_gen_map_close(gen);
145 case DS_TYPE_DERIVE: {
146 derive_t diff = v.derive - (derive_t)start_value;
147 snprintf(integer, sizeof(integer), "%" PRIi64, diff);
150 case DS_TYPE_COUNTER: {
151 counter_t diff = counter_diff((counter_t)start_value, v.counter);
152 snprintf(integer, sizeof(integer), "%llu", diff);
155 case DS_TYPE_ABSOLUTE: {
156 snprintf(integer, sizeof(integer), "%" PRIu64, v.absolute);
160 ERROR("format_typed_value: unknown value type %d.", ds_type);
165 int status = json_string(gen, "int64Value") || json_string(gen, integer);
170 yajl_gen_map_close(gen);
172 } /* }}} int format_typed_value */
181 static int format_metric_kind(yajl_gen gen, int ds_type) {
184 case DS_TYPE_ABSOLUTE:
185 return json_string(gen, "GAUGE");
186 case DS_TYPE_COUNTER:
188 return json_string(gen, "CUMULATIVE");
190 ERROR("format_metric_kind: unknown value type %d.", ds_type);
202 static int format_value_type(yajl_gen gen, int ds_type) {
203 return json_string(gen, (ds_type == DS_TYPE_GAUGE) ? "DOUBLE" : "INT64");
206 static int metric_type(char *buffer, size_t buffer_size, data_set_t const *ds,
207 value_list_t const *vl, int ds_index) {
209 char const *ds_name = ds->ds[ds_index].name;
211 #define GCM_PREFIX "custom.googleapis.com/collectd/"
212 if ((ds_index != 0) || strcmp("value", ds_name) != 0) {
213 snprintf(buffer, buffer_size, GCM_PREFIX "%s/%s_%s", vl->plugin, vl->type,
216 snprintf(buffer, buffer_size, GCM_PREFIX "%s/%s", vl->plugin, vl->type);
219 char const *whitelist = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
220 "abcdefghijklmnopqrstuvwxyz"
222 char *ptr = buffer + strlen(GCM_PREFIX);
224 while ((ok_len = strspn(ptr, whitelist)) != strlen(ptr)) {
230 } /* }}} int metric_type */
232 /* The metric type, including its DNS name prefix. The type is not URL-encoded.
233 * All user-defined custom metric types have the DNS name custom.googleapis.com.
234 * Metric types should use a natural hierarchical grouping. */
235 static int format_metric_type(yajl_gen gen, data_set_t const *ds,
236 value_list_t const *vl, int ds_index) {
238 char buffer[4 * DATA_MAX_NAME_LEN];
239 metric_type(buffer, sizeof(buffer), ds, vl, ds_index);
241 return json_string(gen, buffer);
242 } /* }}} int format_metric_type */
248 * "startTime": string,
251 static int format_time_interval(yajl_gen gen, int ds_type,
252 value_list_t const *vl, cdtime_t start_time) {
254 yajl_gen_map_open(gen);
256 int status = json_string(gen, "endTime") || json_time(gen, vl->time);
260 if ((ds_type == DS_TYPE_DERIVE) || (ds_type == DS_TYPE_COUNTER)) {
261 int status = json_string(gen, "startTime") || json_time(gen, start_time);
266 yajl_gen_map_close(gen);
268 } /* }}} int format_time_interval */
270 /* read_cumulative_state reads the start time and start value of cumulative
271 * (i.e. DERIVE or COUNTER) metrics from the cache. If a metric is seen for the
272 * first time, or when a DERIVE metric is reset, the start time is (re)set to
274 static int read_cumulative_state(data_set_t const *ds, value_list_t const *vl,
275 int ds_index, cdtime_t *ret_start_time,
276 int64_t *ret_start_value) {
277 int ds_type = ds->ds[ds_index].type;
278 if ((ds_type != DS_TYPE_DERIVE) && (ds_type != DS_TYPE_COUNTER)) {
282 char start_value_key[DATA_MAX_NAME_LEN];
283 snprintf(start_value_key, sizeof(start_value_key),
284 "stackdriver:start_value[%d]", ds_index);
287 uc_meta_data_get_signed_int(vl, start_value_key, ret_start_value);
288 if ((status == 0) && ((ds_type != DS_TYPE_DERIVE) ||
289 (*ret_start_value <= vl->values[ds_index].derive))) {
290 return uc_meta_data_get_unsigned_int(vl, "stackdriver:start_time",
294 if (ds_type == DS_TYPE_DERIVE) {
295 *ret_start_value = vl->values[ds_index].derive;
297 *ret_start_value = (int64_t)vl->values[ds_index].counter;
299 *ret_start_time = vl->time;
301 status = uc_meta_data_add_signed_int(vl, start_value_key, *ret_start_value);
305 return uc_meta_data_add_unsigned_int(vl, "stackdriver:start_time",
307 } /* int read_cumulative_state */
313 * object(TimeInterval)
320 static int format_point(yajl_gen gen, data_set_t const *ds,
321 value_list_t const *vl, int ds_index,
322 cdtime_t start_time, int64_t start_value) {
324 yajl_gen_map_open(gen);
326 int ds_type = ds->ds[ds_index].type;
329 json_string(gen, "interval") ||
330 format_time_interval(gen, ds_type, vl, start_time) ||
331 json_string(gen, "value") ||
332 format_typed_value(gen, ds_type, vl->values[ds_index], start_value);
336 yajl_gen_map_close(gen);
338 } /* }}} int format_point */
350 static int format_metric(yajl_gen gen, data_set_t const *ds,
351 value_list_t const *vl, int ds_index) {
353 yajl_gen_map_open(gen);
355 int status = json_string(gen, "type") ||
356 format_metric_type(gen, ds, vl, ds_index) ||
357 json_string(gen, "labels");
362 yajl_gen_map_open(gen);
363 status = json_string(gen, "host") || json_string(gen, vl->host) ||
364 json_string(gen, "plugin_instance") ||
365 json_string(gen, vl->plugin_instance) ||
366 json_string(gen, "type_instance") ||
367 json_string(gen, vl->type_instance);
371 yajl_gen_map_close(gen);
373 yajl_gen_map_close(gen);
375 } /* }}} int format_metric */
384 * object(MonitoredResource)
386 * "metricKind": enum(MetricKind),
387 * "valueType": enum(ValueType),
395 /* format_time_series formats a TimeSeries object. Returns EAGAIN when a
396 * cumulative metric is seen for the first time and cannot be sent to
397 * Stackdriver due to lack of state. */
398 static int format_time_series(yajl_gen gen, data_set_t const *ds,
399 value_list_t const *vl, int ds_index,
400 sd_resource_t *res) {
401 int ds_type = ds->ds[ds_index].type;
403 cdtime_t start_time = 0;
404 int64_t start_value = 0;
406 read_cumulative_state(ds, vl, ds_index, &start_time, &start_value);
410 if (start_time == vl->time) {
411 /* for cumulative metrics, the interval must not be zero. */
415 yajl_gen_map_open(gen);
417 status = json_string(gen, "metric") || format_metric(gen, ds, vl, ds_index) ||
418 json_string(gen, "resource") || format_gcm_resource(gen, res) ||
419 json_string(gen, "metricKind") || format_metric_kind(gen, ds_type) ||
420 json_string(gen, "valueType") || format_value_type(gen, ds_type) ||
421 json_string(gen, "points");
425 yajl_gen_array_open(gen);
427 status = format_point(gen, ds, vl, ds_index, start_time, start_value);
431 yajl_gen_array_close(gen);
432 yajl_gen_map_close(gen);
434 } /* }}} int format_time_series */
446 static int sd_output_initialize(sd_output_t *out) /* {{{ */
448 yajl_gen_map_open(out->gen);
450 int status = json_string(out->gen, "timeSeries");
455 yajl_gen_array_open(out->gen);
457 } /* }}} int sd_output_initialize */
459 static int sd_output_finalize(sd_output_t *out) /* {{{ */
461 yajl_gen_array_close(out->gen);
462 yajl_gen_map_close(out->gen);
465 } /* }}} int sd_output_finalize */
467 static void sd_output_reset_staged(sd_output_t *out) /* {{{ */
471 while (c_avl_pick(out->staged, &key, &(void *){NULL}) == 0)
473 } /* }}} void sd_output_reset_staged */
475 sd_output_t *sd_output_create(sd_resource_t *res) /* {{{ */
477 sd_output_t *out = calloc(1, sizeof(*out));
483 out->gen = yajl_gen_alloc(/* funcs = */ NULL);
484 if (out->gen == NULL) {
485 sd_output_destroy(out);
489 out->staged = c_avl_create((void *)strcmp);
490 if (out->staged == NULL) {
491 sd_output_destroy(out);
495 out->metric_descriptors = c_avl_create((void *)strcmp);
496 if (out->metric_descriptors == NULL) {
497 sd_output_destroy(out);
501 sd_output_initialize(out);
504 } /* }}} sd_output_t *sd_output_create */
506 void sd_output_destroy(sd_output_t *out) /* {{{ */
511 if (out->metric_descriptors != NULL) {
513 while (c_avl_pick(out->metric_descriptors, &key, &(void *){NULL}) == 0) {
516 c_avl_destroy(out->metric_descriptors);
517 out->metric_descriptors = NULL;
520 if (out->staged != NULL) {
521 sd_output_reset_staged(out);
522 c_avl_destroy(out->staged);
526 if (out->gen != NULL) {
527 yajl_gen_free(out->gen);
531 if (out->res != NULL) {
532 sd_resource_destroy(out->res);
537 } /* }}} void sd_output_destroy */
539 int sd_output_add(sd_output_t *out, data_set_t const *ds,
540 value_list_t const *vl) /* {{{ */
542 /* first, check that we have all appropriate metric descriptors. */
543 for (size_t i = 0; i < ds->ds_num; i++) {
544 char buffer[4 * DATA_MAX_NAME_LEN];
545 metric_type(buffer, sizeof(buffer), ds, vl, i);
547 if (c_avl_get(out->metric_descriptors, buffer, NULL) != 0) {
552 char key[6 * DATA_MAX_NAME_LEN];
553 int status = FORMAT_VL(key, sizeof(key), vl);
555 ERROR("sd_output_add: FORMAT_VL failed with status %d.", status);
559 if (c_avl_get(out->staged, key, NULL) == 0) {
564 for (size_t i = 0; i < ds->ds_num; i++) {
565 int status = format_time_series(out->gen, ds, vl, i, out->res);
566 if (status == EAGAIN) {
567 /* first instance of a cumulative metric */
571 ERROR("sd_output_add: format_time_series failed with status %d.", status);
578 c_avl_insert(out->staged, strdup(key), NULL);
581 size_t json_buffer_size = 0;
582 yajl_gen_get_buf(out->gen, &(unsigned char const *){NULL}, &json_buffer_size);
583 if (json_buffer_size > 65535)
587 } /* }}} int sd_output_add */
589 int sd_output_register_metric(sd_output_t *out, data_set_t const *ds,
590 value_list_t const *vl) {
592 for (size_t i = 0; i < ds->ds_num; i++) {
593 char buffer[4 * DATA_MAX_NAME_LEN];
594 metric_type(buffer, sizeof(buffer), ds, vl, i);
596 char *key = strdup(buffer);
597 int status = c_avl_insert(out->metric_descriptors, key, NULL);
605 } /* }}} int sd_output_register_metric */
607 char *sd_output_reset(sd_output_t *out) /* {{{ */
609 sd_output_finalize(out);
611 unsigned char const *json_buffer = NULL;
612 yajl_gen_get_buf(out->gen, &json_buffer, &(size_t){0});
613 char *ret = strdup((void const *)json_buffer);
615 sd_output_reset_staged(out);
617 yajl_gen_free(out->gen);
618 out->gen = yajl_gen_alloc(/* funcs = */ NULL);
620 sd_output_initialize(out);
623 } /* }}} char *sd_output_reset */
625 sd_resource_t *sd_resource_create(char const *type) /* {{{ */
627 sd_resource_t *res = malloc(sizeof(*res));
630 memset(res, 0, sizeof(*res));
632 res->type = strdup(type);
633 if (res->type == NULL) {
642 } /* }}} sd_resource_t *sd_resource_create */
644 void sd_resource_destroy(sd_resource_t *res) /* {{{ */
649 for (size_t i = 0; i < res->labels_num; i++) {
650 sfree(res->labels[i].key);
651 sfree(res->labels[i].value);
656 } /* }}} void sd_resource_destroy */
658 int sd_resource_add_label(sd_resource_t *res, char const *key,
659 char const *value) /* {{{ */
661 if ((res == NULL) || (key == NULL) || (value == NULL))
665 realloc(res->labels, sizeof(*res->labels) * (res->labels_num + 1));
670 l = res->labels + res->labels_num;
672 l->key = strdup(key);
673 l->value = strdup(value);
674 if ((l->key == NULL) || (l->value == NULL)) {
682 } /* }}} int sd_resource_add_label */
688 * "valueType": enum(ValueType),
689 * "description": string,
692 static int format_label_descriptor(yajl_gen gen, char const *key) {
694 yajl_gen_map_open(gen);
696 int status = json_string(gen, "key") || json_string(gen, key) ||
697 json_string(gen, "valueType") || json_string(gen, "STRING");
702 yajl_gen_map_close(gen);
704 } /* }}} int format_label_descriptor */
713 * object(LabelDescriptor)
716 * "metricKind": enum(MetricKind),
717 * "valueType": enum(ValueType),
719 * "description": string,
720 * "displayName": string,
723 int sd_format_metric_descriptor(char *buffer, size_t buffer_size,
724 data_set_t const *ds, value_list_t const *vl,
727 yajl_gen gen = yajl_gen_alloc(/* funcs = */ NULL);
732 int ds_type = ds->ds[ds_index].type;
734 yajl_gen_map_open(gen);
737 json_string(gen, "type") || format_metric_type(gen, ds, vl, ds_index) ||
738 json_string(gen, "metricKind") || format_metric_kind(gen, ds_type) ||
739 json_string(gen, "valueType") || format_value_type(gen, ds_type) ||
740 json_string(gen, "labels");
746 char const *labels[] = {"host", "plugin_instance", "type_instance"};
747 yajl_gen_array_open(gen);
749 for (size_t i = 0; i < STATIC_ARRAY_SIZE(labels); i++) {
750 int status = format_label_descriptor(gen, labels[i]);
757 yajl_gen_array_close(gen);
758 yajl_gen_map_close(gen);
760 unsigned char const *tmp = NULL;
761 yajl_gen_get_buf(gen, &tmp, &(size_t){0});
762 sstrncpy(buffer, (void const *)tmp, buffer_size);
766 } /* }}} int sd_format_metric_descriptor */