From e324dc1e06b6816613a1af271ac6a0adc49355dc Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Thu, 4 Sep 2014 07:18:51 +0200 Subject: [PATCH] write_http plugin: Implement the "BufferSize" option. Fixes: #722 --- src/collectd.conf.in | 1 + src/collectd.conf.pod | 15 +++++++++++++-- src/write_http.c | 50 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/collectd.conf.in b/src/collectd.conf.in index eed7bc81..279e8045 100644 --- a/src/collectd.conf.in +++ b/src/collectd.conf.in @@ -1212,6 +1212,7 @@ # SSLVersion "TLSv1" # Format "Command" # StoreRates false +# BufferSize 4096 # # diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index 9293445e..03483672 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -6659,8 +6659,9 @@ want to use authentication all three fields must be set. =head2 Plugin C -This output plugin submits values to an http server by POST them using the -PUTVAL plain-text protocol. Each destination you want to post data to needs to +This output plugin submits values to an HTTP server using POST requests and +encoding metircs with JSON or using the C command described in +L. Each destination you want to post data to needs to have one B block, within which the destination can be configured further, for example by specifying authentication data. @@ -6670,6 +6671,7 @@ Synopsis: User "collectd" Password "weCh3ik0" + Format JSON @@ -6746,6 +6748,15 @@ If set to B, convert counter values to rates. If set to B (the default) counter values are stored as is, i.Ee. as an increasing integer number. +=item B I + +Sets the send buffer size to I. By increasing this buffer, less HTTP +requests will be generated, but more metrics will be batched / metrics are +cached for longer before being sent, introducing additional delay until they +are available on the server side. I must be at least 1024 and cannot +exceed the size of an C, i.e. 2EGByte. +Defaults to C<4096>. + =back =head2 Plugin C diff --git a/src/write_http.c b/src/write_http.c index eee5a1c6..198fb48d 100644 --- a/src/write_http.c +++ b/src/write_http.c @@ -2,7 +2,7 @@ * collectd - src/write_http.c * Copyright (C) 2009 Paul Sadauskas * Copyright (C) 2009 Doug MacEachern - * Copyright (C) 2007-2009 Florian octo Forster + * Copyright (C) 2007-2014 Florian octo Forster * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -36,6 +36,10 @@ #include +#ifndef WRITE_HTTP_DEFAULT_BUFFER_SIZE +# define WRITE_HTTP_DEFAULT_BUFFER_SIZE 4096 +#endif + /* * Private variables */ @@ -63,7 +67,8 @@ struct wh_callback_s CURL *curl; char curl_errbuf[CURL_ERROR_SIZE]; - char send_buffer[4096]; + char *send_buffer; + size_t send_buffer_size; size_t send_buffer_free; size_t send_buffer_fill; cdtime_t send_buffer_init_time; @@ -74,8 +79,8 @@ typedef struct wh_callback_s wh_callback_t; static void wh_reset_buffer (wh_callback_t *cb) /* {{{ */ { - memset (cb->send_buffer, 0, sizeof (cb->send_buffer)); - cb->send_buffer_free = sizeof (cb->send_buffer); + memset (cb->send_buffer, 0, cb->send_buffer_size); + cb->send_buffer_free = cb->send_buffer_size; cb->send_buffer_fill = 0; cb->send_buffer_init_time = cdtime (); @@ -280,7 +285,11 @@ static void wh_callback_free (void *data) /* {{{ */ wh_flush_nolock (/* timeout = */ 0, cb); - curl_easy_cleanup (cb->curl); + if (cb->curl != NULL) + { + curl_easy_cleanup (cb->curl); + cb->curl = NULL; + } sfree (cb->location); sfree (cb->user); sfree (cb->pass); @@ -290,6 +299,7 @@ static void wh_callback_free (void *data) /* {{{ */ sfree (cb->clientkey); sfree (cb->clientcert); sfree (cb->clientkeypass); + sfree (cb->send_buffer); sfree (cb); } /* }}} void wh_callback_free */ @@ -371,8 +381,8 @@ static int wh_write_command (const data_set_t *ds, const value_list_t *vl, /* {{ DEBUG ("write_http plugin: <%s> buffer %zu/%zu (%g%%) \"%s\"", cb->location, - cb->send_buffer_fill, sizeof (cb->send_buffer), - 100.0 * ((double) cb->send_buffer_fill) / ((double) sizeof (cb->send_buffer)), + cb->send_buffer_fill, cb->send_buffer_size, + 100.0 * ((double) cb->send_buffer_fill) / ((double) cb->send_buffer_size), command); /* Check if we have enough space for this command. */ @@ -426,8 +436,8 @@ static int wh_write_json (const data_set_t *ds, const value_list_t *vl, /* {{{ * DEBUG ("write_http plugin: <%s> buffer %zu/%zu (%g%%)", cb->location, - cb->send_buffer_fill, sizeof (cb->send_buffer), - 100.0 * ((double) cb->send_buffer_fill) / ((double) sizeof (cb->send_buffer))); + cb->send_buffer_fill, cb->send_buffer_size, + 100.0 * ((double) cb->send_buffer_fill) / ((double) cb->send_buffer_size)); /* Check if we have enough space for this command. */ pthread_mutex_unlock (&cb->send_lock); @@ -485,6 +495,7 @@ static int config_set_format (wh_callback_t *cb, /* {{{ */ static int wh_config_url (oconfig_item_t *ci) /* {{{ */ { wh_callback_t *cb; + int buffer_size = 0; user_data_t user_data; int i; @@ -560,6 +571,8 @@ static int wh_config_url (oconfig_item_t *ci) /* {{{ */ config_set_format (cb, child); else if (strcasecmp ("StoreRates", child->key) == 0) cf_util_get_boolean (child, &cb->store_rates); + else if (strcasecmp ("BufferSize", child->key) == 0) + cf_util_get_int (child, &buffer_size); else { ERROR ("write_http plugin: Invalid configuration " @@ -567,6 +580,25 @@ static int wh_config_url (oconfig_item_t *ci) /* {{{ */ } } + /* Determine send_buffer_size. */ + cb->send_buffer_size = WRITE_HTTP_DEFAULT_BUFFER_SIZE; + if (buffer_size >= 1024) + cb->send_buffer_size = (size_t) buffer_size; + else if (buffer_size != 0) + ERROR ("write_http plugin: Ignoring invalid BufferSize setting (%d).", + buffer_size); + + /* Allocate the buffer. */ + cb->send_buffer = malloc (cb->send_buffer_size); + if (cb->send_buffer == 0) + { + ERROR ("write_http plugin: malloc(%zu) failed.", cb->send_buffer_size); + wh_callback_free (cb); + return (-1); + } + /* Nulls the buffer and sets ..._free and ..._fill. */ + wh_reset_buffer (cb); + DEBUG ("write_http: Registering write callback with URL %s", cb->location); -- 2.11.0