From 4aecad69a5640d208567d569636bfc2ddfed4b92 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Sun, 24 Feb 2008 12:11:36 +0100 Subject: [PATCH] tail plugin: Add a plugin to tail and parse logfiles. --- README | 4 + configure.in | 1 + src/Makefile.am | 8 ++ src/collectd.conf.pod | 106 +++++++++++++++ src/tail.c | 353 +++++++++++++++++++++++++++++++++++++++++++++++++ src/utils_tail_match.c | 6 + 6 files changed, 478 insertions(+) create mode 100644 src/tail.c diff --git a/README b/README index 4f3c8120..aff69564 100644 --- a/README +++ b/README @@ -159,6 +159,10 @@ Features - swap Pages swapped out onto harddisk or whatever is called `swap' by the OS.. + - tail + Follows (tails) logfiles, parses them by lines and submits matched + values. + - tape Bytes and operations read and written on tape devices. Solaris only. diff --git a/configure.in b/configure.in index f59fc6b0..c27297de 100644 --- a/configure.in +++ b/configure.in @@ -2075,6 +2075,7 @@ AC_PLUGIN([serial], [$plugin_serial], [serial port traffic]) AC_PLUGIN([snmp], [$with_libnetsnmp], [SNMP querying plugin]) AC_PLUGIN([swap], [$plugin_swap], [Swap usage statistics]) AC_PLUGIN([syslog], [$have_syslog], [Syslog logging plugin]) +AC_PLUGIN([tail], [yes], [Parsing of logfiles]) AC_PLUGIN([tape], [$plugin_tape], [Tape drive statistics]) AC_PLUGIN([tcpconns], [$plugin_tcpconns], [TCP connection statistics]) AC_PLUGIN([unixsock], [yes], [Unixsock communication plugin]) diff --git a/src/Makefile.am b/src/Makefile.am index a51c161b..c3d0b1d3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -566,6 +566,14 @@ collectd_LDADD += "-dlopen" syslog.la collectd_DEPENDENCIES += syslog.la endif +if BUILD_PLUGIN_TAIL +pkglib_LTLIBRARIES += tail.la +tail_la_SOURCES = tail.c +tail_la_LDFLAGS = -module -avoid-version +collectd_LDADD += "-dlopen" tail.la +collectd_DEPENDENCIES += tail.la +endif + if BUILD_PLUGIN_TAPE pkglib_LTLIBRARIES += tape.la tape_la_SOURCES = tape.c diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index 383dda63..7afe3f12 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -958,6 +958,112 @@ debugging support. =back +=head2 Plugin C + +The C plugins follows logfiles, just like L does, parses +each line and dispatches found values. What is matched can be configured by the +user using (extended) regular expressions, as described in L. + + + + Instance "exim" + + Regex "S=([1-9][0-9]*)" + DSType "CounterAdd" + Type "ipt_bytes" + Instance "total" + + + Regex "\\" + DSType "CounterInc" + Type "email_count" + Instance "local_user" + + + + +The config consists of one or more B blocks, each of which configures one +logfile to parse. Within each B block, there are one or more B +blocks, which configure a regular expression to search for. + +The B option in the B block may be used to set the plugin +instance. So in the above example the plugin name C would be used. +This plugin instance is for all B blocks that B it, until the +next B option. This way you can extract several plugin instances from +one logfile, handy when parsing syslog and the like. + +Each B block has the following options to describe how the match should +be performed: + +=over 4 + +=item B I + +Sets the regular expression to use for matching against a line. The first +subexpression has to match something that can be turned into a number by +L or L, depending on the value of C, see +below. Because B regular expressions are used, you do not need to use +backslashes for subexpressions! If in doubt, please consult L. Due to +collectd's config parsing you need to escape backslashes, though. So if you +want to match literal parentheses you need to do the following: + + Regex "SPAM \\(Score: (-?[0-9]+\\.[0-9]+)\\)" + +=item B I + +Sets how the values are cumulated. I is one of: + +=over 4 + +=item B + +Calculate the average. + +=item B + +Use the smallest number only. + +=item B + +Use the greatest number only. + +=item B + +Use the last number found. + +=item B + +The matched number is a counter. Simply sets the internal counter to this +value. + +=item B + +Add the matched value to the internal counter. + +=item B + +Increase the internal counter by one. This B is the only one that does +not use the matched subexpression, but simply counts the number of matched +lines. Thus, you may use a regular expression without submatch in this case. + +=back + +As you'd expect the B types interpret the submatch as a floating point +number, using L. The B and B interpret the +submatch as an integer using L. B does not use the +submatch at all and it may be omitted in this case. + +=item B I + +Sets the type used to dispatch this value. Detailed information about types and +their configuration can be found in L. + +=item B I + +This optional setting sets the type instance to use. + +=back + =head2 Plugin C The C counts the number of currently established TCP diff --git a/src/tail.c b/src/tail.c new file mode 100644 index 00000000..7d77e253 --- /dev/null +++ b/src/tail.c @@ -0,0 +1,353 @@ +/** + * collectd - src/tail.c + * Copyright (C) 2008 Florian octo Forster + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; only version 2 of the License is applicable. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Florian octo Forster + **/ + +#include "collectd.h" +#include "common.h" +#include "plugin.h" +#include "utils_tail_match.h" + +/* + * + * + * Instance "exim" + * + * Regex "S=([1-9][0-9]*)" + * DSType "CouterAdd" + * Type "ipt_bytes" + * Instance "total" + * + * + * + */ + +struct ctail_config_match_s +{ + char *regex; + int flags; + char *type; + char *type_instance; +}; +typedef struct ctail_config_match_s ctail_config_match_t; + +cu_tail_match_t **tail_match_list = NULL; +size_t tail_match_list_num = 0; + +static int ctail_config_add_string (const char *name, char **dest, oconfig_item_t *ci) +{ + if ((ci->values_num != 1) || (ci->values[0].type != OCONFIG_TYPE_STRING)) + { + WARNING ("tail plugin: `%s' needs exactly one string argument.", name); + return (-1); + } + + sfree (*dest); + *dest = strdup (ci->values[0].value.string); + if (*dest == NULL) + return (-1); + + return (0); +} /* int ctail_config_add_string */ + +static int ctail_config_add_match_dstype (ctail_config_match_t *cm, + oconfig_item_t *ci) +{ + if ((ci->values_num != 1) || (ci->values[0].type != OCONFIG_TYPE_STRING)) + { + WARNING ("tail plugin: `DSType' needs exactly one string argument."); + return (-1); + } + + if (strncasecmp ("Gauge", ci->values[0].value.string, strlen ("Gauge")) == 0) + { + cm->flags = UTILS_MATCH_DS_TYPE_GAUGE; + if (strcasecmp ("GaugeAverage", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_GAUGE_AVERAGE; + else if (strcasecmp ("GaugeMin", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_GAUGE_MIN; + else if (strcasecmp ("GaugeMax", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_GAUGE_MAX; + else if (strcasecmp ("GaugeLast", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_GAUGE_LAST; + else + cm->flags = 0; + } + else if (strncasecmp ("Counter", ci->values[0].value.string, strlen ("Counter")) == 0) + { + cm->flags = UTILS_MATCH_DS_TYPE_COUNTER; + if (strcasecmp ("CouterSet", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_COUNTER_SET; + else if (strcasecmp ("CouterAdd", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_COUNTER_ADD; + else if (strcasecmp ("CouterInc", ci->values[0].value.string) == 0) + cm->flags |= UTILS_MATCH_CF_COUNTER_INC; + else + cm->flags = 0; + } + else + { + cm->flags = 0; + } + + if (cm->flags == 0) + { + WARNING ("tail plugin: `%s' is not a valid argument to `DSType'.", + ci->values[0].value.string); + return (-1); + } + + return (0); +} /* int ctail_config_add_match_dstype */ + +static int ctail_config_add_match (cu_tail_match_t *tm, + const char *plugin_instance, oconfig_item_t *ci) +{ + ctail_config_match_t cm; + int status; + int i; + + memset (&cm, '\0', sizeof (cm)); + + if (ci->values_num != 0) + { + WARNING ("tail plugin: Ignoring arguments for the `Match' block."); + } + + status = 0; + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *option = ci->children + i; + + if (strcasecmp ("Regex", option->key) == 0) + status = ctail_config_add_string ("Regex", &cm.regex, option); + else if (strcasecmp ("DSType", option->key) == 0) + status = ctail_config_add_match_dstype (&cm, option); + else if (strcasecmp ("Type", option->key) == 0) + status = ctail_config_add_string ("Type", &cm.type, option); + else if (strcasecmp ("Instance", option->key) == 0) + status = ctail_config_add_string ("Instance", &cm.type_instance, option); + else + { + WARNING ("tail plugin: Option `%s' not allowed here.", option->key); + status = -1; + } + + if (status != 0) + break; + } /* for (i = 0; i < ci->children_num; i++) */ + + while (status == 0) + { + if (cm.regex == NULL) + { + WARNING ("tail plugin: `Regex' missing in `Match' block."); + status = -1; + break; + } + + if (cm.type == NULL) + { + WARNING ("tail plugin: `Type' missing in `Match' block."); + status = -1; + break; + } + + if (cm.flags == 0) + { + WARNING ("tail plugin: `DSType' missing in `Match' block."); + status = -1; + break; + } + + break; + } /* while (status == 0) */ + + if (status == 0) + { + status = tail_match_add_match_simple (tm, cm.regex, cm.flags, + "tail", plugin_instance, cm.type, cm.type_instance); + + if (status != 0) + { + ERROR ("tail plugin: tail_match_add_match_simple failed."); + } + } + + sfree (cm.regex); + sfree (cm.type); + sfree (cm.type_instance); + + return (status); +} /* int ctail_config_add_match */ + +static int ctail_config_add_file (oconfig_item_t *ci) +{ + cu_tail_match_t *tm; + char *plugin_instance = NULL; + int num_matches = 0; + int status; + int i; + + if ((ci->values_num != 1) || (ci->values[0].type != OCONFIG_TYPE_STRING)) + { + WARNING ("tail plugin: `File' needs exactly one string argument."); + return (-1); + } + + tm = tail_match_create (ci->values[0].value.string); + if (tm == NULL) + { + ERROR ("tail plugin: tail_match_create (%s) failed.", + ci->values[0].value.string); + return (-1); + } + + status = 0; + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *option = ci->children + i; + + if (strcasecmp ("Match", option->key) == 0) + { + status = ctail_config_add_match (tm, plugin_instance, option); + if (status == 0) + num_matches++; + /* Be mild with failed matches.. */ + status = 0; + } + else if (strcasecmp ("Instance", option->key) == 0) + status = ctail_config_add_string ("Instance", &plugin_instance, option); + else + { + WARNING ("tail plugin: Option `%s' not allowed here.", option->key); + status = -1; + } + + if (status != 0) + break; + } /* for (i = 0; i < ci->children_num; i++) */ + + if (num_matches == 0) + { + ERROR ("tail plugin: No (valid) matches found for file `%s'.", + ci->values[0].value.string); + tail_match_destroy (tm); + return (-1); + } + else + { + cu_tail_match_t **temp; + + temp = (cu_tail_match_t **) realloc (tail_match_list, + sizeof (cu_tail_match_t *) * (tail_match_list_num + 1)); + if (temp == NULL) + { + ERROR ("tail plugin: realloc failed."); + tail_match_destroy (tm); + return (-1); + } + + tail_match_list = temp; + tail_match_list[tail_match_list_num] = tm; + tail_match_list_num++; + } + + return (0); +} /* int ctail_config_add_file */ + +static int ctail_config (oconfig_item_t *ci) +{ + int i; + + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *option = ci->children + i; + + if (strcasecmp ("File", option->key) == 0) + ctail_config_add_file (option); + else + { + WARNING ("tail plugin: Option `%s' not allowed here.", option->key); + } + } /* for (i = 0; i < ci->children_num; i++) */ + + return (0); +} /* int ctail_config */ + +static int ctail_init (void) +{ + if (tail_match_list_num == 0) + { + WARNING ("tail plugin: File list is empty. Returning an error."); + return (-1); + } + + return (0); +} /* int ctail_init */ + +static int ctail_read (void) +{ + int success = 0; + int i; + + for (i = 0; i < tail_match_list_num; i++) + { + int status; + + status = tail_match_read (tail_match_list[i]); + if (status != 0) + { + ERROR ("tail plugin: tail_match_read[%i] failed.", i); + } + else + { + success++; + } + } + + if (success == 0) + return (-1); + return (0); +} /* int ctail_read */ + +static int ctail_shutdown (void) +{ + int i; + + for (i = 0; i < tail_match_list_num; i++) + { + tail_match_destroy (tail_match_list[i]); + tail_match_list[i] = NULL; + } + sfree (tail_match_list); + tail_match_list_num = 0; + + return (0); +} /* int ctail_shutdown */ + +void module_register (void) +{ + plugin_register_complex_config ("tail", ctail_config); + plugin_register_init ("tail", ctail_init); + plugin_register_read ("tail", ctail_read); + plugin_register_shutdown ("tail", ctail_shutdown); +} /* void module_register */ + +/* vim: set sw=2 sts=2 ts=8 : */ diff --git a/src/utils_tail_match.c b/src/utils_tail_match.c index b71e2059..06412b9e 100644 --- a/src/utils_tail_match.c +++ b/src/utils_tail_match.c @@ -86,6 +86,12 @@ static int simple_submit_match (cu_match_t *match, void *user_data) plugin_dispatch_values (data->type, &vl); + if (match_value->ds_type & UTILS_MATCH_DS_TYPE_GAUGE) + { + match_value->value.gauge = NAN; + match_value->values_num = 0; + } + return (0); } /* int simple_submit_match */ -- 2.11.0