3 # collectd - contrib/rrd_filter.px
4 # Copyright (C) 2007-2008 Florian octo Forster
6 # This program is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by the
8 # Free Software Foundation; only version 2 of the License is applicable.
10 # This program is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 # Florian octo Forster <octo at verplant.org>
27 rrd_filter.px - Perform same advanced non-standard operations on an RRD file.
31 rrd_filter.px -i input.rrd -o output.rrd [options]
35 extractDS.px requires the RRDTool binary, Perl and the included L<Getopt::Long>
40 use Getopt::Long ('GetOptions');
54 The following options can be passed on the command line:
58 =item B<--infile> I<file>
62 Reads from I<file>. If I<file> ends in C<.rrd>, then C<rrdtool dump> is invoked
63 to create an XML dump of the RRD file. Otherwise the XML dump is expected
64 directly. The special filename C<-> can be used to read from STDIN.
66 =item B<--outfile> I<file>
70 Writes output to I<file>. If I<file> ends in C<.rrd>, then C<rrdtool restore>
71 is invoked to create a binary RRD file. Otherwise an XML output is written. The
72 special filename C<-> can be used to write to STDOUT.
74 =item B<--map> I<in_ds>:I<out_ds>
76 =item B<-m> I<in_ds>:I<out_ds>
78 Writes the datasource I<in_ds> to the output and renames it to I<out_ds>. This
79 is useful to extract one DS from an RRD file.
81 =item B<--step> I<seconds>
83 =item B<-s> I<seconds>
85 Changes the step of the output RRD file to be I<seconds>. The new stepsize must
86 be a multiple of the old stepsize of the other way around. When increasing the
87 stepsize the number of PDPs in each RRA must be dividable by the factor by
88 which the stepsize is increased. The length of CDPs and the absolute length of
89 RRAs (and thus the data itself) is not altered.
93 step = 10, rra_steps = 12 => step = 60, rra_steps = 2
94 step = 300, rra_steps = 1 => step = 10, rra_steps = 30
96 =item B<--rra> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
98 =item B<-a> B<RRA>:I<CF>:I<XFF>:I<steps>:I<rows>
100 Inserts a new RRA in the generated RRD file. This is done B<after> the step has
101 been adjusted, take that into account when specifying I<steps> and I<rows>. For
102 an explanation of the format please see L<rrdcreate(1)>.
108 GetOptions ("infile|i=s" => \$InFile,
109 "outfile|o=s" => \$OutFile,
112 my ($in_ds, $out_ds) = split (':', $_[1]);
113 if (!defined ($in_ds) || !defined ($out_ds))
115 print STDERR "Argument for `map' incorrect! The format is `--map in_ds:out_ds'\n";
118 push (@$InDS, $in_ds);
119 push (@$OutDS, $out_ds);
121 'step|s=i' => \$Step,
124 my ($rra, $cf, $xff, $steps, $rows) = split (':', $_[1]);
125 if (($rra ne 'RRA') || !defined ($rows))
127 print STDERR "Please use the standard RRDTool syntax when adding RRAs. I. e. RRA:<cf><xff>:<steps>:<rows>.\n";
130 push (@$NewRRAs, {cf => $cf, xff => $xff, steps => $steps, rows => $rows});
134 if (!$InFile || !$OutFile)
136 print STDERR "Usage: $0 -i <infile> -m <in_ds>:<out_ds> -s <step>\n";
139 if ((1 + @$InDS) != (1 + @$OutDS))
141 print STDERR "You need the same amount of in- and out-DSes\n";
145 main ($InFile, $OutFile);
151 # state 0 == searching for DS index
152 # state 1 == parse RRA header
153 # state 2 == parse values
156 sub handle_line_dsmap
162 if ((@$InDS == 0) || (@$OutDS == 0))
164 post_line ($line, $index + 1);
168 if (!defined ($state))
174 # $ds_index->[new_index] = old_index
176 for (my $i = 0; $i < @$InDS; $i++)
178 $ds_index->[$i] = -1;
184 if ($line =~ m/<ds>/)
187 $out_cache->[$current_index] = $line;
189 elsif ($line =~ m#<name>\s*([^<\s]+)\s*</name>#)
191 # old_index == $current_index
193 for (my $i = 0; $i < @$InDS; $i++)
195 next if ($ds_index->[$i] >= 0);
197 if ($1 eq $InDS->[$i])
199 $line =~ s#<name>\s*([^<\s]+)\s*</name>#<name> $OutDS->[$i] </name>#;
200 $ds_index->[$i] = $current_index;
205 $out_cache->[$current_index] .= $line;
207 elsif ($line =~ m#</ds>#)
209 $out_cache->[$current_index] .= $line;
211 elsif ($line =~ m#<rra>#)
213 # Print out all the DS definitions we need
214 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
216 my $old_index = $ds_index->[$new_index];
217 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
219 post_line ("$1\n", $index + 1);
223 # Clear the cache - it's used in state1, too.
224 for (my $i = 0; $i <= $current_index; $i++)
226 $out_cache->[$i] = '';
233 elsif ($current_index == -1)
235 # Print all the lines before the first DS definition
240 # Something belonging to a DS-definition
241 $out_cache->[$current_index] .= $line;
246 if ($line =~ m#<ds>#)
249 $out_cache->[$current_index] .= $line;
251 elsif ($line =~ m#</cdp_prep>#)
253 # Print out all the DS definitions we need
254 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
256 my $old_index = $ds_index->[$new_index];
257 while ($out_cache->[$old_index] =~ m/^(.*)$/gm)
259 post_line ("$1\n", $index + 1);
264 for (my $i = 0; $i <= $current_index; $i++)
266 $out_cache->[$i] = '';
272 elsif ($line =~ m#<database>#)
277 elsif ($current_index == -1)
279 # Print all the lines before the first DS definition
285 # Something belonging to a DS-definition
286 $out_cache->[$current_index] .= $line;
291 if ($line =~ m#</database>#)
304 if ($line =~ m#(<!-- .*? -->)#)
311 while ($line =~ m#<v>\s*([^<\s]+)\s*</v>#g)
317 for (my $new_index = 0; $new_index < @$InDS; $new_index++)
319 my $old_index = $ds_index->[$new_index];
320 $ret .= '<v> ' . $values[$old_index] . ' </v> ';
332 post_line ($ret, $index + 1);
334 }} # handle_line_dsmap
341 my $step_factor_down;
349 post_line ($line, $index + 1);
353 $step_factor_up ||= 0;
354 $step_factor_down ||= 0;
356 if (($step_factor_up == 0) && ($step_factor_down == 0))
358 if ($line =~ m#<step>\s*(\d+)\s*</step>#i)
360 my $old_step = 0 + $1;
361 if ($Step < $old_step)
363 $step_factor_down = int ($old_step / $Step);
364 if (($step_factor_down * $Step) != $old_step)
366 print STDERR "The old step ($old_step seconds) "
367 . "is not a multiple of the new step "
368 . "($Step seconds).\n";
371 $line = "<step> $Step </step>\n";
373 elsif ($Step > $old_step)
375 $step_factor_up = int ($Step / $old_step);
376 if (($step_factor_up * $old_step) != $Step)
378 print STDERR "The new step ($Step seconds) "
379 . "is not a multiple of the old step "
380 . "($old_step seconds).\n";
383 $line = "<step> $Step </step>\n";
391 elsif ($line =~ m#<pdp_per_row>\s*(\d+)\s*</pdp_per_row>#i)
393 my $old_val = 0 + $1;
397 $new_val = int ($old_val / $step_factor_up);
398 if (($new_val * $step_factor_up) != $old_val)
400 print STDERR "Can't divide number of PDPs per row ($old_val) by step-factor ($step_factor_up).\n";
406 $new_val = $step_factor_down * $old_val;
408 $line = "<pdp_per_row> $new_val </pdp_per_row>\n";
411 post_line ($line, $index + 1);
412 }} # handle_line_step
415 # The _add RRA_ handler
420 sub handle_line_add_rra
425 my $post = sub { for (@_) { post_line ($_, $index + 1); } };
429 if (!@$NewRRAs || $add_rra_done)
435 if ($line =~ m#<ds>#i)
439 elsif ($line =~ m#<rra>#i)
441 for (my $i = 0; $i < @$NewRRAs; $i++)
443 my $rra = $NewRRAs->[$i];
446 "\t\t<cf> $rra->{'cf'} </cf>\n",
447 "\t\t<pdp_per_row> $rra->{'steps'} </pdp_per_row>\n",
449 "\t\t\t<xff> $rra->{'xff'} </xff>\n",
453 for (my $j = 0; $j < $num_ds; $j++)
455 $post->("\t\t\t<ds>\n",
456 "\t\t\t\t<primary_value> NaN </primary_value>\n",
457 "\t\t\t\t<secondary_value> NaN </secondary_value>\n",
458 "\t\t\t\t<value> NaN </value>\n",
459 "\t\t\t\t<unknown_datapoints> 0 </unknown_datapoints>\n",
463 $post->("\t\t</cdp_prep>\n", "\t\t<database>\n");
464 $temp = "\t\t\t<row>" . join ('', map { "<v> NaN </v>" } (1 .. $num_ds)) . "</row>\n";
465 for (my $j = 0; $j < $rra->{'rows'}; $j++)
469 $post->("\t\t</database>\n");
474 }} # handle_line_add_rra
477 # The _output_ handler
486 sub handle_line_output
493 post_line ($line, $index + 1);
498 }} # handle_line_output
509 die unless (ref ($handler) eq 'CODE');
510 push (@handlers, $handler);
522 print "DEBUG: post_line ($copy, $index);\n";
525 if ($index > $#handlers)
529 $handlers[$index]->($line, $index);
537 set_output ($out_fh);
541 add_handler (\&handle_line_dsmap);
546 add_handler (\&handle_line_step);
551 add_handler (\&handle_line_add_rra);
554 add_handler (\&handle_line_output);
556 while (my $line = <$in_fh>)
558 post_line ($line, 0);
565 my $out_file = shift;
570 my $in_needs_close = 1;
571 my $out_needs_close = 1;
573 if ($in_file =~ m/\.rrd$/i)
575 open ($in_fh, '-|', 'rrdtool', 'dump', $in_file) or die ("open (rrdtool): $!");
577 elsif ($in_file eq '-')
584 open ($in_fh, '<', $in_file) or die ("open ($in_file): $!");
587 if ($out_file =~ m/\.rrd$/i)
589 open ($out_fh, '|-', 'rrdtool', 'restore', '-', $out_file) or die ("open (rrdtool): $!");
591 elsif ($out_file eq '-')
594 $out_needs_close = 0;
598 open ($out_fh, '>', $out_file) or die ("open ($out_file): $!");
601 handle_fh ($in_fh, $out_fh);
607 if ($out_needs_close)
615 This script is licensed under the GNU general public license, versionE<nbsp>2
620 Florian octo Forster E<lt>octo at verplant.orgE<gt>