1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_create.c creates new rrds
5 *****************************************************************************/
8 #include "rrd_rpncalc.h"
11 #include "rrd_is_thread_safe.h"
13 unsigned long FnvHash(
15 int create_hw_contingent_rras(
17 unsigned short period,
18 unsigned long hashed_name);
28 struct option long_options[] = {
29 {"start", required_argument, 0, 'b'},
30 {"step", required_argument, 0, 's'},
35 time_t last_up = time(NULL) - 10;
36 unsigned long pdp_step = 300;
37 struct rrd_time_value last_up_tv;
38 char *parsetime_error = NULL;
43 opterr = 0; /* initialize getopt */
46 opt = getopt_long(argc, argv, "b:s:", long_options, &option_index);
53 if ((parsetime_error = parsetime(optarg, &last_up_tv))) {
54 rrd_set_error("start time: %s", parsetime_error);
57 if (last_up_tv.type == RELATIVE_TO_END_TIME ||
58 last_up_tv.type == RELATIVE_TO_START_TIME) {
59 rrd_set_error("specifying time relative to the 'start' "
60 "or 'end' makes no sense here");
64 last_up = mktime(&last_up_tv.tm) +last_up_tv.offset;
66 if (last_up < 3600 * 24 * 365 * 10) {
68 ("the first entry to the RRD should be after 1980");
74 long_tmp = atol(optarg);
76 rrd_set_error("step size should be no less than one second");
84 rrd_set_error("unknown option '%c'", optopt);
86 rrd_set_error("unknown option '%s'", argv[optind - 1]);
91 rrd_set_error("need name of an rrd file to create");
94 rc = rrd_create_r(argv[optind],
96 argc - optind - 1, (const char **) (argv + optind + 1));
103 const char *filename,
104 unsigned long pdp_step,
113 char dummychar1[2], dummychar2[2];
114 unsigned short token_idx, error_flag, period = 0;
115 unsigned long hashed_name;
120 if ((rrd.stat_head = calloc(1, sizeof(stat_head_t))) == NULL) {
121 rrd_set_error("allocating rrd.stat_head");
127 if ((rrd.live_head = calloc(1, sizeof(live_head_t))) == NULL) {
128 rrd_set_error("allocating rrd.live_head");
133 /* set some defaults */
134 strcpy(rrd.stat_head->cookie, RRD_COOKIE);
135 strcpy(rrd.stat_head->version, RRD_VERSION);
136 rrd.stat_head->float_cookie = FLOAT_COOKIE;
137 rrd.stat_head->ds_cnt = 0; /* this will be adjusted later */
138 rrd.stat_head->rra_cnt = 0; /* ditto */
139 rrd.stat_head->pdp_step = pdp_step; /* 5 minute default */
141 /* a default value */
145 rrd.live_head->last_up = last_up;
147 /* optind points to the first non-option command line arg,
148 * in this case, the file name. */
149 /* Compute the FNV hash value (used by SEASONAL and DEVSEASONAL
151 hashed_name = FnvHash(filename);
152 for (i = 0; i < argc; i++) {
155 if (strncmp(argv[i], "DS:", 3) == 0) {
156 size_t old_size = sizeof(ds_def_t) * (rrd.stat_head->ds_cnt);
158 if ((rrd.ds_def = rrd_realloc(rrd.ds_def,
159 old_size + sizeof(ds_def_t))) ==
161 rrd_set_error("allocating rrd.ds_def");
165 memset(&rrd.ds_def[rrd.stat_head->ds_cnt], 0, sizeof(ds_def_t));
166 /* extract the name and type */
167 switch (sscanf(&argv[i][3],
168 DS_NAM_FMT "%1[:]" DST_FMT "%1[:]%n",
169 rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
171 rrd.ds_def[rrd.stat_head->ds_cnt].dst,
172 dummychar2, &offset)) {
175 rrd_set_error("Invalid DS name");
179 rrd_set_error("Invalid DS type");
181 case 4: /* (%n may or may not be counted) */
182 case 5: /* check for duplicate datasource names */
183 for (ii = 0; ii < rrd.stat_head->ds_cnt; ii++)
184 if (strcmp(rrd.ds_def[rrd.stat_head->ds_cnt].ds_nam,
185 rrd.ds_def[ii].ds_nam) == 0)
186 rrd_set_error("Duplicate DS name: %s",
187 rrd.ds_def[ii].ds_nam);
188 /* DS_type may be valid or not. Checked later */
191 rrd_set_error("invalid DS format");
193 if (rrd_test_error()) {
198 /* parse the remainder of the arguments */
199 switch (dst_conv(rrd.ds_def[rrd.stat_head->ds_cnt].dst)) {
204 parseGENERIC_DS(&argv[i][offset + 3], &rrd,
205 rrd.stat_head->ds_cnt);
208 parseCDEF_DS(&argv[i][offset + 3], &rrd,
209 rrd.stat_head->ds_cnt);
212 rrd_set_error("invalid DS type specified");
216 if (rrd_test_error()) {
220 rrd.stat_head->ds_cnt++;
221 } else if (strncmp(argv[i], "RRA:", 4) == 0) {
224 size_t old_size = sizeof(rra_def_t) * (rrd.stat_head->rra_cnt);
226 if ((rrd.rra_def = rrd_realloc(rrd.rra_def,
227 old_size + sizeof(rra_def_t))) ==
229 rrd_set_error("allocating rrd.rra_def");
233 memset(&rrd.rra_def[rrd.stat_head->rra_cnt], 0,
236 argvcopy = strdup(argv[i]);
237 token = strtok_r(&argvcopy[4], ":", &tokptr);
238 token_idx = error_flag = 0;
239 while (token != NULL) {
242 if (sscanf(token, CF_NAM_FMT,
243 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) !=
245 rrd_set_error("Failed to parse CF name");
247 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
250 /* initialize some parameters */
251 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
253 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
255 rrd.rra_def[rrd.stat_head->rra_cnt].
256 par[RRA_dependent_rra_idx].u_cnt =
257 rrd.stat_head->rra_cnt;
261 /* initialize some parameters */
262 rrd.rra_def[rrd.stat_head->rra_cnt].
263 par[RRA_seasonal_gamma].u_val = 0.1;
266 rrd.rra_def[rrd.stat_head->rra_cnt].
267 par[RRA_dependent_rra_idx].u_cnt = -1;
270 rrd.rra_def[rrd.stat_head->rra_cnt].
271 par[RRA_delta_pos].u_val = 2.0;
272 rrd.rra_def[rrd.stat_head->rra_cnt].
273 par[RRA_delta_neg].u_val = 2.0;
274 rrd.rra_def[rrd.stat_head->rra_cnt].
275 par[RRA_window_len].u_cnt = 3;
276 rrd.rra_def[rrd.stat_head->rra_cnt].
277 par[RRA_failure_threshold].u_cnt = 2;
278 rrd.rra_def[rrd.stat_head->rra_cnt].
279 par[RRA_dependent_rra_idx].u_cnt = -1;
281 /* invalid consolidation function */
284 ("Unrecognized consolidation function %s",
285 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
289 /* default: 1 pdp per cdp */
290 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt = 1;
294 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
301 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
305 rrd.rra_def[rrd.stat_head->rra_cnt].
306 par[RRA_cdp_xff_val].u_val = atof(token);
307 if (rrd.rra_def[rrd.stat_head->rra_cnt].
308 par[RRA_cdp_xff_val].u_val < 0.0
309 || rrd.rra_def[rrd.stat_head->rra_cnt].
310 par[RRA_cdp_xff_val].u_val >= 1.0)
312 ("Invalid xff: must be between 0 and 1");
318 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
321 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_alpha].
323 if (atof(token) <= 0.0 || atof(token) >= 1.0)
325 ("Invalid alpha: must be between 0 and 1");
329 rrd.rra_def[rrd.stat_head->rra_cnt].
330 par[RRA_seasonal_gamma].u_val = atof(token);
331 if (atof(token) <= 0.0 || atof(token) >= 1.0)
333 ("Invalid gamma: must be between 0 and 1");
334 rrd.rra_def[rrd.stat_head->rra_cnt].
335 par[RRA_seasonal_smooth_idx].u_cnt =
337 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt;
340 /* specifies the # of violations that constitutes the failure threshold */
341 rrd.rra_def[rrd.stat_head->rra_cnt].
342 par[RRA_failure_threshold].u_cnt = atoi(token);
344 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
346 ("Failure threshold is out of range %d, %d",
347 1, MAX_FAILURES_WINDOW_LEN);
350 /* specifies the index (1-based) of CF_DEVSEASONAL array
351 * associated with this CF_DEVPREDICT array. */
352 rrd.rra_def[rrd.stat_head->rra_cnt].
353 par[RRA_dependent_rra_idx].u_cnt =
357 rrd.rra_def[rrd.stat_head->rra_cnt].pdp_cnt =
364 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
367 rrd.rra_def[rrd.stat_head->rra_cnt].par[RRA_hw_beta].
369 if (atof(token) < 0.0 || atof(token) > 1.0)
371 ("Invalid beta: must be between 0 and 1");
375 /* specifies the index (1-based) of CF_HWPREDICT array
376 * associated with this CF_DEVSEASONAL or CF_SEASONAL array.
378 rrd.rra_def[rrd.stat_head->rra_cnt].
379 par[RRA_dependent_rra_idx].u_cnt =
383 /* specifies the window length */
384 rrd.rra_def[rrd.stat_head->rra_cnt].
385 par[RRA_window_len].u_cnt = atoi(token);
387 || atoi(token) > MAX_FAILURES_WINDOW_LEN)
389 ("Window length is out of range %d, %d", 1,
390 MAX_FAILURES_WINDOW_LEN);
391 /* verify that window length exceeds the failure threshold */
392 if (rrd.rra_def[rrd.stat_head->rra_cnt].
393 par[RRA_window_len].u_cnt <
394 rrd.rra_def[rrd.stat_head->rra_cnt].
395 par[RRA_failure_threshold].u_cnt)
397 ("Window length is shorter than the failure threshold");
400 /* shouldn't be any more arguments */
402 ("Unexpected extra argument for consolidation function DEVPREDICT");
405 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt =
412 (rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam)) {
414 /* specifies the index (1-based) of CF_DEVSEASONAL array
415 * associated with this CF_DEVFAILURES array. */
416 rrd.rra_def[rrd.stat_head->rra_cnt].
417 par[RRA_dependent_rra_idx].u_cnt =
422 /* length of the associated CF_SEASONAL and CF_DEVSEASONAL arrays. */
423 period = atoi(token);
425 rrd.rra_def[rrd.stat_head->rra_cnt].row_cnt)
427 ("Length of seasonal cycle exceeds length of HW prediction array");
430 /* shouldn't be any more arguments */
432 ("Unexpected extra argument for consolidation function %s",
433 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam);
438 /* If we are here, this must be a CF_HWPREDICT RRA.
439 * Specifies the index (1-based) of CF_SEASONAL array
440 * associated with this CF_HWPREDICT array. If this argument
441 * is missing, then the CF_SEASONAL, CF_DEVSEASONAL, CF_DEVPREDICT,
443 * arrays are created automatically. */
444 rrd.rra_def[rrd.stat_head->rra_cnt].
445 par[RRA_dependent_rra_idx].u_cnt = atoi(token) - 1;
448 /* should never get here */
449 rrd_set_error("Unknown error");
452 if (rrd_test_error()) {
453 /* all errors are unrecoverable */
458 token = strtok_r(NULL, ":", &tokptr);
464 "Creating RRA CF: %s, dep idx %lu, current idx %lu\n",
465 rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam,
466 rrd.rra_def[rrd.stat_head->rra_cnt].
467 par[RRA_dependent_rra_idx].u_cnt, rrd.stat_head->rra_cnt);
469 /* should we create CF_SEASONAL, CF_DEVSEASONAL, and CF_DEVPREDICT? */
470 if ((cf_conv(rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) ==
472 || cf_conv(rrd.rra_def[rrd.stat_head->rra_cnt].cf_nam) ==
474 && rrd.rra_def[rrd.stat_head->rra_cnt].
475 par[RRA_dependent_rra_idx].u_cnt == rrd.stat_head->rra_cnt) {
477 fprintf(stderr, "Creating HW contingent RRAs\n");
479 if (create_hw_contingent_rras(&rrd, period, hashed_name) ==
481 rrd_set_error("creating contingent RRA");
486 rrd.stat_head->rra_cnt++;
488 rrd_set_error("can't parse argument '%s'", argv[i]);
495 if (rrd.stat_head->rra_cnt < 1) {
496 rrd_set_error("you must define at least one Round Robin Archive");
501 if (rrd.stat_head->ds_cnt < 1) {
502 rrd_set_error("you must define at least one Data Source");
506 return rrd_create_fn(filename, &rrd);
509 void parseGENERIC_DS(
514 char minstr[DS_NAM_SIZE], maxstr[DS_NAM_SIZE];
519 temp = sscanf(def,"%lu:%18[^:]:%18[^:]",
520 &(rrd -> ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
523 if (sscanf(def, "%lu:%18[^:]:%18[^:]",
524 &(rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt),
525 minstr, maxstr) == 3) {
526 if (minstr[0] == 'U' && minstr[1] == 0)
527 rrd->ds_def[ds_idx].par[DS_min_val].u_val = DNAN;
529 rrd->ds_def[ds_idx].par[DS_min_val].u_val = atof(minstr);
531 if (maxstr[0] == 'U' && maxstr[1] == 0)
532 rrd->ds_def[ds_idx].par[DS_max_val].u_val = DNAN;
534 rrd->ds_def[ds_idx].par[DS_max_val].u_val = atof(maxstr);
536 if (!isnan(rrd->ds_def[ds_idx].par[DS_min_val].u_val) &&
537 !isnan(rrd->ds_def[ds_idx].par[DS_max_val].u_val) &&
538 rrd->ds_def[ds_idx].par[DS_min_val].u_val
539 >= rrd->ds_def[ds_idx].par[DS_max_val].u_val) {
540 rrd_set_error("min must be less than max in DS definition");
544 rrd_set_error("failed to parse data source %s", def);
548 /* Create the CF_DEVPREDICT, CF_DEVSEASONAL, CF_SEASONAL, and CF_FAILURES RRAs
549 * associated with a CF_HWPREDICT RRA. */
550 int create_hw_contingent_rras(
552 unsigned short period,
553 unsigned long hashed_name)
556 rra_def_t *current_rra;
558 /* save index to CF_HWPREDICT */
559 unsigned long hw_index = rrd->stat_head->rra_cnt;
561 /* advance the pointer */
562 (rrd->stat_head->rra_cnt)++;
563 /* allocate the memory for the 4 contingent RRAs */
564 old_size = sizeof(rra_def_t) * (rrd->stat_head->rra_cnt);
565 if ((rrd->rra_def = rrd_realloc(rrd->rra_def,
566 old_size + 4 * sizeof(rra_def_t))) ==
568 rrd_set_error("allocating rrd.rra_def");
572 memset(&(rrd->rra_def[rrd->stat_head->rra_cnt]), 0,
573 4 * sizeof(rra_def_t));
575 /* create the CF_SEASONAL RRA */
576 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
577 strcpy(current_rra->cf_nam, "SEASONAL");
578 current_rra->row_cnt = period;
579 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
580 current_rra->pdp_cnt = 1;
581 current_rra->par[RRA_seasonal_gamma].u_val =
582 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
583 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
584 rrd->rra_def[hw_index].par[RRA_dependent_rra_idx].u_cnt =
585 rrd->stat_head->rra_cnt;
587 /* create the CF_DEVSEASONAL RRA */
588 (rrd->stat_head->rra_cnt)++;
589 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
590 strcpy(current_rra->cf_nam, "DEVSEASONAL");
591 current_rra->row_cnt = period;
592 current_rra->par[RRA_seasonal_smooth_idx].u_cnt = hashed_name % period;
593 current_rra->pdp_cnt = 1;
594 current_rra->par[RRA_seasonal_gamma].u_val =
595 rrd->rra_def[hw_index].par[RRA_hw_alpha].u_val;
596 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index;
598 /* create the CF_DEVPREDICT RRA */
599 (rrd->stat_head->rra_cnt)++;
600 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
601 strcpy(current_rra->cf_nam, "DEVPREDICT");
602 current_rra->row_cnt = (rrd->rra_def[hw_index]).row_cnt;
603 current_rra->pdp_cnt = 1;
604 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
606 /* create the CF_FAILURES RRA */
607 (rrd->stat_head->rra_cnt)++;
608 current_rra = &(rrd->rra_def[rrd->stat_head->rra_cnt]);
609 strcpy(current_rra->cf_nam, "FAILURES");
610 current_rra->row_cnt = period;
611 current_rra->pdp_cnt = 1;
612 current_rra->par[RRA_delta_pos].u_val = 2.0;
613 current_rra->par[RRA_delta_neg].u_val = 2.0;
614 current_rra->par[RRA_failure_threshold].u_cnt = 7;
615 current_rra->par[RRA_window_len].u_cnt = 9;
616 current_rra->par[RRA_dependent_rra_idx].u_cnt = hw_index + 2; /* DEVSEASONAL */
620 /* create and empty rrd file according to the specs given */
623 const char *file_name,
628 rrd_value_t *unknown;
633 if ((rrd_file = fopen(file_name, "wb")) == NULL) {
634 rrd_set_error("creating '%s': %s", file_name, rrd_strerror(errno));
635 free(rrd->stat_head);
636 rrd->stat_head = NULL;
644 fwrite(rrd->stat_head, sizeof(stat_head_t), 1, rrd_file);
646 fwrite(rrd->ds_def, sizeof(ds_def_t), rrd->stat_head->ds_cnt, rrd_file);
649 sizeof(rra_def_t), rrd->stat_head->rra_cnt, rrd_file);
651 fwrite(rrd->live_head, sizeof(live_head_t), 1, rrd_file);
653 if ((rrd->pdp_prep = calloc(1, sizeof(pdp_prep_t))) == NULL) {
654 rrd_set_error("allocating pdp_prep");
660 strcpy(rrd->pdp_prep->last_ds, "UNKN");
662 rrd->pdp_prep->scratch[PDP_val].u_val = 0.0;
663 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt =
664 rrd->live_head->last_up % rrd->stat_head->pdp_step;
666 for (i = 0; i < rrd->stat_head->ds_cnt; i++)
667 fwrite(rrd->pdp_prep, sizeof(pdp_prep_t), 1, rrd_file);
669 if ((rrd->cdp_prep = calloc(1, sizeof(cdp_prep_t))) == NULL) {
670 rrd_set_error("allocating cdp_prep");
677 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
678 switch (cf_conv(rrd->rra_def[i].cf_nam)) {
681 init_hwpredict_cdp(rrd->cdp_prep);
685 init_seasonal_cdp(rrd->cdp_prep);
688 /* initialize violation history to 0 */
689 for (ii = 0; ii < MAX_CDP_PAR_EN; ii++) {
690 /* We can zero everything out, by setting u_val to the
691 * NULL address. Each array entry in scratch is 8 bytes
692 * (a double), but u_cnt only accessed 4 bytes (long) */
693 rrd->cdp_prep->scratch[ii].u_val = 0.0;
697 /* can not be zero because we don't know anything ... */
698 rrd->cdp_prep->scratch[CDP_val].u_val = DNAN;
699 /* startup missing pdp count */
700 rrd->cdp_prep->scratch[CDP_unkn_pdp_cnt].u_cnt =
701 ((rrd->live_head->last_up -
702 rrd->pdp_prep->scratch[PDP_unkn_sec_cnt].u_cnt)
703 % (rrd->stat_head->pdp_step
704 * rrd->rra_def[i].pdp_cnt)) / rrd->stat_head->pdp_step;
708 for (ii = 0; ii < rrd->stat_head->ds_cnt; ii++) {
709 fwrite(rrd->cdp_prep, sizeof(cdp_prep_t), 1, rrd_file);
713 /* now, we must make sure that the rest of the rrd
714 struct is properly initialized */
716 if ((rrd->rra_ptr = calloc(1, sizeof(rra_ptr_t))) == NULL) {
717 rrd_set_error("allocating rra_ptr");
723 /* changed this initialization to be consistent with
724 * rrd_restore. With the old value (0), the first update
725 * would occur for cur_row = 1 because rrd_update increments
726 * the pointer a priori. */
727 for (i = 0; i < rrd->stat_head->rra_cnt; i++) {
728 rrd->rra_ptr->cur_row = rrd->rra_def[i].row_cnt - 1;
729 fwrite(rrd->rra_ptr, sizeof(rra_ptr_t), 1, rrd_file);
731 rrd_head_size = ftell(rrd_file);
733 /* write the empty data area */
734 if ((unknown = (rrd_value_t *) malloc(512 * sizeof(rrd_value_t))) == NULL) {
735 rrd_set_error("allocating unknown");
740 for (i = 0; i < 512; ++i)
744 for (i = 0; i < rrd->stat_head->rra_cnt; i++)
745 unkn_cnt += rrd->stat_head->ds_cnt * rrd->rra_def[i].row_cnt;
747 while (unkn_cnt > 0) {
748 fwrite(unknown, sizeof(rrd_value_t), min(unkn_cnt, 512), rrd_file);
753 /* lets see if we had an error */
754 if (ferror(rrd_file)) {
755 rrd_set_error("a file error occurred while creating '%s'", file_name);
760 #ifdef HAVE_POSIX_FADVISE
761 /* this file is not going to be read again any time
762 soon, so we drop everything except the header portion from
763 the buffer cache. for this to work, we have to fdsync the file
764 first though. This will not be all that fast, but 'good' data
765 like other rrdfiles headers will stay in cache. Now this only works if creating
766 a single rrd file is not too large, but I assume this should not be the case
767 in general. Otherwhise we would have to sync and release while writing all
770 fdatasync(fileno(rrd_file));
772 posix_fadvise(fileno(rrd_file), rrd_head_size, 0,
773 POSIX_FADV_DONTNEED)) {
774 rrd_set_error("setting POSIX_FADV_DONTNEED on '%s': %s", file_name,
775 rrd_strerror(errno));