1 /*****************************************************************************
2 * RRDtool 1.0.33 Copyright Tobias Oetiker, 1997, 1998, 1999
3 *****************************************************************************
4 * rrd_format.h RRD Database Format header
5 *****************************************************************************/
12 /*****************************************************************************
13 * put this in your /usr/lib/magic file (/etc/magic on HPUX)
15 * # rrd database format
16 * 0 string RRD\0 rrd file
17 * >5 string >\0 version '%s'
19 *****************************************************************************/
21 #define RRD_COOKIE "RRD"
22 #define RRD_VERSION "0002"
23 #define FLOAT_COOKIE 8.642135E130
26 #define DNAN ((double)fmod(0.0,0.0))
27 #define DINF ((double)log(0.0))
30 #define DNAN ((double)(0.0/0.0)) /* we use a DNAN to
31 * represent the UNKNOWN
33 #define DINF ((double)(1.0/0.0)) /* we use a DINF to
34 * represent a value at the upper or
35 * lower border of the graph ...
39 typedef union unival {
45 /****************************************************************************
46 * The RRD Database Structure
47 * ---------------------------
49 * In oder to properly describe the database structure lets define a few
52 * ds - Data Source (ds) providing input to the database. A Data Source (ds)
53 * can be a traffic counter, a temperature, the number of users logged
54 * into a system. The rrd database format can handle the input of
55 * several Data Sources (ds) in a singe database.
57 * dst - Data Source Type (dst). The Data Source Type (dst) defines the rules
58 * applied to Build Primary Data Points from the input provided by the
61 * pdp - Primary Data Point (pdp). After the database has accepted the
62 * input from the data sources (ds). It starts building Primary
63 * Data Points (pdp) from the data. Primary Data Points (pdp)
64 * are evenly spaced along the time axis (pdp_step). The values
65 * of the Primary Data Points are calculated from the values of
66 * the data source (ds) and the exact time these values were
67 * provided by the data source (ds).
69 * pdp_st - PDP Start (pdp_st). The moments (pdp_st) in time where
70 * these steps occur are defined by the moments where the
71 * number of seconds since 1970-jan-1 modulo pdp_step equals
74 * cf - Consolidation Function (cf). An arbitrary Consolidation Function (cf)
75 * (averaging, min, max) is applied to the primary data points (pdp) to
76 * calculate the consolidated data point.
78 * cdp - Consolidated Data Point (cdp) is the long term storage format for data
79 * in the rrd database. Consolidated Data Points represent one or
80 * several primary data points collected along the time axis. The
81 * Consolidated Data Points (cdp) are stored in Round Robin Archives
84 * rra - Round Robin Archive (rra). This is the place where the
85 * consolidated data points (cdp) get stored. The data is
86 * organized in rows (row) and columns (col). The Round Robin
87 * Archive got its name from the method data is stored in
88 * there. An RRD database can contain several Round Robin
89 * Archives. Each Round Robin Archive can have a different row
90 * spacing along the time axis (pdp_cnt) and a different
91 * consolidation function (cf) used to build its consolidated
94 * rra_st - RRA Start (rra_st). The moments (rra_st) in time where
95 * Consolidated Data Points (cdp) are added to an rra are
96 * defined by the moments where the number of seconds since
97 * 1970-jan-1 modulo pdp_cnt*pdp_step equals zero (rra_st).
99 * row - Row (row). A row represent all consolidated data points (cdp)
100 * in a round robin archive who are of the same age.
102 * col - Column (col). A column (col) represent all consolidated
103 * data points (cdp) in a round robin archive (rra) who
104 * originated from the same data source (ds).
108 /****************************************************************************
109 * POS 1: stat_head_t static header of the database
110 ****************************************************************************/
112 typedef struct stat_head_t {
114 /* Data Base Identification Section ***/
115 char cookie[4]; /* RRD */
116 char version[5]; /* version of the format */
117 double float_cookie; /* is it the correct double
118 * representation ? */
120 /* Data Base Structure Definition *****/
121 unsigned long ds_cnt; /* how many different ds provide
122 * input to the rrd */
123 unsigned long rra_cnt; /* how many rras will be maintained
125 unsigned long pdp_step; /* pdp interval in seconds */
127 unival par[10]; /* global parameters ... unused
132 /****************************************************************************
133 * POS 2: ds_def_t (* ds_cnt) Data Source definitions
134 ****************************************************************************/
136 enum dst_en { DST_COUNTER=0, /* data source types available */
142 enum ds_param_en { DS_mrhb_cnt=0, /* minimum required heartbeat. A
143 * data source must provide input at
144 * least every ds_mrhb seconds,
145 * otherwise it is regarded dead and
146 * will be set to UNKNOWN */
147 DS_min_val, /* the processed input of a ds must */
148 DS_max_val, /* be between max_val and min_val
149 * both can be set to UNKNOWN if you
150 * do not care. Data outside the limits
152 DS_cdef = DS_mrhb_cnt}; /* pointer to encoded rpn
153 * expression only applies to DST_CDEF */
155 /* The magic number here is one less than DS_NAM_SIZE */
156 #define DS_NAM_FMT "%19[a-zA-Z0-9_-]"
157 #define DS_NAM_SIZE 20
159 #define DST_FMT "%19[A-Z]"
162 typedef struct ds_def_t {
163 char ds_nam[DS_NAM_SIZE]; /* Name of the data source (null terminated)*/
164 char dst[DST_SIZE]; /* Type of data source (null terminated)*/
165 unival par[10]; /* index of this array see ds_param_en */
168 /****************************************************************************
169 * POS 3: rra_def_t ( * rra_cnt) one for each store to be maintained
170 ****************************************************************************/
171 enum cf_en { CF_AVERAGE=0, /* data consolidation functions */
176 /* An array of predictions using the seasonal
177 * Holt-Winters algorithm. Requires an RRA of type
178 * CF_SEASONAL for this data source. */
180 /* An array of seasonal effects. Requires an RRA of
181 * type CF_HWPREDICT for this data source. */
183 /* An array of deviation predictions based upon
184 * smoothed seasonal deviations. Requires an RRA of
185 * type CF_DEVSEASONAL for this data source. */
187 /* An array of smoothed seasonal deviations. Requires
188 * an RRA of type CF_HWPREDICT for this data source.
191 /* A binary array of failure indicators: 1 indicates
192 * that the number of violations in the prescribed
193 * window exceeded the prescribed threshold. */
195 #define MAX_RRA_PAR_EN 10
196 enum rra_par_en { RRA_cdp_xff_val=0, /* what part of the consolidated
197 * datapoint must be known, to produce a
198 * valid entry in the rra */
200 /* exponential smoothing parameter for the intercept in
201 * the Holt-Winters prediction algorithm. */
203 /* exponential smoothing parameter for the slope in
204 * the Holt-Winters prediction algorithm. */
205 RRA_dependent_rra_idx,
206 /* For CF_HWPREDICT: index of the RRA with the seasonal
207 * effects of the Holt-Winters algorithm (of type
209 * For CF_DEVPREDICT: index of the RRA with the seasonal
210 * deviation predictions (of type CF_DEVSEASONAL).
211 * For CF_SEASONAL: index of the RRA with the Holt-Winters
212 * intercept and slope coefficient (of type CF_HWPREDICT).
213 * For CF_DEVSEASONAL: index of the RRA with the
214 * Holt-Winters prediction (of type CF_HWPREDICT).
215 * For CF_FAILURES: index of the CF_DEVSEASONAL array.
217 RRA_seasonal_smooth_idx,
218 /* For CF_SEASONAL and CF_DEVSEASONAL:
219 * an integer between 0 and row_count - 1 which
220 * is index in the seasonal cycle for applying
221 * the period smoother. */
222 RRA_failure_threshold,
223 /* For CF_FAILURES, number of violations within the last
224 * window required to mark a failure. */
225 RRA_seasonal_gamma = RRA_hw_alpha,
226 /* exponential smoothing parameter for seasonal effects.
228 RRA_delta_pos = RRA_hw_alpha,
229 RRA_delta_neg = RRA_hw_beta,
230 /* confidence bound scaling parameters for the
231 * the FAILURES RRA. */
232 RRA_window_len = RRA_seasonal_smooth_idx};
233 /* For CF_FAILURES, the length of the window for measuring
236 #define CF_NAM_FMT "%19[A-Z]"
237 #define CF_NAM_SIZE 20
239 typedef struct rra_def_t {
240 char cf_nam[CF_NAM_SIZE];/* consolidation function (null term) */
241 unsigned long row_cnt; /* number of entries in the store */
242 unsigned long pdp_cnt; /* how many primary data points are
243 * required for a consolidated data
245 unival par[MAX_RRA_PAR_EN]; /* index see rra_param_en */
250 /****************************************************************************
251 ****************************************************************************
252 ****************************************************************************
253 * LIVE PART OF THE HEADER. THIS WILL BE WRITTEN ON EVERY UPDATE *
254 ****************************************************************************
255 ****************************************************************************
256 ****************************************************************************/
257 /****************************************************************************
259 ****************************************************************************/
261 typedef struct live_head_t {
262 time_t last_up; /* when was rrd last updated */
266 /****************************************************************************
267 * POS 5: pdp_prep_t (* ds_cnt) here we prepare the pdps
268 ****************************************************************************/
269 #define LAST_DS_LEN 30 /* DO NOT CHANGE THIS ... */
271 enum pdp_par_en { PDP_unkn_sec_cnt=0, /* how many seconds of the current
272 * pdp value is unknown data? */
274 PDP_val}; /* current value of the pdp.
275 this depends on dst */
277 typedef struct pdp_prep_t{
278 char last_ds[LAST_DS_LEN]; /* the last reading from the data
279 * source. this is stored in ASCII
280 * to cater for very large counters
281 * we might encounter in connection
283 unival scratch[10]; /* contents according to pdp_par_en */
286 /* data is passed from pdp to cdp when seconds since epoch modulo pdp_step == 0
287 obviously the updates do not occur at these times only. Especially does the
288 format allow for updates to occur at different times for each data source.
289 The rules which makes this work is as follows:
291 * DS updates may only occur at ever increasing points in time
292 * When any DS update arrives after a cdp update time, the *previous*
293 update cycle gets executed. All pdps are transfered to cdps and the
294 cdps feed the rras where necessary. Only then the new DS value
295 is loaded into the PDP. */
298 /****************************************************************************
299 * POS 6: cdp_prep_t (* rra_cnt * ds_cnt ) data prep area for cdp values
300 ****************************************************************************/
301 #define MAX_CDP_PAR_EN 10
302 #define MAX_CDP_FAILURES_IDX 8
303 /* max CDP scratch entries avail to record violations for a FAILURES RRA */
304 #define MAX_FAILURES_WINDOW_LEN 28
305 enum cdp_par_en { CDP_val=0,
306 /* the base_interval is always an
309 /* how many unknown pdp were
310 * integrated. This and the cdp_xff
311 * will decide if this is going to
312 * be a UNKNOWN or a valid value */
314 /* Current intercept coefficient for the Holt-Winters
315 * prediction algorithm. */
316 CDP_hw_last_intercept,
317 /* Last iteration intercept coefficient for the Holt-Winters
318 * prediction algorihtm. */
320 /* Current slope coefficient for the Holt-Winters
321 * prediction algorithm. */
323 /* Last iteration slope coeffient. */
325 /* Number of sequential Unknown (DNAN) values + 1 preceding
326 * the current prediction.
329 /* Last iteration count of Unknown (DNAN) values. */
331 /* optimization for bulk updates: the value of the first CDP
332 * value to be written in the bulk update. */
333 CDP_secondary_val = 9,
334 /* optimization for bulk updates: the value of subsequent
335 * CDP values to be written in the bulk update. */
336 CDP_hw_seasonal = CDP_hw_intercept,
337 /* Current seasonal coefficient for the Holt-Winters
338 * prediction algorithm. This is stored in CDP prep to avoid
339 * redundant seek operations. */
340 CDP_hw_last_seasonal = CDP_hw_last_intercept,
341 /* Last iteration seasonal coeffient. */
342 CDP_seasonal_deviation = CDP_hw_intercept,
343 CDP_last_seasonal_deviation = CDP_hw_last_intercept,
344 CDP_init_seasonal = CDP_null_count};
345 /* init_seasonal is a flag which when > 0, forces smoothing updates
346 * to occur when rra_ptr.cur_row == 0 */
348 typedef struct cdp_prep_t{
349 unival scratch[MAX_CDP_PAR_EN];
350 /* contents according to cdp_par_en *
351 * init state should be NAN */
355 /****************************************************************************
356 * POS 7: rra_ptr_t (* rra_cnt) pointers to the current row in each rra
357 ****************************************************************************/
359 typedef struct rra_ptr_t {
360 unsigned long cur_row; /* current row in the rra*/
364 /****************************************************************************
365 ****************************************************************************
366 * One single struct to hold all the others. For convenience.
367 ****************************************************************************
368 ****************************************************************************/
369 typedef struct rrd_t {
370 stat_head_t *stat_head; /* the static header */
371 ds_def_t *ds_def; /* list of data source definitions */
372 rra_def_t *rra_def; /* list of round robin archive def */
373 live_head_t *live_head;
374 pdp_prep_t *pdp_prep; /* pdp data prep area */
375 cdp_prep_t *cdp_prep; /* cdp prep area */
376 rra_ptr_t *rra_ptr; /* list of rra pointers */
377 rrd_value_t *rrd_value; /* list of rrd values */
380 /****************************************************************************
381 ****************************************************************************
382 * AFTER the header section we have the DATA STORAGE AREA it is made up from
383 * Consolidated Data Points organized in Round Robin Archives.
384 ****************************************************************************
385 ****************************************************************************
388 (0,0) .................... ( ds_cnt -1 , 0)
392 (0, row_cnt -1) ... (ds_cnt -1, row_cnt -1)
399 ****************************************************************************/