1 /*****************************************************************************
2 * RRDtool 1.2.23 Copyright by Tobi Oetiker, 1997-2007
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
8 * Revision 1.10 2004/05/26 22:11:12 oetiker
9 * reduce compiler warnings. Many small fixes. -- Mike Slifcak <slif@bellsouth.net>
11 * Revision 1.9 2003/04/29 21:56:49 oetiker
12 * readline in rrd_open.c reads the file in 8 KB blocks, and calls realloc for
13 * each block. realloc is very slow in Mac OS X for huge blocks, e.g. when
14 * restoring databases from huge xml files. This patch finds the size of the
15 * file, and starts out with malloc'ing the full size.
16 * -- Peter Speck <speck@ruc.dk>
18 * Revision 1.8 2003/04/11 19:43:44 oetiker
19 * New special value COUNT which allows calculations based on the position of a
20 * value within a data set. Bug fix in rrd_rpncalc.c. PREV returned erroneus
21 * value for the second value. Bug fix in rrd_restore.c. Bug causing seek error
22 * when accesing an RRD restored from an xml that holds an RRD version <3.
23 * -- Ruben Justo <ruben@ainek.com>
25 * Revision 1.7 2003/03/31 21:22:12 oetiker
26 * enables RRDtool updates with microsecond or in case of windows millisecond
27 * precision. This is needed to reduce time measurement error when archive step
28 * is small. (<30s) -- Sasha Mikheev <sasha@avalon-net.co.il>
30 * Revision 1.6 2003/02/13 07:05:27 oetiker
31 * Find attached the patch I promised to send to you. Please note that there
32 * are three new source files (src/rrd_is_thread_safe.h, src/rrd_thread_safe.c
33 * and src/rrd_not_thread_safe.c) and the introduction of librrd_th. This
34 * library is identical to librrd, but it contains support code for per-thread
35 * global variables currently used for error information only. This is similar
36 * to how errno per-thread variables are implemented. librrd_th must be linked
37 * alongside of libpthred
39 * There is also a new file "THREADS", holding some documentation.
41 * -- Peter Stamfest <peter@stamfest.at>
43 * Revision 1.5 2002/06/20 00:21:03 jake
44 * More Win32 build changes; thanks to Kerry Calvert.
46 * Revision 1.4 2002/02/01 20:34:49 oetiker
47 * fixed version number and date/time
49 * Revision 1.3 2001/03/04 13:01:55 oetiker
50 * Aberrant Behavior Detection support. A brief overview added to rrdtool.pod.
51 * Major updates to rrd_update.c, rrd_create.c. Minor update to other core files.
52 * This is backwards compatible! But new files using the Aberrant stuff are not readable
53 * by old rrdtool versions. See http://cricket.sourceforge.net/aberrant/rrd_hw.htm
54 * -- Jake Brutlag <jakeb@corp.webtv.net>
56 * Revision 1.2 2001/03/04 10:29:20 oetiker
57 * fixed filedescriptor leak
58 * -- Mike Franusich <mike@franusich.com>
60 * Revision 1.1.1.1 2001/02/25 22:25:05 oetiker
63 *****************************************************************************/
69 /* DEBUG 2 prints information obtained via mincore(2) */
71 /* do not calculate exact madvise hints but assume 1 page for headers and
72 * set DONTNEED for the rest, which is assumed to be data */
74 /* Avoid calling madvise on areas that were already hinted. May be benefical if
75 * your syscalls are very slow */
76 //#define CHECK_MADVISE_OVERLAPS 1
79 #define __rrd_read(dst, dst_t, cnt) \
80 (dst) = (dst_t*) (data + offset); \
81 offset += sizeof(dst_t) * (cnt)
83 #define __rrd_read(dst, dst_t, cnt) \
84 if ((dst = malloc(sizeof(dst_t)*(cnt))) == NULL) { \
85 rrd_set_error(#dst " malloc"); \
86 goto out_nullify_head; \
88 offset += read (rrd_file->fd, dst, sizeof(dst_t)*(cnt))
91 /* next page-aligned (i.e. page-align up) */
93 #define PAGE_ALIGN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
95 /* previous page-aligned (i.e. page-align down) */
96 #ifndef PAGE_ALIGN_DOWN
97 #define PAGE_ALIGN_DOWN(addr) (((addr)+_page_size-1)&(~(_page_size-1)))
101 /* vector of last madvise hint */
102 typedef struct _madvise_vec_t {
106 _madvise_vec_t _madv_vec = { NULL, 0 };
109 #if defined CHECK_MADVISE_OVERLAPS
110 #define _madvise(_start, _off, _hint) \
111 if ((_start) != _madv_vec.start && (ssize_t)(_off) != _madv_vec.length) { \
112 _madv_vec.start = (_start) ; _madv_vec.length = (_off); \
113 madvise((_start), (_off), (_hint)); \
116 #define _madvise(_start, _off, _hint) \
117 madvise((_start), (_off), (_hint))
120 /* Open a database file, return its header and an open filehandle,
121 * positioned to the first cdp in the first rra.
122 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
123 * before returning an error. Do not call rrd_close upon failure of rrd_open.
126 rrd_file_t *rrd_open(
127 const char *const file_name,
132 mode_t mode = S_IRUSR;
136 ssize_t _page_size = sysconf(_SC_PAGESIZE);
137 int mm_prot = PROT_READ, mm_flags = 0;
142 rrd_file_t *rrd_file = NULL;
145 rrd_file = malloc(sizeof(rrd_file_t));
146 if (rrd_file == NULL) {
147 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name);
150 memset(rrd_file, 0, sizeof(rrd_file_t));
153 if ((rdwr & (RRD_READONLY | RRD_READWRITE)) ==
154 (RRD_READONLY | RRD_READWRITE)) {
155 /* Both READONLY and READWRITE were given, which is invalid. */
156 rrd_set_error("in read/write request mask");
160 if (rdwr & RRD_READONLY) {
163 mm_flags = MAP_PRIVATE;
164 # ifdef MAP_NORESERVE
165 mm_flags |= MAP_NORESERVE; /* readonly, so no swap backing needed */
169 if (rdwr & RRD_READWRITE) {
173 mm_flags = MAP_SHARED;
174 mm_prot |= PROT_WRITE;
177 if (rdwr & RRD_CREAT) {
178 flags |= (O_CREAT | O_TRUNC);
181 if (rdwr & RRD_READAHEAD) {
183 mm_flags |= MAP_POPULATE; /* populate ptes and data */
185 #if defined MAP_NONBLOCK
186 mm_flags |= MAP_NONBLOCK; /* just populate ptes */
197 if ((rrd_file->fd = open(file_name, flags, mode)) < 0) {
198 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
202 /* Better try to avoid seeks as much as possible. stat may be heavy but
203 * many concurrent seeks are even worse. */
204 if ((fstat(rrd_file->fd, &statb)) < 0) {
205 rrd_set_error("fstat '%s': %s", file_name, rrd_strerror(errno));
208 rrd_file->file_len = statb.st_size;
210 #ifdef HAVE_POSIX_FADVISE
211 /* In general we need no read-ahead when dealing with rrd_files.
212 When we stop reading, it is highly unlikely that we start up again.
213 In this manner we actually save time and diskaccess (and buffer cache).
214 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
215 if (0 != posix_fadvise(rrd_file->fd, 0, 0, POSIX_FADV_RANDOM)) {
216 rrd_set_error("setting POSIX_FADV_RANDOM on '%s': %s", file_name,
217 rrd_strerror(errno));
223 if (rdwr & RRD_READWRITE)
225 if (setvbuf((rrd_file->fd),NULL,_IONBF,2)) {
226 rrd_set_error("failed to disable the stream buffer\n");
232 data = mmap(0, rrd_file->file_len, mm_prot, mm_flags,
233 rrd_file->fd, offset);
235 /* lets see if the first read worked */
236 if (data == MAP_FAILED) {
237 rrd_set_error("error mmaping file '%s': %s", file_name,
238 rrd_strerror(errno));
241 rrd_file->file_start = data;
244 if (rdwr & RRD_COPY) {
245 /* We will read everything in a moment (copying) */
246 _madvise(data, rrd_file->file_len, MADV_WILLNEED | MADV_SEQUENTIAL);
249 /* We do not need to read anything in for the moment */
251 _madvise(data, rrd_file->file_len, MADV_DONTNEED);
252 // _madvise(data, rrd_file->file_len, MADV_RANDOM);
254 /* alternatively: keep 2 pages worth of data, likely headers,
255 * don't need the rest. */
256 _madvise(data, _page_size, MADV_WILLNEED | MADV_SEQUENTIAL);
257 _madvise(data + _page_size, (rrd_file->file_len >= _page_size)
258 ? rrd_file->file_len - _page_size : 0, MADV_DONTNEED);
262 #if defined USE_MADVISE && !defined ONE_PAGE
263 /* the stat_head will be needed soonish, so hint accordingly */
264 // too finegrained to calc the individual sizes, just keep 2 pages worth of hdr
265 _madvise(data + PAGE_ALIGN_DOWN(offset), PAGE_ALIGN(sizeof(stat_head_t)),
270 __rrd_read(rrd->stat_head, stat_head_t,
273 /* lets do some test if we are on track ... */
274 if (memcmp(rrd->stat_head->cookie, RRD_COOKIE, sizeof(RRD_COOKIE)) != 0) {
275 rrd_set_error("'%s' is not an RRD file", file_name);
276 goto out_nullify_head;
279 if (rrd->stat_head->float_cookie != FLOAT_COOKIE) {
280 rrd_set_error("This RRD was created on other architecture");
281 goto out_nullify_head;
284 version = atoi(rrd->stat_head->version);
286 if (version > atoi(RRD_VERSION)) {
287 rrd_set_error("can't handle RRD file version %s",
288 rrd->stat_head->version);
289 goto out_nullify_head;
291 #if defined USE_MADVISE && !defined ONE_PAGE
292 /* the ds_def will be needed soonish, so hint accordingly */
293 _madvise(data + PAGE_ALIGN_DOWN(offset),
294 PAGE_ALIGN(sizeof(ds_def_t) * rrd->stat_head->ds_cnt),
297 __rrd_read(rrd->ds_def, ds_def_t,
298 rrd->stat_head->ds_cnt);
300 #if defined USE_MADVISE && !defined ONE_PAGE
301 /* the rra_def will be needed soonish, so hint accordingly */
302 _madvise(data + PAGE_ALIGN_DOWN(offset),
303 PAGE_ALIGN(sizeof(rra_def_t) * rrd->stat_head->rra_cnt),
306 __rrd_read(rrd->rra_def, rra_def_t,
307 rrd->stat_head->rra_cnt);
309 /* handle different format for the live_head */
311 rrd->live_head = (live_head_t *) malloc(sizeof(live_head_t));
312 if (rrd->live_head == NULL) {
313 rrd_set_error("live_head_t malloc");
317 memmove(&rrd->live_head->last_up, data + offset, sizeof(long));
318 offset += sizeof(long);
320 offset += read(rrd_file->fd, &rrd->live_head->last_up, sizeof(long));
322 rrd->live_head->last_up_usec = 0;
324 #if defined USE_MADVISE && !defined ONE_PAGE
325 /* the live_head will be needed soonish, so hint accordingly */
326 _madvise(data + PAGE_ALIGN_DOWN(offset),
327 PAGE_ALIGN(sizeof(live_head_t)), MADV_WILLNEED);
329 __rrd_read(rrd->live_head, live_head_t,
332 //XXX: This doesn't look like it needs madvise
333 __rrd_read(rrd->pdp_prep, pdp_prep_t,
334 rrd->stat_head->ds_cnt);
336 //XXX: This could benefit from madvise()ing
337 __rrd_read(rrd->cdp_prep, cdp_prep_t,
338 rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt);
340 //XXX: This could benefit from madvise()ing
341 __rrd_read(rrd->rra_ptr, rra_ptr_t,
342 rrd->stat_head->rra_cnt);
347 rrd_file->header_len = offset;
348 rrd_file->pos = offset;
352 rrd->stat_head = NULL;
359 /* Close a reference to an rrd_file. */
362 rrd_file_t *rrd_file)
366 #if defined HAVE_MMAP || defined DEBUG
367 ssize_t _page_size = sysconf(_SC_PAGESIZE);
369 #if defined DEBUG && DEBUG > 1
370 /* pretty print blocks in core */
374 off = rrd_file->file_len +
375 ((rrd_file->file_len + _page_size - 1) / _page_size);
379 if (mincore(rrd_file->file_start, rrd_file->file_len, vec) == 0) {
381 unsigned is_in = 0, was_in = 0;
383 for (off = 0, prev = 0; off < rrd_file->file_len; ++off) {
384 is_in = vec[off] & 1; /* if lsb set then is core resident */
387 if (was_in != is_in) {
388 fprintf(stderr, "%sin core: %p len %ld\n",
389 was_in ? "" : "not ", vec + prev, off - prev);
395 "%sin core: %p len %ld\n",
396 was_in ? "" : "not ", vec + prev, off - prev);
398 fprintf(stderr, "mincore: %s", rrd_strerror(errno));
404 /* Keep headers around, round up to next page boundary. */
406 PAGE_ALIGN(rrd_file->header_len % _page_size + rrd_file->header_len);
407 if (rrd_file->file_len > ret)
408 _madvise(rrd_file->file_start + ret,
409 rrd_file->file_len - ret, MADV_DONTNEED);
411 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
412 _madvise(rrd_file->file_start + PAGE_ALIGN_DOWN(rrd_file->header_len),
413 rrd_file->file_len - PAGE_ALIGN(rrd_file->header_len),
418 ret = munmap(rrd_file->file_start, rrd_file->file_len);
420 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno));
422 ret = close(rrd_file->fd);
424 rrd_set_error("closing file: %s", rrd_strerror(errno));
431 /* Set position of rrd_file. */
434 rrd_file_t *rrd_file,
441 if (whence == SEEK_SET)
443 else if (whence == SEEK_CUR)
444 rrd_file->pos += off;
445 else if (whence == SEEK_END)
446 rrd_file->pos = rrd_file->file_len + off;
448 ret = lseek(rrd_file->fd, off, whence);
450 rrd_set_error("lseek: %s", rrd_strerror(errno));
453 //XXX: mimic fseek, which returns 0 upon success
454 return ret == -1; //XXX: or just ret to mimic lseek
458 /* Get current position in rrd_file. */
460 inline off_t rrd_tell(rrd_file_t *rrd_file)
462 return rrd_file->pos;
466 /* read count bytes into buffer buf, starting at rrd_file->pos.
467 * Returns the number of bytes read. */
470 rrd_file_t *rrd_file,
475 buf = memmove(buf, rrd_file->file_start + rrd_file->pos, count);
476 rrd_file->pos += count; /* mimmic read() semantics */
481 ret = read(rrd_file->fd, buf, count);
482 //XXX: eventually add generic rrd_set_error(""); here
483 rrd_file->pos += count; /* mimmic read() semantics */
489 /* write count bytes from buffer buf to the current position
490 * rrd_file->pos of rrd_file->fd.
491 * Returns the number of bytes written. */
494 rrd_file_t *rrd_file,
499 memmove(rrd_file->file_start + rrd_file->pos, buf, count);
500 return count; /* mimmic write() semantics */
502 return write(rrd_file->fd, buf, count);
507 /* flush all data pending to be written to FD. */
509 inline void rrd_flush(
510 rrd_file_t *rrd_file)
512 if (fdatasync(rrd_file->fd) != 0) {
513 rrd_set_error("flushing fd %d: %s", rrd_file->fd,
514 rrd_strerror(errno));
519 /* Initialize RRD header. */
524 rrd->stat_head = NULL;
527 rrd->live_head = NULL;
529 rrd->pdp_prep = NULL;
530 rrd->cdp_prep = NULL;
531 rrd->rrd_value = NULL;
535 /* free RRD header data. */
538 inline void rrd_free(
539 rrd_t UNUSED(*rrd)) {}
544 if (atoi(rrd->stat_head->version) < 3)
545 free(rrd->live_head);
546 free(rrd->stat_head);
552 free(rrd->rrd_value);
557 /* routine used by external libraries to free memory allocated by
567 /* XXX: FIXME: missing documentation. */
568 /*XXX: FIXME should be renamed to rrd_readfile or _rrd_readfile */
570 int /*_rrd_*/ readfile(
571 const char *file_name,
575 long writecnt = 0, totalcnt = MEMBLK;
580 if ((strcmp("-", file_name) == 0)) {
583 if ((input = fopen(file_name, "rb")) == NULL) {
584 rrd_set_error("opening '%s': %s", file_name, rrd_strerror(errno));
592 } while (c != '\n' && !feof(input));
594 if (strcmp("-", file_name)) {
595 fseek(input, 0, SEEK_END);
596 /* have extra space for detecting EOF without realloc */
597 totalcnt = (ftell(input) + 1) / sizeof(char) - offset;
598 if (totalcnt < MEMBLK)
599 totalcnt = MEMBLK; /* sanitize */
600 fseek(input, offset * sizeof(char), SEEK_SET);
602 if (((*buffer) = (char *) malloc((totalcnt + 4) * sizeof(char))) == NULL) {
603 perror("Allocate Buffer:");
608 fread((*buffer) + writecnt, 1,
609 (totalcnt - writecnt) * sizeof(char), input);
610 if (writecnt >= totalcnt) {
613 rrd_realloc((*buffer),
614 (totalcnt + 4) * sizeof(char))) == NULL) {
615 perror("Realloc Buffer:");
619 } while (!feof(input));
620 (*buffer)[writecnt] = '\0';
621 if (strcmp("-", file_name) != 0) {