search for: have_posix_fadvise

...git a/builder/pxzcat-c.c b/builder/pxzcat-c.c index 0bbd296..dec9cc2 100644 --- a/builder/pxzcat-c.c +++ b/builder/pxzcat-c.c @@ -214,8 +214,10 @@ pxzcat (value filenamev, value outputfilev, unsigned nr_threads) unix_error (err, (char *) "ftruncate", outputfilev); } +#if defined HAVE_POSIX_FADVISE /* Tell the kernel we won't read the output file. */ ignore_value (posix_fadvise (fd, 0, 0, POSIX_FADV_RANDOM|POSIX_FADV_DONTNEED)); +#endif /* Iterate over blocks. */ iter_blocks (idx, nr_threads, filenamev, fd, outputfilev, ofd); -- 1.9.3

...git a/builder/pxzcat-c.c b/builder/pxzcat-c.c index 1f5ceeb..44722bc 100644 --- a/builder/pxzcat-c.c +++ b/builder/pxzcat-c.c @@ -214,10 +214,7 @@ pxzcat (value filenamev, value outputfilev, unsigned nr_threads) unix_error (err, (char *) "ftruncate", outputfilev); } -#if defined HAVE_POSIX_FADVISE - /* Tell the kernel we won't read the output file. */ - ignore_value (posix_fadvise (fd, 0, 0, POSIX_FADV_RANDOM|POSIX_FADV_DONTNEED)); -#endif + guestfs_int_fadvise_noreuse (fd); /* Iterate over blocks. */ iter_blocks (idx, nr_threads, filenamev, fd, outputfilev, ofd); diff --git a...

The way we used posix_fadvise was wrong, and yet right! Rich.

...iff --git a/plugins/file/file.c b/plugins/file/file.c index 076e7531..a8e37cd9 100644 --- a/plugins/file/file.c +++ b/plugins/file/file.c @@ -66,6 +66,18 @@ static char *filename = NULL; +/* posix_fadvise mode: -1 = don't set it, or POSIX_FADV_*. */ +static int fadvise_mode = +#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_NORMAL) + POSIX_FADV_NORMAL +#else + -1 +#endif + ; + +/* cache mode */ +static enum { cache_default, cache_none } cache_mode = cache_default; + /* Any callbacks using lseek must be protected by this lock. */ static pthread_mutex_t lseek_lock = PTHREAD_MUTEX_INI...

...ck; + else if (strcmp (value, "writethrough") == 0) + cache_mode = cache_writethrough; else { nbdkit_error ("unknown cache mode: %s", value); return -1; @@ -423,7 +423,7 @@ file_pread (void *handle, void *buf, uint32_t count, uint64_t offset, #ifdef HAVE_POSIX_FADVISE /* On Linux this will evict the pages we just read from the page cache. */ - if (cache_mode == cache_none) + if (cache_mode == cache_writethrough) posix_fadvise (h->fd, orig_offset, orig_count, POSIX_FADV_DONTNEED); #endif @@ -441,11 +441,11 @@ file_pwrite (void *handle, const void...

...PS_MAYBE_EXTENTS \ + FILEOPS_MAYBE_CACHE + +#ifdef SEEK_HOLE +#define FILEOPS_MAYBE_EXTENTS \ + .can_extents = fileops_can_extents, \ + .extents = fileops_extents, +#else +#define FILEOPS_MAYBE_EXTENTS /* nothing */ +#endif + +#ifdef HAVE_POSIX_FADVISE +#define FILEOPS_MAYBE_CACHE \ + .cache = fileops_cache, +#else +#define FILEOPS_MAYBE_CACHE /* nothing */ +#endif + +/* Don’t call these directly. */ +extern int64_t fileops_get_size (void *handle); +extern int fileops_can_trim (void *handle); +extern int fileops_c...

There's a lot of code in nbdkit-file-plugin which it would be nice to reuse elsewhere. One possible approach (as outlined here) is simply to move the file callbacks (like file.pread, file.pwrite, file.zero etc) to a new mini-library. They can then be consumed by other plugins fairly easily by doing: static void * foo_open (int readonly) { struct fileops *fops; int fd, flags; /*

Needs some work still, see in particular the commit message for patch 3. Rich.

...ate mode, then maybe we should rename cache=none. cache=advise? cache=dontneed? I can't think of a good name! > >@@ -355,6 +428,17 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, > > { > > struct handle *h = handle; > >+#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED) > >+ uint32_t orig_count = count; > >+ uint64_t orig_offset = offset; > >+ > >+ /* If cache=none we want to force pages we have just written to the > >+ * file to be flushed to disk so we can immediately evict them from &g...

--- src/inspect-apps.c | 13 ++++++++++++- src/inspect-fs-windows.c | 6 ++++++ src/journal.c | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/inspect-apps.c b/src/inspect-apps.c index 20cf00a..8fbae9c 100644 --- a/src/inspect-apps.c +++ b/src/inspect-apps.c @@ -35,11 +35,22 @@ #include <sys/endian.h> #endif -/* be32toh is usually a macro

...uot;writethrough") == 0) > + cache_mode = cache_writethrough; > else { > nbdkit_error ("unknown cache mode: %s", value); > return -1; > @@ -423,7 +423,7 @@ file_pread (void *handle, void *buf, uint32_t count, uint64_t offset, > > #ifdef HAVE_POSIX_FADVISE > /* On Linux this will evict the pages we just read from the page cache. */ > - if (cache_mode == cache_none) > + if (cache_mode == cache_writethrough) > posix_fadvise (h->fd, orig_offset, orig_count, POSIX_FADV_DONTNEED); > #endif > > @@ -441,11 +441,11 @@ fil...

...so we may add it as another mode later on. But in the meantime, cache=none is fairly nice while still avoiding O_DIRECT. > @@ -355,6 +428,17 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, > { > struct handle *h = handle; > > +#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED) > + uint32_t orig_count = count; > + uint64_t orig_offset = offset; > + > + /* If cache=none we want to force pages we have just written to the > + * file to be flushed to disk so we can immediately evict them from > + * the page ca...

...if you use the cache. How about advise=? I would keep cache semantics similar to qemu. > > >@@ -355,6 +428,17 @@ file_pwrite (void *handle, const void *buf, > uint32_t count, uint64_t offset, > > > { > > > struct handle *h = handle; > > >+#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED) > > >+ uint32_t orig_count = count; > > >+ uint64_t orig_offset = offset; > > >+ > > >+ /* If cache=none we want to force pages we have just written to the > > >+ * file to be flushed to disk so we can immed...

...et, flags, extents); } + +/* Initial state. */ +static int +file_init_sparse (void *handle) +{ + struct handle *h = handle; + + return h->init_sparse; +} + +static int +file_init_zero (void *handle) +{ + struct handle *h = handle; + + return h->init_zero; +} #endif /* SEEK_HOLE */ #if HAVE_POSIX_FADVISE @@ -668,6 +724,8 @@ static struct nbdkit_plugin plugin = { #ifdef SEEK_HOLE .can_extents = file_can_extents, .extents = file_extents, + .init_sparse = file_init_sparse, + .init_zero = file_init_zero, #endif #if HAVE_POSIX_FADVISE .cache = file...

...h->files[i].can_extents = true; +#else + h->files[i].can_extents = false; +#endif } h->size = offset; nbdkit_debug ("total size=%" PRIu64, h->size); @@ -319,6 +341,104 @@ split_cache (void *handle, uint32_t count, uint64_t offset, uint32_t flags) } #endif /* HAVE_POSIX_FADVISE */ +#ifdef SEEK_HOLE +static int64_t +do_extents (struct file *file, uint32_t count, uint64_t offset, + bool req_one, struct nbdkit_extents *extents) +{ + int64_t r = 0; + uint64_t end = offset + count; + + do { + off_t pos; + + pos = lseek (file->fd, offset, SEEK_DATA); +...

...r a modification (stolen from src/lib-storage/index/index-mail.c) ... | --- dovecot-2.2-modified/src/lib-fs/fs-posix.c 2012-10-23 20:27:31.348919455 +0200 | +++ dovecot-2.2/src/lib-fs/fs-posix.c 2012-10-23 20:26:39.435300269 +0200 | @@ -295,10 +295,12 @@ | return TRUE; | } | | +#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) | if (posix_fadvise(file->fd, 0, length, POSIX_FADV_WILLNEED) < 0) { | i_error("posix_fadvise(%s) failed: %m", _file->path); | return TRUE; | } | +#endif | return FALSE; | } ... the compilations runs to completion, and doveco...

...; > + cache_mode = cache_writethrough; > > else { > > nbdkit_error ("unknown cache mode: %s", value); > > return -1; > > @@ -423,7 +423,7 @@ file_pread (void *handle, void *buf, uint32_t count, uint64_t offset, > > > > #ifdef HAVE_POSIX_FADVISE > > /* On Linux this will evict the pages we just read from the page cache. */ > > - if (cache_mode == cache_none) > > + if (cache_mode == cache_writethrough) > > posix_fadvise (h->fd, orig_offset, orig_count, POSIX_FADV_DONTNEED); > > #endif > > &g...

I'm still working my way through the filters before this series will be complete, but this is enough of a start to at least get some feedback on the idea of implementing another NBD protocol extension. Eric Blake (9): server: Internal hooks for implementing NBD_CMD_CACHE plugins: Add .cache callback file, split: Implement .cache with posix_fadvise nbd: Implement NBD_CMD_CACHE

More posix_fadvise stuff, and document what Linux really does with these calls. Also fixes a nasty bug in virt-builder. Rich.

Since v1: - rework .can_cache to be tri-state, with default of no advertisement (ripple effect through other patches) - add a lot more patches in order to round out filter support And in the meantime, Rich pushed NBD_CMD_CACHE support into libnbd, so in theory we now have a way to test cache commands through the entire stack. Eric Blake (24): server: Internal hooks for implementing