search for: blk_set_size

...dex 7410f0d..9473f2c 100644 --- a/filters/cache/cache.c +++ b/filters/cache/cache.c @@ -72,7 +72,7 @@ static int fd = -1; static uint8_t *bitmap; /* Size of the bitmap in bytes. */ -static uint64_t bm_size; +static size_t bm_size; enum bm_entry { BLOCK_NOT_CACHED = 0, @@ -167,7 +167,14 @@ blk_set_size (uint64_t new_size) { uint8_t *new_bm; const size_t old_bm_size = bm_size; - size_t new_bm_size = DIV_ROUND_UP (new_size, BLKSIZE*8/2); + uint64_t new_bm_size_u64 = DIV_ROUND_UP (new_size, BLKSIZE*8/2); + size_t new_bm_size; + + if (new_bm_size_u64 > SIZE_MAX) { + nbdkit_error (&qu...

...free the bitmap. */ extern void blk_free (void); +/*---------------------------------------------------------------------- + * ** NOTE ** + * + * An exclusive lock must be held when you call any function below + * this line. + */ + /* Allocate or resize the cache file and bitmap. */ extern int blk_set_size (uint64_t new_size); diff --git a/filters/cache/cache.c b/filters/cache/cache.c index 0d006bc..580e4f5 100644 --- a/filters/cache/cache.c +++ b/filters/cache/cache.c @@ -46,6 +46,8 @@ #include <sys/ioctl.h> #include <assert.h> +#include <pthread.h> + #ifdef HAVE_ALLOCA_H #...

...hin bounds; and even if the plugin tolerates the past-EOF read, we aren't guaranteeing that the client will always read 0 bytes in the past-EOF tail. Several ideas of fixing it, each with some drawbacks: + in cache/cow_get_size(), truncate the plugin's size down to blksize prior to calling blk_set_size() (renders the plugin's tail unusable) + reject serving images that aren't already aligned to blksize (avoids missing bytes or worrying about past-EOF slop, but can be mean, unless...) + document that for unaligned images, you can use --filter=cache --filter=truncate round-up=BLKSIZE, to le...

...{ int64_t size; int r; @@ -201,6 +204,7 @@ cache_get_size (struct nbdkit_next_ops *next_ops, void *nxdata, return -1; nbdkit_debug ("cache: underlying file size: %" PRIi64, size); + size = ROUND_DOWN (size, blksize); ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); r = blk_set_size (size); diff --git a/filters/cow/cow.c b/filters/cow/cow.c index 1ce5893..aa1348b 100644 --- a/filters/cow/cow.c +++ b/filters/cow/cow.c @@ -49,6 +49,7 @@ #include "blk.h" #include "isaligned.h" #include "minmax.h" +#include "rounding.h" #define THREAD_M...

...filters/cache/cache.c +++ b/filters/cache/cache.c @@ -209,9 +209,8 @@ cache_get_size (struct nbdkit_next_ops *next_ops, void *nxdata, nbdkit_debug ("cache: underlying file size: %" PRIi64, size); - pthread_mutex_lock (&lock); + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); r = blk_set_size (size); - pthread_mutex_unlock (&lock); if (r == -1) return -1; @@ -266,9 +265,10 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, if (n > count) n = count; - pthread_mutex_lock (&lock); - r = blk_read (next_ops, nxdata, blknum, block, err); -...

When using a filter that rounds up to alignment boundaries, the tail bytes of the plugin are difficult to access correctly. Rather than duplicating lots of code already in the truncate filter, it's easier to just make the other filters default to rounding down and add doc links on how to round up instead. Eric Blake (2): blocksize: Lift restriction against 0-size file cache, cow: Round

...exit (EXIT_FAILURE); - } - - nbdkit_debug ("cow: filesystem block size: %d", blksize); } static void @@ -147,6 +147,34 @@ cow_open (nbdkit_next_open *next, void *nxdata, int readonly) return &handle; } +/* Allocate or resize the overlay file and bitmap. */ +static int +blk_set_size (uint64_t new_size) +{ + uint8_t *new_bm; + const size_t old_bm_size = bm_size; + size_t new_bm_size = DIV_ROUND_UP (new_size, BLKSIZE*8); + + new_bm = realloc (bitmap, new_bm_size); + if (new_bm == NULL) { + nbdkit_error ("realloc: %m"); + return -1; + } + bitmap = new_bm; +...

v1 was here: https://www.redhat.com/archives/libguestfs/2019-January/msg00008.html In v2 I have provided two patches: The first patch extends attribute((nonnull)) to most internal functions, but not to the external API. The second patch uses a macro so that attribute((format)) is only used in the public API on GCC or Clang. At least in theory these headers could be used by a C compiler which

...gt; https://gcc.gnu.org/bugzilla/show_bug.cgi?id=17308 > --- > +++ b/filters/cache/blk.h This file is not on the current nbdkit.git master; I'm assuming the patch depends on some of your other patches landing first. > @@ -51,10 +51,15 @@ extern void blk_free (void); > extern int blk_set_size (uint64_t new_size); > > /* Read a single block from the cache or plugin. */ > -extern int blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, uint64_t blknum, uint8_t *block, int *err); > +extern int blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, > +...

...@ cache_load (void) size_t len; char *template; + bitmap_init (&bm, BLKSIZE, 2 /* bits per block */); + tmpdir = getenv ("TMPDIR"); if (!tmpdir) tmpdir = LARGE_TMPDIR; @@ -180,28 +177,8 @@ cache_open (nbdkit_next_open *next, void *nxdata, int readonly) static int blk_set_size (uint64_t new_size) { - uint8_t *new_bm; - const size_t old_bm_size = bm_size; - uint64_t new_bm_size_u64 = DIV_ROUND_UP (new_size, BLKSIZE*8/2); - size_t new_bm_size; - - if (new_bm_size_u64 > SIZE_MAX) { - nbdkit_error ("bitmap too large for this architecture"); + if (bitma...

...ir = getenv ("TMPDIR"); if (!tmpdir) tmpdir = LARGE_TMPDIR; @@ -133,6 +130,8 @@ cache_unload (void) { if (fd >= 0) close (fd); + + bitmap_free (&bm); } static int @@ -180,28 +179,8 @@ cache_open (nbdkit_next_open *next, void *nxdata, int readonly) static int blk_set_size (uint64_t new_size) { - uint8_t *new_bm; - const size_t old_bm_size = bm_size; - uint64_t new_bm_size_u64 = DIV_ROUND_UP (new_size, BLKSIZE*8/2); - size_t new_bm_size; - - if (new_bm_size_u64 > SIZE_MAX) { - nbdkit_error ("bitmap too large for this architecture"); + if (bitma...

...ir = getenv ("TMPDIR"); if (!tmpdir) tmpdir = LARGE_TMPDIR; @@ -133,6 +130,8 @@ cache_unload (void) { if (fd >= 0) close (fd); + + bitmap_free (&bm); } static int @@ -180,28 +179,8 @@ cache_open (nbdkit_next_open *next, void *nxdata, int readonly) static int blk_set_size (uint64_t new_size) { - uint8_t *new_bm; - const size_t old_bm_size = bm_size; - uint64_t new_bm_size_u64 = DIV_ROUND_UP (new_size, BLKSIZE*8/2); - size_t new_bm_size; - - if (new_bm_size_u64 > SIZE_MAX) { - nbdkit_error ("bitmap too large for this architecture"); + if (bitma...

I have some patches I'm working on to fix the cache filter. However this is a prelude. It should be simply pure refactoring. All tests pass still. Rich.

I do have a question about whether patch 2 is right, or whether I've exposed a bigger problem in the truncate (and possibly other) filter, but the rest seem fairly straightforward. Eric Blake (4): server: Check for pthread lock failures truncate: Factor out reading real_size under mutex plugins: Check for mutex failures filters: Check for mutex failures filters/cache/cache.c

...uint32_t count, uint64_t offset) + __attribute__((__nonnull__ (1))); #endif /* NBDKIT_SPARSE_H */ diff --git a/filters/cache/blk.h b/filters/cache/blk.h index ab9134e..2a65bdb 100644 --- a/filters/cache/blk.h +++ b/filters/cache/blk.h @@ -51,10 +51,15 @@ extern void blk_free (void); extern int blk_set_size (uint64_t new_size); /* Read a single block from the cache or plugin. */ -extern int blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, uint64_t blknum, uint8_t *block, int *err); +extern int blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, + uint64_t blknum,...

This patch series enhances the cache filter in a few ways, primarily adding a "cache-on-read" feature (similar to qemu's copyonread); and adding the ability to limit the cache size and the antecedent of that which is having a method to reclaim cache blocks. As the cache is stored as a sparse temporary file, reclaiming cache blocks simply means punching holes in the temporary file.

...lk_init (void) size_t len; char *template; + lru_init (); + bitmap_init (&bm, BLKSIZE, 2 /* bits per block */); tmpdir = getenv ("TMPDIR"); @@ -115,6 +118,8 @@ blk_free (void) close (fd); bitmap_free (&bm); + + lru_free (); } int @@ -128,6 +133,9 @@ blk_set_size (uint64_t new_size) return -1; } + if (lru_set_size (new_size) == -1) + return -1; + return 0; } @@ -163,6 +171,7 @@ blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, return -1; } bitmap_set_blk (&bm, blknum, BLOCK_CLEAN); + lru_set_recent...

...lk_init (void) size_t len; char *template; + lru_init (); + bitmap_init (&bm, BLKSIZE, 2 /* bits per block */); tmpdir = getenv ("TMPDIR"); @@ -115,6 +118,8 @@ blk_free (void) close (fd); bitmap_free (&bm); + + lru_free (); } int @@ -128,6 +133,9 @@ blk_set_size (uint64_t new_size) return -1; } + if (lru_set_size (new_size) == -1) + return -1; + return 0; } @@ -163,6 +171,7 @@ blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, return -1; } bitmap_set_blk (&bm, blknum, BLOCK_CLEAN); + lru_set_recent...

This adds a cache filter, which works like the COW filter in reverse. For realistic use it needs a bit more work, especially to add limits on the size of the cache, a more sensible cache replacement policy, and perhaps some kind of background worker to write dirty blocks out. Rich.

v2: https://www.redhat.com/archives/libguestfs/2018-December/msg00039.html v2 -> v3: - Fix all the issues raised in Eric's review. - Precompute some numbers to make the calculations easier. - Calculations now use bitshifts and masks in preference to division and modulo. - Clear existing bits before setting (which fixes a bug in the cache filter). Rich.