search for: blknum

...uot; +#include "isaligned.h" +#include "minmax.h" #define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL @@ -231,43 +233,68 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, uint32_t flags, int *err) { CLEANUP_FREE uint8_t *block = NULL; + uint64_t blknum, blkoffs; + int r; assert (!flags); - block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); - return -1; + if (!IS_ALIGNED (count | offset, blksize)) { + block = malloc (blksize); + if (block == NULL) { + *err = errno;...

Based on Rich's review of my v1 that touched only cache.c, I have now tried to bring all three filters with alignment rounding in line with one another. There is definitely room for future improvements once we teach nbdkit to let filters and plugins advertise block sizes, but I'm hoping to get NBD_CMD_CACHE implemented first. Eric Blake (2): blocksize: Process requests in linear order

...bove bit works with virtual disk offset in bytes. */ +static inline void +bitmap_set (const struct bitmap *bm, uint64_t offset, unsigned v) +{ + return bitmap_set_blk (bm, offset / bm->blksize, v); +} + +/* Iterate over blocks represented in the bitmap. */ +#define bitmap_for(bm, /* uint64_t */ blknum) \ + for (blknum = 0; blknum < (bm)->size * (8 / (bm)->bpb); ++blknum) + +#endif /* NBDKIT_BITMAP_H */ diff --git a/configure.ac b/configure.ac index 39a7e6d..a3e4457 100644 --- a/configure.ac +++ b/configure.ac @@ -779,6 +779,7 @@ AC_CONFIG_FILES([podwrapper.pl]...

...bove bit works with virtual disk offset in bytes. */ +static inline void +bitmap_set (const struct bitmap *bm, uint64_t offset, unsigned v) +{ + return bitmap_set_blk (bm, offset / bm->blksize, v); +} + +/* Iterate over blocks represented in the bitmap. */ +#define bitmap_for(bm, /* uint64_t */ blknum) \ + for (blknum = 0; blknum < (bm)->size * (8 / (bm)->bpb); ++blknum) + +#endif /* NBDKIT_BITMAP_H */ diff --git a/configure.ac b/configure.ac index 39a7e6d..a3e4457 100644 --- a/configure.ac +++ b/configure.ac @@ -779,6 +779,7 @@ AC_CONFIG_FILES([podwrapper.pl]...

This is exactly the same as v1: https://www.redhat.com/archives/libguestfs/2018-December/msg00004.html except that it now frees the bitmap on unload (which the old code did not - there was always a memory leak). Rich.

...bove bit works with virtual disk offset in bytes. */ +static inline void +bitmap_set (const struct bitmap *bm, uint64_t offset, unsigned v) +{ + return bitmap_set_blk (bm, offset / bm->blksize, v); +} + +/* Iterate over blocks represented in the bitmap. */ +#define bitmap_for(bm, /* uint64_t */ blknum) \ + for ((blknum) = 0; (blknum) < (bm)->size * (8 / (bm)->bpb); ++(blknum)) + +#endif /* NBDKIT_BITMAP_H */ diff --git a/common/bitmap/bitmap.c b/common/bitmap/bitmap.c new file mode 100644 index 0000000..fb5dbe7 --- /dev/null +++ b/common/bitmap/bitmap.c @@ -0,...

I have some patches I'm working on to fix the cache filter. However this is a prelude. It should be simply pure refactoring. All tests pass still. Rich.

v2: https://www.redhat.com/archives/libguestfs/2018-December/msg00039.html v2 -> v3: - Fix all the issues raised in Eric's review. - Precompute some numbers to make the calculations easier. - Calculations now use bitshifts and masks in preference to division and modulo. - Clear existing bits before setting (which fixes a bug in the cache filter). Rich.

...if (blk_set_size (size)) + pthread_mutex_lock (&lock); + r = blk_set_size (size); + pthread_mutex_unlock (&lock); + if (r == -1) return -1; return size; @@ -179,6 +189,7 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, while (count > 0) { uint64_t blknum, blkoffs, n; + int r; blknum = offset / BLKSIZE; /* block number */ blkoffs = offset % BLKSIZE; /* offset within the block */ @@ -186,7 +197,10 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, if (n > count) n = count; - if (blk_read (next_ops, nxd...

...ize); + if (blk_set_size (size)) + return -1; + return size; } @@ -200,6 +228,36 @@ cow_can_flush (struct nbdkit_next_ops *next_ops, void *nxdata, void *handle) return 1; } +/* Return true if the block is allocated. Consults the bitmap. */ +static bool +blk_is_allocated (uint64_t blknum) +{ + uint64_t bm_offset = blknum / 8; + uint64_t bm_bit = blknum % 8; + + if (bm_offset >= bm_size) { + nbdkit_debug ("blk_is_allocated: block number is out of range"); + return false; + } + + return bitmap[bm_offset] & (1 << bm_bit); +} + +/* Mark a block as allo...

This adds a cache filter, which works like the COW filter in reverse. For realistic use it needs a bit more work, especially to add limits on the size of the cache, a more sensible cache replacement policy, and perhaps some kind of background worker to write dirty blocks out. Rich.

This patch series enhances the cache filter in a few ways, primarily adding a "cache-on-read" feature (similar to qemu's copyonread); and adding the ability to limit the cache size and the antecedent of that which is having a method to reclaim cache blocks. As the cache is stored as a sparse temporary file, reclaiming cache blocks simply means punching holes in the temporary file.

.../ +extern void lru_init (void); + +/* Free the LRU. */ +extern void lru_free (void); + +/* Notify LRU that the virtual size has changed. */ +extern int lru_set_size (uint64_t new_size); + +/* Mark a block as recently accessed in the LRU structure. */ +extern void lru_set_recently_accessed (uint64_t blknum); + +/* Check if a block has been recently accessed. */ +extern bool lru_has_been_recently_accessed (uint64_t blknum); + +#endif /* NBDKIT_LRU_H */ diff --git a/filters/cache/blk.c b/filters/cache/blk.c index 4000276..b256446 100644 --- a/filters/cache/blk.c +++ b/filters/cache/blk.c @@ -53,6 +53,7...

.../ +extern void lru_init (void); + +/* Free the LRU. */ +extern void lru_free (void); + +/* Notify LRU that the virtual size has changed. */ +extern int lru_set_size (uint64_t new_size); + +/* Mark a block as recently accessed in the LRU structure. */ +extern void lru_set_recently_accessed (uint64_t blknum); + +/* Check if a block has been recently accessed. */ +extern bool lru_has_been_recently_accessed (uint64_t blknum); + +#endif /* NBDKIT_LRU_H */ diff --git a/filters/cache/blk.c b/filters/cache/blk.c index 4000276..b256446 100644 --- a/filters/cache/blk.c +++ b/filters/cache/blk.c @@ -53,6 +53,7...

...alloc: %m"); + return -1; + } } /* XXX This breaks up large read requests into smaller ones, which @@ -258,12 +261,14 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, { ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); - r = blk_read (next_ops, nxdata, blknum, block, err); + r = blk_read (next_ops, nxdata, blknum, + blkoffs || n < blksize ? block : buf, err); } if (r == -1) return -1; - memcpy (buf, &block[blkoffs], n); + if (blkoffs || n < blksize) + memcpy (buf, &block[blkoffs], n);...

Eric, you'll probably find the design "interesting" ... It does work, for me at least. Rich.

...prepare, .finalize = plugin_finalize, diff --git a/filters/cache/cache.c b/filters/cache/cache.c index 9473f2c..2ae6f01 100644 --- a/filters/cache/cache.c +++ b/filters/cache/cache.c @@ -292,6 +292,7 @@ blk_writethrough (struct nbdkit_next_ops *next_ops, void *nxdata, uint64_t blknum, const uint8_t *block) { off_t offset = blknum * BLKSIZE; + int r; nbdkit_debug ("cache: blk_writethrough block %" PRIu64 " (offset %" PRIu64 ")", @@ -302,8 +303,9 @@ blk_writethrough (struct nbdkit_next_ops *next_ops, void *nxdata, retur...

...k); r = blk_set_size (size); - pthread_mutex_unlock (&lock); if (r == -1) return -1; @@ -266,9 +265,10 @@ cache_pread (struct nbdkit_next_ops *next_ops, void *nxdata, if (n > count) n = count; - pthread_mutex_lock (&lock); - r = blk_read (next_ops, nxdata, blknum, block, err); - pthread_mutex_unlock (&lock); + { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + r = blk_read (next_ops, nxdata, blknum, block, err); + } if (r == -1) return -1; @@ -316,13 +316,12 @@ cache_pwrite (struct nbdkit_next_ops *next_ops, void *nxdata,...

...ize = MAX (4096, statvfs.f_bsize); + nbdkit_debug ("cache: block size: %u", blksize); + + bitmap_init (&bm, blksize, 2 /* bits per block */); + + lru_init (); + return 0; } @@ -143,9 +161,11 @@ int blk_read (struct nbdkit_next_ops *next_ops, void *nxdata, uint64_t blknum, uint8_t *block, int *err) { - off_t offset = blknum * BLKSIZE; + off_t offset = blknum * blksize; enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_CACHED); + reclaim (fd, &bm); + nbdkit_debug ("cache: blk_read block %" PRIu64 " (offset %" PRIu64...

v4: https://www.redhat.com/archives/libguestfs/2019-January/msg00032.html v5: - Now we set the block size at run time. I'd like to say that I was able to test this change, but unfortunately I couldn't find any easy way to create a filesystem on x86-64 with a block size > 4K. Ext4 doesn't support it at all, and XFS doesn't support block size > page size (and I