search for: ____cacheline_aligned_in_smp

...For sleeping on allocation failure */ > > + wait_queue_head_t wait; > > + > > + /* > > + * Global freelist - it's a stack where nr_free points to the > > + * top > > + */ > > + unsigned nr_free; > > + unsigned *freelist; > > + } ____cacheline_aligned_in_smp; > > Why the ____cacheline_aligned_in_smp? It's separating the RW stuff that isn't always touched from the RO stuff that's used on every allocation. > > > +}; > > > > ... > > > > + > > +/* Percpu IDA */ > > + > > +/* > &gt...

...t; +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <pthread.h> +#include <malloc.h> +#include <assert.h> +#include <errno.h> +#include <limits.h> + +#define SMP_CACHE_BYTES 64 +#define cache_line_size() SMP_CACHE_BYTES +#define ____cacheline_aligned_in_smp __attribute__ ((aligned (SMP_CACHE_BYTES))) +#define unlikely(x) (__builtin_expect(!!(x), 0)) +#define ALIGN(x, a) (((x) + (a) - 1) / (a) * (a)) +typedef pthread_spinlock_t spinlock_t; + +typedef int gfp_t; +static void *kzalloc(unsigned size, gfp_t gfp) +{ + void *p = memalign(64, size); + if...

...t; +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <pthread.h> +#include <malloc.h> +#include <assert.h> +#include <errno.h> +#include <limits.h> + +#define SMP_CACHE_BYTES 64 +#define cache_line_size() SMP_CACHE_BYTES +#define ____cacheline_aligned_in_smp __attribute__ ((aligned (SMP_CACHE_BYTES))) +#define unlikely(x) (__builtin_expect(!!(x), 0)) +#define ALIGN(x, a) (((x) + (a) - 1) / (a) * (a)) +typedef pthread_spinlock_t spinlock_t; + +typedef int gfp_t; +static void *kzalloc(unsigned size, gfp_t gfp) +{ + void *p = memalign(64, size); + if...

...tions(-) > > > > diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c > > index 93468b7c6701..6f7f277495f4 100644 > > --- a/drivers/block/virtio_blk.c > > +++ b/drivers/block/virtio_blk.c > > @@ -33,6 +33,16 @@ struct virtio_blk_vq { > > } ____cacheline_aligned_in_smp; > > > > struct virtio_blk { > > + /* > > + * vdev may be accessed without taking this mutex in blk-mq and > > + * virtqueue code paths because virtblk_remove() stops them before vdev > > + * is freed. > > + * > > + * Everything else must hold...

...tions(-) > > > > diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c > > index 93468b7c6701..6f7f277495f4 100644 > > --- a/drivers/block/virtio_blk.c > > +++ b/drivers/block/virtio_blk.c > > @@ -33,6 +33,16 @@ struct virtio_blk_vq { > > } ____cacheline_aligned_in_smp; > > > > struct virtio_blk { > > + /* > > + * vdev may be accessed without taking this mutex in blk-mq and > > + * virtqueue code paths because virtblk_remove() stops them before vdev > > + * is freed. > > + * > > + * Everything else must hold...

...unsigned cpu_last_stolen; > + > + /* For sleeping on allocation failure */ > + wait_queue_head_t wait; > + > + /* > + * Global freelist - it's a stack where nr_free points to the > + * top > + */ > + unsigned nr_free; > + unsigned *freelist; > + } ____cacheline_aligned_in_smp; Why the ____cacheline_aligned_in_smp? > +}; > > ... > > + > +/* Percpu IDA */ > + > +/* > + * Number of tags we move between the percpu freelist and the global freelist at > + * a time "between a percpu freelist" would be more accurate? > + */ > +...

...unsigned cpu_last_stolen; > + > + /* For sleeping on allocation failure */ > + wait_queue_head_t wait; > + > + /* > + * Global freelist - it's a stack where nr_free points to the > + * top > + */ > + unsigned nr_free; > + unsigned *freelist; > + } ____cacheline_aligned_in_smp; Why the ____cacheline_aligned_in_smp? > +}; > > ... > > + > +/* Percpu IDA */ > + > +/* > + * Number of tags we move between the percpu freelist and the global freelist at > + * a time "between a percpu freelist" would be more accurate? > + */ > +...

...++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 93468b7c6701..9d21bf0f155e 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -33,6 +33,15 @@ struct virtio_blk_vq { } ____cacheline_aligned_in_smp; struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to...

...++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 93468b7c6701..9d21bf0f155e 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -33,6 +33,15 @@ struct virtio_blk_vq { } ____cacheline_aligned_in_smp; struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to...

...a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -25,6 +25,7 @@ #include <scsi/scsi_cmnd.h> #define VIRTIO_SCSI_MEMPOOL_SZ 64 +#define VIRTIO_SCSI_EVENT_LEN 8 /* Command queue element */ struct virtio_scsi_cmd { @@ -43,6 +44,12 @@ struct virtio_scsi_cmd { } resp; } ____cacheline_aligned_in_smp; +struct virtio_scsi_event_node { + struct virtio_scsi *vscsi; + struct virtio_scsi_event event; + struct work_struct work; +}; + struct virtio_scsi_vq { /* Protects vq */ spinlock_t vq_lock; @@ -67,6 +74,9 @@ struct virtio_scsi { struct virtio_scsi_vq event_vq; struct virtio_scsi_vq req...

...a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -25,6 +25,7 @@ #include <scsi/scsi_cmnd.h> #define VIRTIO_SCSI_MEMPOOL_SZ 64 +#define VIRTIO_SCSI_EVENT_LEN 8 /* Command queue element */ struct virtio_scsi_cmd { @@ -43,6 +44,12 @@ struct virtio_scsi_cmd { } resp; } ____cacheline_aligned_in_smp; +struct virtio_scsi_event_node { + struct virtio_scsi *vscsi; + struct virtio_scsi_event event; + struct work_struct work; +}; + struct virtio_scsi_vq { /* Protects vq */ spinlock_t vq_lock; @@ -67,6 +74,9 @@ struct virtio_scsi { struct virtio_scsi_vq event_vq; struct virtio_scsi_vq req...

...++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 93468b7c6701..6f7f277495f4 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -33,6 +33,16 @@ struct virtio_blk_vq { } ____cacheline_aligned_in_smp; struct virtio_blk { + /* + * vdev may be accessed without taking this mutex in blk-mq and + * virtqueue code paths because virtblk_remove() stops them before vdev + * is freed. + * + * Everything else must hold this mutex when accessing vdev and must + * handle the case where vdev is NULL...

...++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 93468b7c6701..6f7f277495f4 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -33,6 +33,16 @@ struct virtio_blk_vq { } ____cacheline_aligned_in_smp; struct virtio_blk { + /* + * vdev may be accessed without taking this mutex in blk-mq and + * virtqueue code paths because virtblk_remove() stops them before vdev + * is freed. + * + * Everything else must hold this mutex when accessing vdev and must + * handle the case where vdev is NULL...

...a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -25,6 +25,7 @@ #include <scsi/scsi_cmnd.h> #define VIRTIO_SCSI_MEMPOOL_SZ 64 +#define VIRTIO_SCSI_EVENT_LEN 8 /* Command queue element */ struct virtio_scsi_cmd { @@ -43,6 +44,12 @@ struct virtio_scsi_cmd { } resp; } ____cacheline_aligned_in_smp; +struct virtio_scsi_event_node { + struct virtio_scsi *vscsi; + struct virtio_scsi_event event; + struct work_struct work; +}; + struct virtio_scsi_vq { /* Protects vq */ spinlock_t vq_lock; @@ -67,6 +74,9 @@ struct virtio_scsi { struct virtio_scsi_vq event_vq; struct virtio_scsi_vq req...

...a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -25,6 +25,7 @@ #include <scsi/scsi_cmnd.h> #define VIRTIO_SCSI_MEMPOOL_SZ 64 +#define VIRTIO_SCSI_EVENT_LEN 8 /* Command queue element */ struct virtio_scsi_cmd { @@ -43,6 +44,12 @@ struct virtio_scsi_cmd { } resp; } ____cacheline_aligned_in_smp; +struct virtio_scsi_event_node { + struct virtio_scsi *vscsi; + struct virtio_scsi_event event; + struct work_struct work; +}; + struct virtio_scsi_vq { /* Protects vq */ spinlock_t vq_lock; @@ -67,6 +74,9 @@ struct virtio_scsi { struct virtio_scsi_vq event_vq; struct virtio_scsi_vq req...

...sm-x86_64/pda.h =================================================================== --- clean-start.orig/include/asm-x86_64/pda.h +++ clean-start/include/asm-x86_64/pda.h @@ -29,6 +29,7 @@ struct x8664_pda { short isidle; struct mm_struct *active_mm; unsigned apic_timer_irqs; + void *vcpu; } ____cacheline_aligned_in_smp; extern struct x8664_pda *_cpu_pda[]; --

...sm-x86_64/pda.h =================================================================== --- clean-start.orig/include/asm-x86_64/pda.h +++ clean-start/include/asm-x86_64/pda.h @@ -29,6 +29,7 @@ struct x8664_pda { short isidle; struct mm_struct *active_mm; unsigned apic_timer_irqs; + void *vcpu; } ____cacheline_aligned_in_smp; extern struct x8664_pda *_cpu_pda[]; --

...to avoid > cacheline sharing between two unrelated VQs. > Want to try? It's still false sharing because the queue objects share cachelines. To operate without contention they have to be physically separated from each other like so: struct vq { struct virtqueue *q; spinlock_t lock; } ____cacheline_aligned_in_smp; struct some_other_struct { .... struct vq vq[MAX_NUM_VQ]; .... }; This keeps locality to objects within a queue, but separates each queue onto it's own cacheline.... Cheers, Dave. -- Dave Chinner david at fromorbit.com

...k.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 7164da8..c5730d3 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -28,8 +28,7 @@ struct virtio_blk_vq { char name[VQ_NAME_LEN]; } ____cacheline_aligned_in_smp; -struct virtio_blk -{ +struct virtio_blk { struct virtio_device *vdev; /* The disk structure for the kernel. */ @@ -52,8 +51,7 @@ struct virtio_blk struct virtio_blk_vq *vqs; }; -struct virtblk_req -{ +struct virtblk_req { struct request *req; struct virtio_blk_outhdr out_hdr; s...

...k.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 7164da8..c5730d3 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -28,8 +28,7 @@ struct virtio_blk_vq { char name[VQ_NAME_LEN]; } ____cacheline_aligned_in_smp; -struct virtio_blk -{ +struct virtio_blk { struct virtio_device *vdev; /* The disk structure for the kernel. */ @@ -52,8 +51,7 @@ struct virtio_blk struct virtio_blk_vq *vqs; }; -struct virtblk_req -{ +struct virtblk_req { struct request *req; struct virtio_blk_outhdr out_hdr; s...