Displaying 20 results from an estimated 50 matches for "encode_tail".
Did you mean:
decode_tail
2014 Jun 16
4
[PATCH 01/11] qspinlock: A simple generic 4-byte queue spinlock
...+ * Exactly fits one cacheline.
> + */
> +static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
> +
> +/*
> + * We must be able to distinguish between no-tail and the tail at 0:0,
> + * therefore increment the cpu number by one.
> + */
> +
> +static inline u32 encode_tail(int cpu, int idx)
> +{
> + u32 tail;
> +
> + tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
> + tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
Should there an
ASSSERT (idx < 4)
just in case we screw up somehow (I can't figure out how, but
that is partially why...
2014 Jun 16
4
[PATCH 01/11] qspinlock: A simple generic 4-byte queue spinlock
...+ * Exactly fits one cacheline.
> + */
> +static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
> +
> +/*
> + * We must be able to distinguish between no-tail and the tail at 0:0,
> + * therefore increment the cpu number by one.
> + */
> +
> +static inline u32 encode_tail(int cpu, int idx)
> +{
> + u32 tail;
> +
> + tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
> + tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
Should there an
ASSSERT (idx < 4)
just in case we screw up somehow (I can't figure out how, but
that is partially why...
2014 Jun 23
0
[PATCH 01/11] qspinlock: A simple generic 4-byte queue spinlock
...> > @@ -17,6 +17,7 @@
> > struct mcs_spinlock {
> > struct mcs_spinlock *next;
> > int locked; /* 1 if lock acquired */
> > + int count;
>
> This could use a comment.
like so?
int count; /* nesting count, see qspinlock.c */
> > +static inline u32 encode_tail(int cpu, int idx)
> > +{
> > + u32 tail;
> > +
> > + tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
> > + tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
>
> Should there an
>
> ASSSERT (idx < 4)
>
> just in case we screw up somehow...
2014 May 07
0
[PATCH v10 08/19] qspinlock: Make a new qnode structure to support virtualization
...irq, hardirq, nmi.
*
* Exactly fits one cacheline.
*/
-static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+static DEFINE_PER_CPU_ALIGNED(struct qnode, qnodes[4]);
/*
* We must be able to distinguish between no-tail and the tail at 0:0,
@@ -79,12 +88,12 @@ static inline u32 encode_tail(int cpu, int idx)
return tail;
}
-static inline struct mcs_spinlock *decode_tail(u32 tail)
+static inline struct qnode *decode_tail(u32 tail)
{
int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
- return per_cpu_ptr(&a...
2014 May 08
1
[PATCH v10 03/19] qspinlock: Add pending bit
...struct qspinlock *lock, u32 val)
>
> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>
> + if (trylock_pending(lock, &val))
> + return; /* Lock acquired */
> +
> node = this_cpu_ptr(&mcs_nodes[0]);
> idx = node->count++;
> tail = encode_tail(smp_processor_id(), idx);
> @@ -119,15 +196,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> node->next = NULL;
>
> /*
> + * we already touched the queueing cacheline; don't bother with pending
> + * stuff.
> + *
> * trylock || xchg(l...
2014 May 08
1
[PATCH v10 03/19] qspinlock: Add pending bit
...struct qspinlock *lock, u32 val)
>
> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>
> + if (trylock_pending(lock, &val))
> + return; /* Lock acquired */
> +
> node = this_cpu_ptr(&mcs_nodes[0]);
> idx = node->count++;
> tail = encode_tail(smp_processor_id(), idx);
> @@ -119,15 +196,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> node->next = NULL;
>
> /*
> + * we already touched the queueing cacheline; don't bother with pending
> + * stuff.
> + *
> * trylock || xchg(l...
2020 Jul 21
2
[PATCH v3 0/6] powerpc: queued spinlocks and rwlocks
...void __queued_spin_lock_slowpath_queue(struct qspinlock *lock)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 old, tail;
+ u32 val;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
node = this_cpu_ptr(&qnodes[0].mcs);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -559,7 +577,6 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*/
__this_cpu_dec(qnodes[0].mcs.count);
}
-EXPORT_SYMBOL(queued_spin_lock_slowpath);
/*
* Generate the paravirt code for queued_spin_unlock_slowpath().
2020 Jul 21
2
[PATCH v3 0/6] powerpc: queued spinlocks and rwlocks
...void __queued_spin_lock_slowpath_queue(struct qspinlock *lock)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 old, tail;
+ u32 val;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
node = this_cpu_ptr(&qnodes[0].mcs);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -559,7 +577,6 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*/
__this_cpu_dec(qnodes[0].mcs.count);
}
-EXPORT_SYMBOL(queued_spin_lock_slowpath);
/*
* Generate the paravirt code for queued_spin_unlock_slowpath().
2014 May 07
0
[PATCH v10 09/19] qspinlock: Prepare for unfair lock support
...*lock)
+static __always_inline int get_qlock(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
+ return 1;
}
/**
@@ -365,7 +368,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
tail = encode_tail(smp_processor_id(), idx);
node += idx;
- node->mcs.locked = 0;
+ node->qhead = 0;
node->mcs.next = NULL;
/*
@@ -391,7 +394,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
prev = decode_tail(old);
ACCESS_ONCE(prev->mcs.next) = (struct mcs_spinlock *)no...
2020 Jul 21
0
[PATCH v3 0/6] powerpc: queued spinlocks and rwlocks
...lock)
> +{
> + struct mcs_spinlock *prev, *next, *node;
> + u32 old, tail;
> + u32 val;
> + int idx;
> +
> + BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
> +
> node = this_cpu_ptr(&qnodes[0].mcs);
> idx = node->count++;
> tail = encode_tail(smp_processor_id(), idx);
> @@ -559,7 +577,6 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> */
> __this_cpu_dec(qnodes[0].mcs.count);
> }
> -EXPORT_SYMBOL(queued_spin_lock_slowpath);
>
> /*
> * Generate the paravirt code for queued_spin...
2014 Jun 15
0
[PATCH 01/11] qspinlock: A simple generic 4-byte queue spinlock
...* contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail...
2014 Mar 03
5
[PATCH v5 3/8] qspinlock, x86: Add x86 specific optimization for 2 contending tasks
...Below is the (rather messy) qspinlock slow path code (the only thing
that really differs between our versions.
I'll try and slot your version in tomorrow.
---
/*
* Exactly fills one cacheline on 64bit.
*/
static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
static inline u32 encode_tail(int cpu, int idx)
{
u32 code;
code = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
code |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
return code;
}
static inline struct mcs_spinlock *decode_tail(u32 code)
{
int cpu = (code >> _Q_TAIL_CPU_OFFSET) - 1;
int idx = (code >&...
2014 Mar 03
5
[PATCH v5 3/8] qspinlock, x86: Add x86 specific optimization for 2 contending tasks
...Below is the (rather messy) qspinlock slow path code (the only thing
that really differs between our versions.
I'll try and slot your version in tomorrow.
---
/*
* Exactly fills one cacheline on 64bit.
*/
static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
static inline u32 encode_tail(int cpu, int idx)
{
u32 code;
code = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
code |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
return code;
}
static inline struct mcs_spinlock *decode_tail(u32 code)
{
int cpu = (code >> _Q_TAIL_CPU_OFFSET) - 1;
int idx = (code >&...
2014 Jun 15
0
[PATCH 03/11] qspinlock: Add pending bit
...D_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qsp
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,...
2015 Mar 16
0
[PATCH 1/9] qspinlock: A simple generic 4-byte queue spinlock
...q, nmi.
+ *
+ * Exactly fits one 64-byte cacheline on a 64-bit architecture.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+#ifdef CONFIG_DEBUG_SPINLOCK
+ BUG_ON(idx > 3);
+#endif
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu...
2014 Apr 17
0
[PATCH v9 03/19] qspinlock: Add pending bit
...10,6 +180,9 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ if (trylock_pending(lock, &val))
+ return; /* Lock acquired */
+
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +192,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; try...
2014 Feb 28
5
[PATCH v5 3/8] qspinlock, x86: Add x86 specific optimization for 2 contending tasks
On Thu, Feb 27, 2014 at 03:42:19PM -0500, Waiman Long wrote:
> >>+ old = xchg(&qlock->lock_wait, _QSPINLOCK_WAITING|_QSPINLOCK_LOCKED);
> >>+
> >>+ if (old == 0) {
> >>+ /*
> >>+ * Got the lock, can clear the waiting bit now
> >>+ */
> >>+ smp_u8_store_release(&qlock->wait, 0);
> >
> >So we just did an
2014 Feb 28
5
[PATCH v5 3/8] qspinlock, x86: Add x86 specific optimization for 2 contending tasks
On Thu, Feb 27, 2014 at 03:42:19PM -0500, Waiman Long wrote:
> >>+ old = xchg(&qlock->lock_wait, _QSPINLOCK_WAITING|_QSPINLOCK_LOCKED);
> >>+
> >>+ if (old == 0) {
> >>+ /*
> >>+ * Got the lock, can clear the waiting bit now
> >>+ */
> >>+ smp_u8_store_release(&qlock->wait, 0);
> >
> >So we just did an
2014 May 07
0
[PATCH v10 03/19] qspinlock: Add pending bit
...10,6 +184,9 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ if (trylock_pending(lock, &val))
+ return; /* Lock acquired */
+
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +196,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; try...
2020 Jul 23
2
[PATCH v3 0/6] powerpc: queued spinlocks and rwlocks
...lock *prev, *next, *node;
>> + u32 old, tail;
>> + u32 val;
>> + int idx;
>> +
>> + BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>> +
>> node = this_cpu_ptr(&qnodes[0].mcs);
>> idx = node->count++;
>> tail = encode_tail(smp_processor_id(), idx);
>> @@ -559,7 +577,6 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>> */
>> __this_cpu_dec(qnodes[0].mcs.count);
>> }
>> -EXPORT_SYMBOL(queued_spin_lock_slowpath);
>>
>> /*
>> * Generate t...