Kieran Mansley
2007-Oct-03 13:34 UTC
[Xen-devel] [PATCH 1/3] TLB flushing and IO memory mapping
Signed-off-by: Kieran Mansley <kmansley@solarflare.com>
Fix TLB flush on grant unmap
diff -r 6f6e364ed0fa xen/common/grant_table.c
--- a/xen/common/grant_table.c Wed Jul 25 09:54:34 2007 +0100
+++ b/xen/common/grant_table.c Wed Jul 25 13:59:10 2007 +0100
@@ -60,13 +60,25 @@ union grant_combo {
/* Used to share code between unmap_grant_ref and unmap_and_replace. */
struct gnttab_unmap_common {
+ /* Input */
uint64_t host_addr;
uint64_t dev_bus_addr;
uint64_t new_addr;
grant_handle_t handle;
+ /* Return */
int16_t status;
+
+ /* Shared state beteen *_unmap and *_unmap_complete */
+ u16 flags;
+ unsigned long frame;
+ struct grant_mapping *map;
+ struct domain *rd;
};
+
+/* Number of unmap operations that are done between each tlb flush */
+#define GNTTAB_UNMAP_BATCH_SIZE 32
+
#define PIN_FAIL(_lbl, _rc, _f, _a...) \
do { \
@@ -411,18 +423,14 @@ __gnttab_unmap_common(
struct gnttab_unmap_common *op)
{
domid_t dom;
- grant_ref_t ref;
struct domain *ld, *rd;
struct active_grant_entry *act;
grant_entry_t *sha;
- struct grant_mapping *map;
- u16 flags;
s16 rc = 0;
- unsigned long frame;
ld = current->domain;
- frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
+ op->frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) )
{
@@ -431,20 +439,19 @@ __gnttab_unmap_common(
return;
}
- map = &maptrack_entry(ld->grant_table, op->handle);
-
- if ( unlikely(!map->flags) )
+ op->map = &maptrack_entry(ld->grant_table, op->handle);
+
+ if ( unlikely(!op->map->flags) )
{
gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n",
op->handle);
op->status = GNTST_bad_handle;
return;
}
- dom = map->domid;
- ref = map->ref;
- flags = map->flags;
-
- if ( unlikely((rd = rcu_lock_domain_by_id(dom)) == NULL) )
+ dom = op->map->domid;
+ op->flags = op->map->flags;
+
+ if ( unlikely((op->rd = rd = rcu_lock_domain_by_id(dom)) == NULL) )
{
/* This can happen when a grant is implicitly unmapped. */
gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
@@ -456,71 +463,47 @@ __gnttab_unmap_common(
spin_lock(&rd->grant_table->lock);
- act = &active_entry(rd->grant_table, ref);
- sha = &shared_entry(rd->grant_table, ref);
-
- if ( frame == 0 )
- {
- frame = act->frame;
+ act = &active_entry(rd->grant_table, op->map->ref);
+ sha = &shared_entry(rd->grant_table, op->map->ref);
+
+ if ( op->frame == 0 )
+ {
+ op->frame = act->frame;
}
else
{
- if ( unlikely(frame != act->frame) )
+ if ( unlikely(op->frame != act->frame) )
PIN_FAIL(unmap_out, GNTST_general_error,
"Bad frame number doesn''t match
gntref.\n");
- if ( flags & GNTMAP_device_map )
+ if ( op->flags & GNTMAP_device_map )
{
ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
- map->flags &= ~GNTMAP_device_map;
- if ( flags & GNTMAP_readonly )
- {
+ op->map->flags &= ~GNTMAP_device_map;
+ if ( op->flags & GNTMAP_readonly )
act->pin -= GNTPIN_devr_inc;
- put_page(mfn_to_page(frame));
- }
else
- {
act->pin -= GNTPIN_devw_inc;
- put_page_and_type(mfn_to_page(frame));
- }
- }
- }
-
- if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
+ }
+ }
+
+ if ( (op->host_addr != 0) && (op->flags &
GNTMAP_host_map) )
{
if ( (rc = replace_grant_host_mapping(op->host_addr,
- frame, op->new_addr, flags))
< 0 )
+ op->frame, op->new_addr,
+ op->flags)) < 0 )
goto unmap_out;
ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
- map->flags &= ~GNTMAP_host_map;
- if ( flags & GNTMAP_readonly )
- {
+ op->map->flags &= ~GNTMAP_host_map;
+ if ( op->flags & GNTMAP_readonly )
act->pin -= GNTPIN_hstr_inc;
- put_page(mfn_to_page(frame));
- }
else
- {
act->pin -= GNTPIN_hstw_inc;
- put_page_and_type(mfn_to_page(frame));
- }
- }
-
- if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
- {
- map->flags = 0;
- put_maptrack_handle(ld->grant_table, op->handle);
}
/* If just unmapped a writable mapping, mark as dirtied */
- if ( !(flags & GNTMAP_readonly) )
- gnttab_mark_dirty(rd, frame);
-
- if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0)
&&
- !(flags & GNTMAP_readonly) )
- gnttab_clear_flag(_GTF_writing, &sha->flags);
-
- if ( act->pin == 0 )
- gnttab_clear_flag(_GTF_reading, &sha->flags);
+ if ( !(op->flags & GNTMAP_readonly) )
+ gnttab_mark_dirty(rd, op->frame);
unmap_out:
op->status = rc;
@@ -529,78 +512,205 @@ __gnttab_unmap_common(
}
static void
+__gnttab_unmap_common_complete(struct gnttab_unmap_common *op)
+{
+ struct domain *ld, *rd;
+ struct active_grant_entry *act;
+ grant_entry_t *sha;
+
+ rd = op->rd;
+
+ if ( rd == NULL ) {
+ /*
+ * Suggests that __gntab_unmap_common failed in
+ * rcu_lock_domain_by_id() or earlier, and so we have nothing
+ * to complete
+ */
+ return;
+ }
+
+ ld = current->domain;
+
+ rcu_lock_domain(rd);
+ spin_lock(&rd->grant_table->lock);
+
+ act = &active_entry(rd->grant_table, op->map->ref);
+ sha = &shared_entry(rd->grant_table, op->map->ref);
+
+ if ( unlikely(op->frame != act->frame) )
+ {
+ /*
+ * Suggests that __gntab_unmap_common failed early and so
+ * nothing further to do
+ */
+ goto unmap_out;
+ }
+
+ if ( op->flags & GNTMAP_device_map )
+ {
+ if ( op->flags & GNTMAP_readonly )
+ put_page(mfn_to_page(op->frame));
+ else
+ put_page_and_type(mfn_to_page(op->frame));
+ }
+
+ if ( (op->host_addr != 0) && (op->flags &
GNTMAP_host_map) )
+ {
+ if ( op->status != 0 )
+ {
+ /*
+ * Suggests that __gntab_unmap_common failed in
+ * replace_grant_host_mapping() so nothing further to do
+ */
+ goto unmap_out;
+ }
+
+ if ( op->flags & GNTMAP_readonly )
+ put_page(mfn_to_page(op->frame));
+ else
+ put_page_and_type(mfn_to_page(op->frame));
+ }
+
+ if ( (op->map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0
)
+ {
+ op->map->flags = 0;
+ put_maptrack_handle(ld->grant_table, op->handle);
+ }
+
+ if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0)
&&
+ !(op->flags & GNTMAP_readonly) )
+ gnttab_clear_flag(_GTF_writing, &sha->flags);
+
+ if ( act->pin == 0 )
+ gnttab_clear_flag(_GTF_reading, &sha->flags);
+
+ unmap_out:
+ spin_unlock(&rd->grant_table->lock);
+ rcu_unlock_domain(rd);
+}
+
+static void
__gnttab_unmap_grant_ref(
- struct gnttab_unmap_grant_ref *op)
-{
- struct gnttab_unmap_common common = {
- .host_addr = op->host_addr,
- .dev_bus_addr = op->dev_bus_addr,
- .handle = op->handle,
- };
-
- __gnttab_unmap_common(&common);
- op->status = common.status;
-}
+ struct gnttab_unmap_grant_ref *op,
+ struct gnttab_unmap_common *common)
+{
+ common->host_addr = op->host_addr;
+ common->dev_bus_addr = op->dev_bus_addr;
+ common->handle = op->handle;
+
+ /* Intialise these in case common contains old state */
+ common->new_addr = 0;
+ common->rd = NULL;
+
+ __gnttab_unmap_common(common);
+ op->status = common->status;
+}
+
static long
gnttab_unmap_grant_ref(
XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
{
- int i;
+ int i, c, partial_done, done = 0;
struct gnttab_unmap_grant_ref op;
-
- for ( i = 0; i < count; i++ )
- {
- if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
- goto fault;
- __gnttab_unmap_grant_ref(&op);
- if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
- goto fault;
- }
-
- flush_tlb_mask(current->domain->domain_dirty_cpumask);
+ struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
+
+ while (count != 0) {
+ c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
+ partial_done = 0;
+
+ for ( i = 0; i < c; i++ )
+ {
+ if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
+ goto fault;
+ __gnttab_unmap_grant_ref(&op, &(common[i]));
+ ++partial_done;
+ if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
+ goto fault;
+ }
+
+ flush_tlb_mask(current->domain->domain_dirty_cpumask);
+
+ for ( i = 0; i < partial_done; i++ )
+ {
+ __gnttab_unmap_common_complete(&(common[i]));
+ }
+
+ count -= c;
+ done += c;
+ }
+
return 0;
fault:
flush_tlb_mask(current->domain->domain_dirty_cpumask);
- return -EFAULT;
+
+ for ( i = 0; i < partial_done; i++ )
+ {
+ __gnttab_unmap_common_complete(&(common[i]));
+ }
+ return -EFAULT;
}
static void
__gnttab_unmap_and_replace(
- struct gnttab_unmap_and_replace *op)
-{
- struct gnttab_unmap_common common = {
- .host_addr = op->host_addr,
- .new_addr = op->new_addr,
- .handle = op->handle,
- };
-
- __gnttab_unmap_common(&common);
- op->status = common.status;
+ struct gnttab_unmap_and_replace *op,
+ struct gnttab_unmap_common *common)
+{
+ common->host_addr = op->host_addr;
+ common->new_addr = op->new_addr;
+ common->handle = op->handle;
+
+ /* Intialise these in case common contains old state */
+ common->dev_bus_addr = 0;
+ common->rd = NULL;
+
+ __gnttab_unmap_common(common);
+ op->status = common->status;
}
static long
gnttab_unmap_and_replace(
XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) uop, unsigned int count)
{
- int i;
+ int i, c, partial_done, done = 0;
struct gnttab_unmap_and_replace op;
-
- for ( i = 0; i < count; i++ )
- {
- if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
- goto fault;
- __gnttab_unmap_and_replace(&op);
- if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
- goto fault;
- }
-
- flush_tlb_mask(current->domain->domain_dirty_cpumask);
+ struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
+
+ while (count != 0) {
+ c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
+ partial_done = 0;
+
+ for ( i = 0; i < c; i++ )
+ {
+ if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
+ goto fault;
+ __gnttab_unmap_and_replace(&op, &(common[i]));
+ ++partial_done;
+ if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
+ goto fault;
+ }
+
+ flush_tlb_mask(current->domain->domain_dirty_cpumask);
+
+ for ( i = 0; i < partial_done; i++ )
+ {
+ __gnttab_unmap_common_complete(&(common[i]));
+ }
+
+ count -= c;
+ done += c;
+ }
+
return 0;
fault:
flush_tlb_mask(current->domain->domain_dirty_cpumask);
+
+ for ( i = 0; i < partial_done; i++ )
+ {
+ __gnttab_unmap_common_complete(&(common[i]));
+ }
return -EFAULT;
}
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel