Xen devel - Jan 2013 - [PATCH 00/10 V5] arm: support for initial modules (e.g. dom0) and DTB supplied in RAM

The following series implements support for initial images and DTB in
RAM, as opposed to in flash (dom0 kernel) or compiled into the
hypervisor (DTB). It arranges to not clobber these with either the h/v
text on relocation or with the heaps and frees them as appropriate.

Most of this is independent of the specific bootloader protocol which is
used to tell Xen where these modules actually are, but I have included a
simple PoC bootloader protocol based around device tree which is similar
to the protocol used by Linux to find its initrd
(where /chosen/linux,initrd-{start,end} indicate the physical address).

The PoC protocol is documented in docs/misc/arm/device-tree/booting.txt
which is added by this series.

The major change in this posting is to push the device-tree PoC bits to
the end of the series after the more generic mechanisms, as discussed in
<1355913325.14620.303.camel@zakaz.uk.xensource.com>. The earlier patches
can now potentially be committed while the later ones (specifically
patches 8 + 9) are still under discussion.

I''ve included Stefano''s "flush dcache after
memcpy''ing the kernel image"
patch here too since it depends on the refactoring in this series.

Ian.

The intention is that these will eventually be filled in with
information from the bootloader, perhaps via a DTB binding.

Allow for 2 modules (kernel and initrd), plus a third pseudo-module
which is the hypervisor itself. Currently we neither parse nor do
anything with them.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v5: Split from "xen: arm: parse modules from DT during early boot."
    so we can discuss the specifics of the DTB interface separately.
v4: Use /chosen/modules/module@N
    Identify module type by compatible property not number.
v3: Use a reg = < > property for the module address/length.
v2: Reserve the zeroeth module for Xen itself (not used yet)
    Use a more idiomatic DT layout
    Document said layout
---
 xen/include/xen/device_tree.h |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)

diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
index a0e3a97..52ef258 100644
--- a/xen/include/xen/device_tree.h
+++ b/xen/include/xen/device_tree.h
@@ -15,6 +15,7 @@
 #define DEVICE_TREE_MAX_DEPTH 16
 
 #define NR_MEM_BANKS 8
+#define NR_MODULES 2
 
 struct membank {
     paddr_t start;
@@ -33,9 +34,22 @@ struct dt_gic_info {
     paddr_t gic_vcpu_addr;
 };
 
+struct dt_mb_module {
+    paddr_t start;
+    paddr_t size;
+    char cmdline[1024];
+};
+
+struct dt_module_info {
+    int nr_mods;
+    /* Module 0 is Xen itself, followed by the provided modules-proper */
+    struct dt_mb_module module[NR_MODULES + 1];
+};
+
 struct dt_early_info {
     struct dt_mem_info mem;
     struct dt_gic_info gic;
+    struct dt_module_info modules;
 };
 
 typedef int (*device_tree_node_func)(const void *fdt,
-- 
1.7.9.1

This will still fail if the modules are such that Xen is pushed out of
the top 32M of RAM since it will then overlap with the domheap (or
possibly xenheap). This will be dealt with later.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
v2: Xen is module 0, modules start at 1.
---
 xen/arch/arm/setup.c |   68 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index b5cb912..9b5d4ed 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -68,17 +68,55 @@ static void __init processor_id(void)
            READ_CP32(ID_ISAR3), READ_CP32(ID_ISAR4), READ_CP32(ID_ISAR5));
 }
 
+/*
+ * Returns the end address of the highest region in the range s..e
+ * with required size and alignment that does not conflict with the
+ * modules from first_mod to nr_modules.
+ *
+ * For non-recursive callers first_mod should normally be 1.
+ */
+static paddr_t __init consider_modules(paddr_t s, paddr_t e,
+                                       uint32_t size, paddr_t align,
+                                       int first_mod)
+{
+    const struct dt_module_info *mi = &early_info.modules;
+    int i;
+
+    s = (s+align-1) & ~(align-1);
+    e = e & ~(align-1);
+
+    if ( s > e ||  e - s < size )
+        return 0;
+
+    for ( i = first_mod; i <= mi->nr_mods; i++ )
+    {
+        paddr_t mod_s = mi->module[i].start;
+        paddr_t mod_e = mod_s + mi->module[i].size;
+
+        if ( s < mod_e && mod_s < e )
+        {
+            mod_e = consider_modules(mod_e, e, size, align, i+1);
+            if ( mod_e )
+                return mod_e;
+
+            return consider_modules(s, mod_s, size, align, i+1);
+        }
+    }
+
+    return e;
+}
+
 /**
  * get_xen_paddr - get physical address to relocate Xen to
  *
- * Xen is relocated to the top of RAM and aligned to a XEN_PADDR_ALIGN
- * boundary.
+ * Xen is relocated to as near to the top of RAM as possible and
+ * aligned to a XEN_PADDR_ALIGN boundary.
  */
 static paddr_t __init get_xen_paddr(void)
 {
     struct dt_mem_info *mi = &early_info.mem;
     paddr_t min_size;
-    paddr_t paddr = 0, t;
+    paddr_t paddr = 0;
     int i;
 
     min_size = (_end - _start + (XEN_PADDR_ALIGN-1)) &
~(XEN_PADDR_ALIGN-1);
@@ -86,17 +124,33 @@ static paddr_t __init get_xen_paddr(void)
     /* Find the highest bank with enough space. */
     for ( i = 0; i < mi->nr_banks; i++ )
     {
-        if ( mi->bank[i].size >= min_size )
+        const struct membank *bank = &mi->bank[i];
+        paddr_t s, e;
+
+        if ( bank->size >= min_size )
         {
-            t = mi->bank[i].start + mi->bank[i].size - min_size;
-            if ( t > paddr )
-                paddr = t;
+            e = consider_modules(bank->start, bank->start +
bank->size,
+                                 min_size, XEN_PADDR_ALIGN, 1);
+            if ( !e )
+                continue;
+
+            s = e - min_size;
+
+            if ( s > paddr )
+                paddr = s;
         }
     }
 
     if ( !paddr )
         early_panic("Not enough memory to relocate Xen\n");
 
+    early_printk("Placing Xen at
0x%"PRIpaddr"-0x%"PRIpaddr"\n",
+                 paddr, paddr + min_size);
+
+    /* Xen is module 0 */
+    early_info.modules.module[0].start = paddr;
+    early_info.modules.module[0].size = min_size;
+
     return paddr;
 }
 
-- 
1.7.9.1

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
 xen/arch/arm/setup.c |   89 +++++++++++++++++++++++++++++++++++++++++++------
 1 files changed, 78 insertions(+), 11 deletions(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 9b5d4ed..e6cefeb 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -73,7 +73,8 @@ static void __init processor_id(void)
  * with required size and alignment that does not conflict with the
  * modules from first_mod to nr_modules.
  *
- * For non-recursive callers first_mod should normally be 1.
+ * For non-recursive callers first_mod should normally be 0 (all
+ * modules and Xen itself) or 1 (all modules but not Xen).
  */
 static paddr_t __init consider_modules(paddr_t s, paddr_t e,
                                        uint32_t size, paddr_t align,
@@ -106,6 +107,34 @@ static paddr_t __init consider_modules(paddr_t s, paddr_t
e,
     return e;
 }
 
+/*
+ * Return the end of the non-module region starting at s. In other
+ * words return s the start of the next modules after s.
+ *
+ * Also returns the end of that module in *n.
+ */
+static paddr_t __init next_module(paddr_t s, paddr_t *n)
+{
+    struct dt_module_info *mi = &early_info.modules;
+    paddr_t lowest = ~(paddr_t)0;
+    int i;
+
+    for ( i = 0; i <= mi->nr_mods; i++ )
+    {
+        paddr_t mod_s = mi->module[i].start;
+        paddr_t mod_e = mod_s + mi->module[i].size;
+
+        if ( mod_s < s )
+            continue;
+        if ( mod_s > lowest )
+            continue;
+        lowest = mod_s;
+        *n = mod_e;
+    }
+    return lowest;
+}
+
+
 /**
  * get_xen_paddr - get physical address to relocate Xen to
  *
@@ -159,6 +188,7 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t
dtb_size)
     paddr_t ram_start;
     paddr_t ram_end;
     paddr_t ram_size;
+    paddr_t s, e;
     unsigned long ram_pages;
     unsigned long heap_pages, xenheap_pages, domheap_pages;
     unsigned long dtb_pages;
@@ -176,22 +206,37 @@ static void __init setup_mm(unsigned long dtb_paddr,
size_t dtb_size)
     ram_pages = ram_size >> PAGE_SHIFT;
 
     /*
-     * Calculate the sizes for the heaps using these constraints:
+     * Locate the xenheap using these constraints:
      *
-     *  - heaps must be 32 MiB aligned
-     *  - must not include Xen itself
-     *  - xen heap must be at most 1 GiB
+     *  - must be 32 MiB aligned
+     *  - must not include Xen itself or the boot modules
+     *  - must be at most 1 GiB
+     *  - must be at least 128M
      *
-     * XXX: needs a platform with at least 1GiB of RAM or the dom
-     * heap will be empty and no domains can be created.
+     * We try to allocate the largest xenheap possible within these
+     * constraints.
      */
-    heap_pages = (ram_size >> PAGE_SHIFT) - (32 << (20 -
PAGE_SHIFT));
+    heap_pages = (ram_size >> PAGE_SHIFT);
     xenheap_pages = min(1ul << (30 - PAGE_SHIFT), heap_pages);
+
+    do
+    {
+        e = consider_modules(ram_start, ram_end,
xenheap_pages<<PAGE_SHIFT,
+                             32<<20, 0);
+        if ( e )
+            break;
+
+        xenheap_pages >>= 1;
+    } while ( xenheap_pages > 128<<(20-PAGE_SHIFT) );
+
+    if ( ! e )
+        panic("Not not enough space for xenheap\n");
+
     domheap_pages = heap_pages - xenheap_pages;
 
     printk("Xen heap: %lu pages  Dom heap: %lu pages\n",
xenheap_pages, domheap_pages);
 
-    setup_xenheap_mappings(ram_start >> PAGE_SHIFT, xenheap_pages);
+    setup_xenheap_mappings((e >> PAGE_SHIFT) - xenheap_pages,
xenheap_pages);
 
     /*
      * Need a single mapped page for populating bootmem_region_list
@@ -215,8 +260,30 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t
dtb_size)
     copy_from_paddr(device_tree_flattened, dtb_paddr, dtb_size, BUFFERABLE);
 
     /* Add non-xenheap memory */
-    init_boot_pages(pfn_to_paddr(xenheap_mfn_start + xenheap_pages),
-                    pfn_to_paddr(xenheap_mfn_start + xenheap_pages +
domheap_pages));
+    s = ram_start;
+    while ( s < ram_end )
+    {
+        paddr_t n = ram_end;
+
+        e = next_module(s, &n);
+
+        if ( e == ~(paddr_t)0 )
+        {
+            e = n = ram_end;
+        }
+
+        /* Avoid the xenheap */
+        if ( s < ((xenheap_mfn_start+xenheap_pages) << PAGE_SHIFT)
+             && (xenheap_mfn_start << PAGE_SHIFT) < e )
+        {
+            e = pfn_to_paddr(xenheap_mfn_start);
+            n = pfn_to_paddr(xenheap_mfn_start+xenheap_pages);
+        }
+
+        init_boot_pages(s, e);
+
+        s = n;
+    }
 
     setup_frametable_mappings(ram_start, ram_end);
     max_page = PFN_DOWN(ram_end);
-- 
1.7.9.1

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
v3: early_panic instead of BUG_ON
v2: drop unrelated white space fixup
---
 xen/common/device_tree.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/xen/common/device_tree.c b/xen/common/device_tree.c
index 8b4ef2f..260c2d4 100644
--- a/xen/common/device_tree.c
+++ b/xen/common/device_tree.c
@@ -84,6 +84,9 @@ static void __init get_val(const u32 **cell, u32 cells, u64
*val)
 {
     *val = 0;
 
+    if ( cells > 2 )
+        early_panic("dtb value contains > 2 cells\n");
+
     while ( cells-- )
     {
         *val <<= 32;
-- 
1.7.9.1

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
v3: - correct limit check in try_zimage_prepare
    - copy zimage header to a local bufffer to avoid issues with
      crossing page boundaries.
    - handle non page aligned source and destinations when loading
    - use a BUFFERABLE mapping when loading kernel from RAM.
---
 xen/arch/arm/kernel.c |   91 ++++++++++++++++++++++++++++++++++--------------
 xen/arch/arm/kernel.h |   11 ++++++
 2 files changed, 75 insertions(+), 27 deletions(-)

diff --git a/xen/arch/arm/kernel.c b/xen/arch/arm/kernel.c
index 2d56130..c9265d7 100644
--- a/xen/arch/arm/kernel.c
+++ b/xen/arch/arm/kernel.c
@@ -22,6 +22,7 @@
 #define ZIMAGE_MAGIC_OFFSET 0x24
 #define ZIMAGE_START_OFFSET 0x28
 #define ZIMAGE_END_OFFSET   0x2c
+#define ZIMAGE_HEADER_LEN   0x30
 
 #define ZIMAGE_MAGIC 0x016f2818
 
@@ -65,40 +66,42 @@ void copy_from_paddr(void *dst, paddr_t paddr, unsigned long
len, int attrindx)
 static void kernel_zimage_load(struct kernel_info *info)
 {
     paddr_t load_addr = info->zimage.load_addr;
+    paddr_t paddr = info->zimage.kernel_addr;
+    paddr_t attr = info->load_attr;
     paddr_t len = info->zimage.len;
-    paddr_t flash = KERNEL_FLASH_ADDRESS;
-    void *src = (void *)FIXMAP_ADDR(FIXMAP_MISC);
     unsigned long offs;
 
-    printk("Loading %"PRIpaddr" byte zImage from flash
%"PRIpaddr" to %"PRIpaddr"-%"PRIpaddr": [",
-           len, flash, load_addr, load_addr + len);
-    for ( offs = 0; offs < len; offs += PAGE_SIZE )
+    printk("Loading zImage from %"PRIpaddr" to
%"PRIpaddr"-%"PRIpaddr"\n",
+           paddr, load_addr, load_addr + len);
+    for ( offs = 0; offs < len; )
     {
-        paddr_t ma = gvirt_to_maddr(load_addr + offs);
+        paddr_t s, l, ma = gvirt_to_maddr(load_addr + offs);
         void *dst = map_domain_page(ma>>PAGE_SHIFT);
 
-        if ( ( offs % (1<<20) ) == 0 )
-            printk(".");
+        s = offs & ~PAGE_MASK;
+        l = min(PAGE_SIZE - s, len);
 
-        set_fixmap(FIXMAP_MISC, (flash+offs) >> PAGE_SHIFT, DEV_SHARED);
-        memcpy(dst, src, PAGE_SIZE);
-        clear_fixmap(FIXMAP_MISC);
+        copy_from_paddr(dst + s, paddr + offs, l, attr);
 
         unmap_domain_page(dst);
+        offs += l;
     }
-    printk("]\n");
 }
 
 /**
  * Check the image is a zImage and return the load address and length
  */
-static int kernel_try_zimage_prepare(struct kernel_info *info)
+static int kernel_try_zimage_prepare(struct kernel_info *info,
+                                     paddr_t addr, paddr_t size)
 {
-    uint32_t *zimage = (void *)FIXMAP_ADDR(FIXMAP_MISC);
+    uint32_t zimage[ZIMAGE_HEADER_LEN/4];
     uint32_t start, end;
     struct minimal_dtb_header dtb_hdr;
 
-    set_fixmap(FIXMAP_MISC, KERNEL_FLASH_ADDRESS >> PAGE_SHIFT,
DEV_SHARED);
+    if ( size < ZIMAGE_HEADER_LEN )
+        return -EINVAL;
+
+    copy_from_paddr(zimage, addr, sizeof(zimage), DEV_SHARED);
 
     if (zimage[ZIMAGE_MAGIC_OFFSET/4] != ZIMAGE_MAGIC)
         return -EINVAL;
@@ -106,16 +109,26 @@ static int kernel_try_zimage_prepare(struct kernel_info
*info)
     start = zimage[ZIMAGE_START_OFFSET/4];
     end = zimage[ZIMAGE_END_OFFSET/4];
 
-    clear_fixmap(FIXMAP_MISC);
+    if ( (end - start) > size )
+        return -EINVAL;
 
     /*
      * Check for an appended DTB.
      */
-    copy_from_paddr(&dtb_hdr, KERNEL_FLASH_ADDRESS + end - start,
sizeof(dtb_hdr), DEV_SHARED);
-    if (be32_to_cpu(dtb_hdr.magic) == DTB_MAGIC) {
-        end += be32_to_cpu(dtb_hdr.total_size);
+    if ( addr + end - start + sizeof(dtb_hdr) <= size )
+    {
+        copy_from_paddr(&dtb_hdr, addr + end - start,
+                        sizeof(dtb_hdr), DEV_SHARED);
+        if (be32_to_cpu(dtb_hdr.magic) == DTB_MAGIC) {
+            end += be32_to_cpu(dtb_hdr.total_size);
+
+            if ( end > addr + size )
+                return -EINVAL;
+        }
     }
 
+    info->zimage.kernel_addr = addr;
+
     /*
      * If start is zero, the zImage is position independent -- load it
      * at 32k from start of RAM.
@@ -142,25 +155,26 @@ static void kernel_elf_load(struct kernel_info *info)
     free_xenheap_pages(info->kernel_img, info->kernel_order);
 }
 
-static int kernel_try_elf_prepare(struct kernel_info *info)
+static int kernel_try_elf_prepare(struct kernel_info *info,
+                                  paddr_t addr, paddr_t size)
 {
     int rc;
 
-    info->kernel_order = get_order_from_bytes(KERNEL_FLASH_SIZE);
+    info->kernel_order = get_order_from_bytes(size);
     info->kernel_img = alloc_xenheap_pages(info->kernel_order, 0);
     if ( info->kernel_img == NULL )
         panic("Cannot allocate temporary buffer for kernel.\n");
 
-    copy_from_paddr(info->kernel_img, KERNEL_FLASH_ADDRESS,
KERNEL_FLASH_SIZE, DEV_SHARED);
+    copy_from_paddr(info->kernel_img, addr, size, info->load_attr);
 
-    if ( (rc = elf_init(&info->elf.elf, info->kernel_img,
KERNEL_FLASH_SIZE )) != 0 )
-        return rc;
+    if ( (rc = elf_init(&info->elf.elf, info->kernel_img, size )) !=
0 )
+        goto err;
 #ifdef VERBOSE
     elf_set_verbose(&info->elf.elf);
 #endif
     elf_parse_binary(&info->elf.elf);
     if ( (rc = elf_xen_parse(&info->elf.elf, &info->elf.parms))
!= 0 )
-        return rc;
+        goto err;
 
     /*
      * TODO: can the ELF header be used to find the physical address
@@ -170,15 +184,38 @@ static int kernel_try_elf_prepare(struct kernel_info
*info)
     info->load = kernel_elf_load;
 
     return 0;
+err:
+    free_xenheap_pages(info->kernel_img, info->kernel_order);
+    return rc;
 }
 
 int kernel_prepare(struct kernel_info *info)
 {
     int rc;
 
-    rc = kernel_try_zimage_prepare(info);
+    paddr_t start, size;
+
+    if ( early_info.modules.nr_mods > 1 )
+        panic("Cannot handle dom0 initrd yet\n");
+
+    if ( early_info.modules.nr_mods < 1 )
+    {
+        printk("No boot modules found, trying flash\n");
+        start = KERNEL_FLASH_ADDRESS;
+        size = KERNEL_FLASH_SIZE;
+        info->load_attr = DEV_SHARED;
+    }
+    else
+    {
+        printk("Loading kernel from boot module 1\n");
+        start = early_info.modules.module[1].start;
+        size = early_info.modules.module[1].size;
+        info->load_attr = BUFFERABLE;
+    }
+
+    rc = kernel_try_zimage_prepare(info, start, size);
     if (rc < 0)
-        rc = kernel_try_elf_prepare(info);
+        rc = kernel_try_elf_prepare(info, start, size);
 
     return rc;
 }
diff --git a/xen/arch/arm/kernel.h b/xen/arch/arm/kernel.h
index 4533568..49fe9da 100644
--- a/xen/arch/arm/kernel.h
+++ b/xen/arch/arm/kernel.h
@@ -22,6 +22,7 @@ struct kernel_info {
 
     union {
         struct {
+            paddr_t kernel_addr;
             paddr_t load_addr;
             paddr_t len;
         } zimage;
@@ -33,9 +34,19 @@ struct kernel_info {
     };
 
     void (*load)(struct kernel_info *info);
+    int load_attr;
 };
 
 int kernel_prepare(struct kernel_info *info);
 void kernel_load(struct kernel_info *info);
 
 #endif /* #ifdef __ARCH_ARM_KERNEL_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
-- 
1.7.9.1

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
 xen/arch/arm/domain_build.c |    3 +++
 xen/arch/arm/setup.c        |   16 ++++++++++++++++
 xen/include/asm-arm/setup.h |    2 ++
 3 files changed, 21 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 1e9776d..96c3464 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -10,6 +10,7 @@
 #include <xen/device_tree.h>
 #include <xen/libfdt/libfdt.h>
 #include <xen/guest_access.h>
+#include <asm/setup.h>
 
 #include <asm/gic.h>
 #include "kernel.h"
@@ -306,6 +307,8 @@ int construct_dom0(struct domain *d)
     dtb_load(&kinfo);
     kernel_load(&kinfo);
 
+    discard_initial_modules();
+
     clear_bit(_VPF_down, &v->pause_flags);
 
     memset(regs, 0, sizeof(*regs));
diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index e6cefeb..e1ab7f6 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -68,6 +68,22 @@ static void __init processor_id(void)
            READ_CP32(ID_ISAR3), READ_CP32(ID_ISAR4), READ_CP32(ID_ISAR5));
 }
 
+void __init discard_initial_modules(void)
+{
+    struct dt_module_info *mi = &early_info.modules;
+    int i;
+
+    for ( i = 1; i <= mi->nr_mods; i++ )
+    {
+        paddr_t s = mi->module[i].start;
+        paddr_t e = s + PAGE_ALIGN(mi->module[i].size);
+
+        init_domheap_pages(s, e);
+    }
+
+    mi->nr_mods = 0;
+}
+
 /*
  * Returns the end address of the highest region in the range s..e
  * with required size and alignment that does not conflict with the
diff --git a/xen/include/asm-arm/setup.h b/xen/include/asm-arm/setup.h
index 5c84334..2f91a21 100644
--- a/xen/include/asm-arm/setup.h
+++ b/xen/include/asm-arm/setup.h
@@ -11,6 +11,8 @@ void arch_get_xen_caps(xen_capabilities_info_t *info);
 
 int construct_dom0(struct domain *d);
 
+void discard_initial_modules(void);
+
 #endif
 /*
  * Local variables:
-- 
1.7.9.1

Fallback to xen,dom0-bootargs if this isn''t present.

Ideally this would use module1-args iff the kernel came from the
modules and the existing xen,dom0-bootargs if the kernel came from
flash, but this approach is simpler and has the same effect in
practice.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Tim Deegan <tim@xen.org>
---
v5: this doesn''t actually depend on the DTB bindings for supplying
    modules. Reword commit log (no code change)
v2: update for new DT layout
---
 xen/arch/arm/domain_build.c |   28 ++++++++++++++++++++++++----
 1 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 96c3464..6abbb03 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -86,8 +86,13 @@ static int write_properties(struct domain *d, struct
kernel_info *kinfo,
                             int node, const char *name, int depth,
                             u32 address_cells, u32 size_cells)
 {
+    const char *bootargs = NULL;
     int prop;
 
+    if ( early_info.modules.nr_mods >= 1 &&
+         early_info.modules.module[1].cmdline[0] )
+        bootargs = &early_info.modules.module[1].cmdline[0];
+
     for ( prop = fdt_first_property_offset(fdt, node);
           prop >= 0;
           prop = fdt_next_property_offset(fdt, prop) )
@@ -104,15 +109,22 @@ static int write_properties(struct domain *d, struct
kernel_info *kinfo,
         prop_len  = fdt32_to_cpu(p->len);
 
         /*
-         * In chosen node: replace bootargs with value from
-         * xen,dom0-bootargs.
+         * In chosen node:
+         *
+         * * remember xen,dom0-bootargs if we don''t already have
+         *   bootargs (from module #1, above).
+         * * remove bootargs and xen,dom0-bootargs.
          */
         if ( device_tree_node_matches(fdt, node, "chosen") )
         {
             if ( strcmp(prop_name, "bootargs") == 0 )
                 continue;
-            if ( strcmp(prop_name, "xen,dom0-bootargs") == 0 )
-                prop_name = "bootargs";
+            else if ( strcmp(prop_name, "xen,dom0-bootargs") == 0 )
+            {
+                if ( !bootargs )
+                    bootargs = prop_data;
+                continue;
+            }
         }
         /*
          * In a memory node: adjust reg property.
@@ -147,6 +159,14 @@ static int write_properties(struct domain *d, struct
kernel_info *kinfo,
         xfree(new_data);
     }
 
+    if ( device_tree_node_matches(fdt, node, "chosen") &&
bootargs )
+        fdt_property(kinfo->fdt, "bootargs", bootargs,
strlen(bootargs) + 1);
+
+    /*
+     * XXX should populate /chosen/linux,initrd-{start,end} here if we
+     * have module[2]
+     */
+
     if ( prop == -FDT_ERR_NOTFOUND )
         return 0;
     return prop;
-- 
1.7.9.1

From: Stefano Stabellini <stefano.stabellini@eu.citrix.com>

After memcpy''ing the kernel in guest memory we need to flush the dcache
to make sure that the data actually reaches the memory before we start
executing guest code with caches disabled.

copy_from_paddr is the function that does the copy, so add a
flush_xen_dcache_va_range there.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/kernel.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/kernel.c b/xen/arch/arm/kernel.c
index c9265d7..2f7a9ff 100644
--- a/xen/arch/arm/kernel.c
+++ b/xen/arch/arm/kernel.c
@@ -54,6 +54,7 @@ void copy_from_paddr(void *dst, paddr_t paddr, unsigned long
len, int attrindx)
 
         set_fixmap(FIXMAP_MISC, p, attrindx);
         memcpy(dst, src + s, l);
+        flush_xen_dcache_va_range(dst, l);
 
         paddr += l;
         dst += l;
-- 
1.7.9.1

The bootloader should populate /chosen/modules/module@<N>/ for each
module it wishes to pass to the hypervisor. The content of these nodes
is described in docs/misc/arm/device-tree/booting.txt

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v5: Moved later in the series
v4: Use /chosen/modules/module@N
    Identify module type by compatible property not number.
v3: Use a reg = < > property for the module address/length.
v2: Reserve the zeroeth module for Xen itself (not used yet)
    Use a more idiomatic DT layout
    Document said layout
---
 docs/misc/arm/device-tree/booting.txt |   25 ++++++++++
 xen/common/device_tree.c              |   86 +++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+), 0 deletions(-)
 create mode 100644 docs/misc/arm/device-tree/booting.txt

diff --git a/docs/misc/arm/device-tree/booting.txt
b/docs/misc/arm/device-tree/booting.txt
new file mode 100644
index 0000000..94cd3f1
--- /dev/null
+++ b/docs/misc/arm/device-tree/booting.txt
@@ -0,0 +1,25 @@
+Xen is passed the dom0 kernel and initrd via a reference in the /chosen
+node of the device tree.
+
+Each node has the form /chosen/modules/module@<N> and contains the
following
+properties:
+
+- compatible
+
+	Must be:
+
+		"xen,<type>", "xen,multiboot-module"
+
+	where <type> must be one of:
+
+	- "linux-zimage" -- the dom0 kernel
+	- "linux-initrd" -- the dom0 ramdisk
+
+- reg
+
+	Specifies the physical address of the module in RAM and the
+	length of the module.
+
+- bootargs (optional)
+
+	Command line associated with this module
diff --git a/xen/common/device_tree.c b/xen/common/device_tree.c
index 260c2d4..9e6a4d4 100644
--- a/xen/common/device_tree.c
+++ b/xen/common/device_tree.c
@@ -337,6 +337,90 @@ static void __init process_gic_node(const void *fdt, int
node,
     early_info.gic.gic_vcpu_addr = start;
 }
 
+static int __init process_chosen_modules_node(const void *fdt, int node,
+                                              const char *name, int *depth,
+                                              u32 address_cells, u32
size_cells)
+{
+    const struct fdt_property *prop;
+    const u32 *cell;
+    int nr, nr_modules = 0;
+    struct dt_mb_module *mod;
+    int len;
+
+    for ( *depth = 1;
+          *depth >= 1;
+          node = fdt_next_node(fdt, node, depth) )
+    {
+        name = fdt_get_name(fdt, node, NULL);
+        if ( strncmp(name, "module@", strlen("module@")) ==
0 ) {
+
+            if ( fdt_node_check_compatible(fdt, node,
+                                           "xen,multiboot-module" )
!= 0 )
+                early_panic("%s not a compatible module node\n",
name);
+
+            if ( fdt_node_check_compatible(fdt, node,
+                                           "xen,linux-zimage") == 0 )
+                nr = 1;
+            else if ( fdt_node_check_compatible(fdt, node,
+                                                "xen,linux-initrd")
== 0)
+                nr = 2;
+            else
+                early_panic("%s not a known xen multiboot byte\n");
+
+            if ( nr > nr_modules )
+                nr_modules = nr;
+
+            mod = &early_info.modules.module[nr];
+
+            prop = fdt_get_property(fdt, node, "reg", NULL);
+            if ( !prop )
+                early_panic("node %s missing `reg''
property\n", name);
+
+            cell = (const u32 *)prop->data;
+            device_tree_get_reg(&cell, address_cells, size_cells,
+                                &mod->start, &mod->size);
+
+            prop = fdt_get_property(fdt, node, "bootargs", &len);
+            if ( prop )
+            {
+                if ( len > sizeof(mod->cmdline) )
+                    early_panic("module %d command line too long\n",
nr);
+
+                safe_strcpy(mod->cmdline, prop->data);
+            }
+            else
+                mod->cmdline[0] = 0;
+        }
+    }
+
+    for ( nr = 1 ; nr < nr_modules ; nr++ )
+    {
+        mod = &early_info.modules.module[nr];
+        if ( !mod->start || !mod->size )
+            early_panic("module %d  missing / invalid\n", nr);
+    }
+
+    early_info.modules.nr_mods = nr_modules;
+    return node;
+}
+
+static void __init process_chosen_node(const void *fdt, int node,
+                                       const char *name,
+                                       u32 address_cells, u32 size_cells)
+{
+    int depth;
+
+    for ( depth = 0;
+          depth >= 0;
+          node = fdt_next_node(fdt, node, &depth) )
+    {
+        name = fdt_get_name(fdt, node, NULL);
+        if ( depth == 1 && strcmp(name, "modules") == 0 )
+            node = process_chosen_modules_node(fdt, node, name, &depth,
+                                               address_cells, size_cells);
+    }
+}
+
 static int __init early_scan_node(const void *fdt,
                                   int node, const char *name, int depth,
                                   u32 address_cells, u32 size_cells,
@@ -348,6 +432,8 @@ static int __init early_scan_node(const void *fdt,
         process_cpu_node(fdt, node, name, address_cells, size_cells);
     else if ( device_tree_node_compatible(fdt, node,
"arm,cortex-a15-gic") )
         process_gic_node(fdt, node, name, address_cells, size_cells);
+    else if ( device_tree_node_matches(fdt, node, "chosen") )
+        process_chosen_node(fdt, node, name, address_cells, size_cells);
 
     return 0;
 }
-- 
1.7.9.1

These nodes are used by Xen to find the initial modules.

Only drop the "xen,multiboot-module" compatible nodes in case someone
else has a similar idea.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v4 - /chosen/modules/modules@N not /chosen/module@N
v3 - use a helper to filter out DT elements which are not for dom0.
     Better than an ad-hoc break in the middle of a loop.
---
 xen/arch/arm/domain_build.c |   40 ++++++++++++++++++++++++++++++++++++++--
 1 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index 6abbb03..512d78c 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -172,6 +172,40 @@ static int write_properties(struct domain *d, struct
kernel_info *kinfo,
     return prop;
 }
 
+/* Returns the next node in fdt (starting from offset) which should be
+ * passed through to dom0.
+ */
+static int fdt_next_dom0_node(const void *fdt, int node,
+                              int *depth_out,
+                              int parents[DEVICE_TREE_MAX_DEPTH])
+{
+    int depth = *depth_out;
+
+    while ( (node = fdt_next_node(fdt, node, &depth)) &&
+            node >= 0 && depth >= 0 )
+    {
+        if ( depth >= DEVICE_TREE_MAX_DEPTH )
+            break;
+
+        parents[depth] = node;
+
+        /* Skip /chosen/modules/module@<N>/ and all subnodes */
+        if ( depth >= 3 &&
+             device_tree_node_matches(fdt, parents[1], "chosen")
&&
+             device_tree_node_matches(fdt, parents[2], "modules")
&&
+             device_tree_node_matches(fdt, parents[3], "module")
&&
+             fdt_node_check_compatible(fdt, parents[3],
+                                       "xen,multiboot-module" ) == 0
)
+            continue;
+
+        /* We''ve arrived at a node which dom0 is interested in. */
+        break;
+    }
+
+    *depth_out = depth;
+    return node;
+}
+
 static int write_nodes(struct domain *d, struct kernel_info *kinfo,
                        const void *fdt)
 {
@@ -179,11 +213,12 @@ static int write_nodes(struct domain *d, struct
kernel_info *kinfo,
     int depth = 0, last_depth = -1;
     u32 address_cells[DEVICE_TREE_MAX_DEPTH];
     u32 size_cells[DEVICE_TREE_MAX_DEPTH];
+    int parents[DEVICE_TREE_MAX_DEPTH];
     int ret;
 
     for ( node = 0, depth = 0;
           node >= 0 && depth >= 0;
-          node = fdt_next_node(fdt, node, &depth) )
+          node = fdt_next_dom0_node(fdt, node, &depth, parents) )
     {
         const char *name;
 
@@ -191,7 +226,8 @@ static int write_nodes(struct domain *d, struct kernel_info
*kinfo,
 
         if ( depth >= DEVICE_TREE_MAX_DEPTH )
         {
-            printk("warning: node `%s'' is nested too
deep\n", name);
+            printk("warning: node `%s'' is nested too deep
(%d)\n",
+                   name, depth);
             continue;
         }
 
-- 
1.7.9.1

On Fri, 18 Jan 2013, Ian Campbell wrote:
> The intention is that these will eventually be filled in with
> information from the bootloader, perhaps via a DTB binding.
> 
> Allow for 2 modules (kernel and initrd), plus a third pseudo-module
> which is the hypervisor itself. Currently we neither parse nor do
> anything with them.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>

> v5: Split from "xen: arm: parse modules from DT during early
boot."
>     so we can discuss the specifics of the DTB interface separately.
> v4: Use /chosen/modules/module@N
>     Identify module type by compatible property not number.
> v3: Use a reg = < > property for the module address/length.
> v2: Reserve the zeroeth module for Xen itself (not used yet)
>     Use a more idiomatic DT layout
>     Document said layout
> ---
>  xen/include/xen/device_tree.h |   14 ++++++++++++++
>  1 files changed, 14 insertions(+), 0 deletions(-)
> 
> diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
> index a0e3a97..52ef258 100644
> --- a/xen/include/xen/device_tree.h
> +++ b/xen/include/xen/device_tree.h
> @@ -15,6 +15,7 @@
>  #define DEVICE_TREE_MAX_DEPTH 16
>  
>  #define NR_MEM_BANKS 8
> +#define NR_MODULES 2
>  
>  struct membank {
>      paddr_t start;
> @@ -33,9 +34,22 @@ struct dt_gic_info {
>      paddr_t gic_vcpu_addr;
>  };
>  
> +struct dt_mb_module {
> +    paddr_t start;
> +    paddr_t size;
> +    char cmdline[1024];
> +};
> +
> +struct dt_module_info {
> +    int nr_mods;
> +    /* Module 0 is Xen itself, followed by the provided modules-proper */
> +    struct dt_mb_module module[NR_MODULES + 1];
> +};
> +
>  struct dt_early_info {
>      struct dt_mem_info mem;
>      struct dt_gic_info gic;
> +    struct dt_module_info modules;
>  };
>  
>  typedef int (*device_tree_node_func)(const void *fdt,
> -- 
> 1.7.9.1
>

On Mon, 2013-01-21 at 11:35 +0000, Stefano Stabellini
wrote:
> On Fri, 18 Jan 2013, Ian Campbell wrote:
> > The intention is that these will eventually be filled in with
> > information from the bootloader, perhaps via a DTB binding.
> > 
> > Allow for 2 modules (kernel and initrd), plus a third pseudo-module
> > which is the hypervisor itself. Currently we neither parse nor do
> > anything with them.
> > 
> > Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> 
> Acked-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Thanks, applied this series up to and including your "xen/arm: flush
dcache after memcpy''ing the kernel image" patch (08/10).

Ian.