From a2c4efbb5b968a80eb552757308c2fb2f28157c6 Mon Sep 17 00:00:00 2001

From: Marcel Apfelbaum <marcel.a@redhat.com>

Date: Sun, 19 Jan 2014 13:07:36 +0100

Subject: [PATCH 11/11] exec: separate sections and nodes per address space

RH-Author: Marcel Apfelbaum <marcel.a@redhat.com>

Message-id: <1390136856-7024-3-git-send-email-marcel.a@redhat.com>

Patchwork-id: 56811

O-Subject: [RHEL-7 qemu-kvm PATCH v2 2/2] exec: separate sections and nodes per address space

Bugzilla: 1003535

RH-Acked-by: Michael S. Tsirkin <mst@redhat.com>

RH-Acked-by: Paolo Bonzini <pbonzini@redhat.com>

RH-Acked-by: Markus Armbruster <armbru@redhat.com>

Every address space has its own nodes and sections, but

it uses the same global arrays of nodes/section.

This limits the number of devices that can be attached

to the guest to 20-30 devices. It happens because:

 - The sections array is limited to 2^12 entries.

 - The main memory has at least 100 sections.

 - Each device address space is actually an alias to

   main memory, multiplying its number of nodes/sections.

Remove the limitation by using separate arrays of

nodes and sections for each address space.

Closest upstream commit: 53cb28cbfea038f8ad50132dc8a684e638c7d48b

Signed-off-by: Marcel Apfelbaum <marcel.a@redhat.com>

---

v1 -> v2:

 - The series confilcted with Juan's series:

   - [RHEL7 qemu-kvm PATCH 00/40] bitmap optmization

 - Conflicts solved:

   - AddressSpaceDispatch was moved to exec.c

   - PhysPageEntry was moved to exec.c

 - Moved also PhysPageMap to exec.c

 exec.c | 166 ++++++++++++++++++++++++++++++++++-------------------------------

 1 file changed, 86 insertions(+), 80 deletions(-)

Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>

---

 exec.c |  166 +++++++++++++++++++++++++++++++++-------------------------------

 1 files changed, 86 insertions(+), 80 deletions(-)

diff --git a/exec.c b/exec.c

index 01c74cd..ce9310c 100644

--- a/exec.c

+++ b/exec.c

@@ -91,25 +91,32 @@ struct PhysPageEntry {

     uint16_t ptr : 15;

 };

+typedef PhysPageEntry Node[L2_SIZE];

+

+typedef struct PhysPageMap {

+    unsigned sections_nb;

+    unsigned sections_nb_alloc;

+    unsigned nodes_nb;

+    unsigned nodes_nb_alloc;

+    Node *nodes;

+    MemoryRegionSection *sections;

+} PhysPageMap;

+

 struct AddressSpaceDispatch {

     /* This is a multi-level map on the physical address space.

      * The bottom level has pointers to MemoryRegionSections.

      */

     PhysPageEntry phys_map;

+    PhysPageMap map;

     MemoryListener listener;

+    AddressSpace *as;

 };

-static MemoryRegionSection *phys_sections;

-static unsigned phys_sections_nb, phys_sections_nb_alloc;

 #define PHYS_SECTION_UNASSIGNED 0

 #define PHYS_SECTION_NOTDIRTY 1

 #define PHYS_SECTION_ROM 2

 #define PHYS_SECTION_WATCH 3

-/* Simple allocator for PhysPageEntry nodes */

-static PhysPageEntry (*phys_map_nodes)[L2_SIZE];

-static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;

-

 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)

 static void io_mem_init(void);

@@ -121,41 +128,38 @@ static MemoryRegion io_mem_watch;

 #if !defined(CONFIG_USER_ONLY)

-static void phys_map_node_reserve(unsigned nodes)

+static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)

 {

-    if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {

-        typedef PhysPageEntry Node[L2_SIZE];

-        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);

-        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,

-                                      phys_map_nodes_nb + nodes);

-        phys_map_nodes = g_renew(Node, phys_map_nodes,

-                                 phys_map_nodes_nb_alloc);

+    if (map->nodes_nb + nodes > map->nodes_nb_alloc) {

+        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);

+        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);

+        map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);

     }

 }

-static uint16_t phys_map_node_alloc(void)

+static uint16_t phys_map_node_alloc(PhysPageMap *map)

 {

     unsigned i;

     uint16_t ret;

-    ret = phys_map_nodes_nb++;

+    ret = map->nodes_nb++;

     assert(ret != PHYS_MAP_NODE_NIL);

-    assert(ret != phys_map_nodes_nb_alloc);

+    assert(ret != map->nodes_nb_alloc);

     for (i = 0; i < L2_SIZE; ++i) {

-        phys_map_nodes[ret][i].is_leaf = 0;

-        phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;

+        map->nodes[ret][i].is_leaf = 0;

+        map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;

     }

     return ret;

 }

-static void phys_map_nodes_reset(void)

+static void phys_map_nodes_reset(PhysPageMap *map)

 {

-    phys_map_nodes_nb = 0;

+    map->nodes_nb = 0;

 }

-static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,

-                                hwaddr *nb, uint16_t leaf,

+static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,

+                                hwaddr *index, hwaddr *nb, uint16_t leaf,

                                 int level)

 {

     PhysPageEntry *p;

@@ -163,8 +167,8 @@ static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,

     hwaddr step = (hwaddr)1 << (level * L2_BITS);

     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {

-        lp->ptr = phys_map_node_alloc();

-        p = phys_map_nodes[lp->ptr];

+        lp->ptr = phys_map_node_alloc(map);

+        p = map->nodes[lp->ptr];

         if (level == 0) {

             for (i = 0; i < L2_SIZE; i++) {

                 p[i].is_leaf = 1;

@@ -172,7 +176,7 @@ static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,

             }

         }

     } else {

-        p = phys_map_nodes[lp->ptr];

+        p = map->nodes[lp->ptr];

     }

     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];

@@ -183,7 +187,7 @@ static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,

             *index += step;

             *nb -= step;

         } else {

-            phys_page_set_level(lp, index, nb, leaf, level - 1);

+            phys_page_set_level(map, lp, index, nb, leaf, level - 1);

         }

         ++lp;

     }

@@ -194,9 +198,10 @@ static void phys_page_set(AddressSpaceDispatch *d,

                           uint16_t leaf)

 {

     /* Wildly overreserve - it doesn't matter much. */

-    phys_map_node_reserve(3 * P_L2_LEVELS);

+    phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);

-    phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

+    phys_page_set_level(&d->map, &d->phys_map, &index,

+                        &nb, leaf, P_L2_LEVELS - 1);

 }

 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)

@@ -210,13 +215,13 @@ MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)

         if (lp.ptr == PHYS_MAP_NODE_NIL) {

             goto not_found;

         }

-        p = phys_map_nodes[lp.ptr];

+        p = d->map.nodes[lp.ptr];

         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];

     }

     s_index = lp.ptr;

 not_found:

-    return &phys_sections[s_index];

+    return &d->map.sections[s_index];

 }

 bool memory_region_is_unassigned(MemoryRegion *mr)

@@ -657,7 +662,7 @@ hwaddr memory_region_section_get_iotlb(CPUArchState *env,

            and avoid full address decoding in every device.

            We can't use the high bits of pd for this because

            IO_MEM_ROMD uses these as a ram address.  */

-        iotlb = section - phys_sections;

+        iotlb = section - address_space_memory.dispatch->map.sections;

         iotlb += memory_region_section_addr(section, paddr);

     }

@@ -683,13 +688,14 @@ hwaddr memory_region_section_get_iotlb(CPUArchState *env,

 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)

 typedef struct subpage_t {

     MemoryRegion iomem;

+    AddressSpace *as;

     hwaddr base;

     uint16_t sub_section[TARGET_PAGE_SIZE];

 } subpage_t;

 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,

                              uint16_t section);

-static subpage_t *subpage_init(hwaddr base);

+static subpage_t *subpage_init(AddressSpace *as, hwaddr base);

 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;

@@ -703,9 +709,9 @@ void phys_mem_set_alloc(void *(*alloc)(size_t))

     phys_mem_alloc = alloc;

 }

-static void destroy_page_desc(uint16_t section_index)

+static void destroy_page_desc(PhysPageMap *map, uint16_t section_index)

 {

-    MemoryRegionSection *section = &phys_sections[section_index];

+    MemoryRegionSection *section = &map->sections[section_index];

     MemoryRegion *mr = section->mr;

     if (mr->subpage) {

@@ -715,7 +721,8 @@ static void destroy_page_desc(uint16_t section_index)

     }

 }

-static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)

+static void destroy_l2_mapping(PhysPageMap *map, PhysPageEntry *lp,

+                               unsigned level)

 {

     unsigned i;

     PhysPageEntry *p;

@@ -724,12 +731,12 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)

         return;

     }

-    p = phys_map_nodes[lp->ptr];

+    p = map->nodes[lp->ptr];

     for (i = 0; i < L2_SIZE; ++i) {

         if (!p[i].is_leaf) {

-            destroy_l2_mapping(&p[i], level - 1);

+            destroy_l2_mapping(map, &p[i], level - 1);

         } else {

-            destroy_page_desc(p[i].ptr);

+            destroy_page_desc(map, p[i].ptr);

         }

     }

     lp->is_leaf = 0;

@@ -738,24 +745,25 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)

 static void destroy_all_mappings(AddressSpaceDispatch *d)

 {

-    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);

-    phys_map_nodes_reset();

+    destroy_l2_mapping(&d->map, &d->phys_map, P_L2_LEVELS - 1);

+    phys_map_nodes_reset(&d->map);

 }

-static uint16_t phys_section_add(MemoryRegionSection *section)

+static uint16_t phys_section_add(PhysPageMap *map,

+                                 MemoryRegionSection *section)

 {

-    if (phys_sections_nb == phys_sections_nb_alloc) {

-        phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);

-        phys_sections = g_renew(MemoryRegionSection, phys_sections,

-                                phys_sections_nb_alloc);

+    if (map->sections_nb == map->sections_nb_alloc) {

+        map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);

+        map->sections = g_renew(MemoryRegionSection, map->sections,

+                                map->sections_nb_alloc);

     }

-    phys_sections[phys_sections_nb] = *section;

-    return phys_sections_nb++;

+    map->sections[map->sections_nb] = *section;

+    return map->sections_nb++;

 }

-static void phys_sections_clear(void)

+static void phys_sections_clear(PhysPageMap *map)

 {

-    phys_sections_nb = 0;

+    map->sections_nb = 0;

 }

 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)

@@ -773,16 +781,16 @@ static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *secti

     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);

     if (!(existing->mr->subpage)) {

-        subpage = subpage_init(base);

+        subpage = subpage_init(d->as, base);

         subsection.mr = &subpage->iomem;

         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,

-                      phys_section_add(&subsection));

+                      phys_section_add(&d->map, &subsection));

     } else {

         subpage = container_of(existing->mr, subpage_t, iomem);

     }

     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;

     end = start + section->size - 1;

-    subpage_register(subpage, start, end, phys_section_add(section));

+    subpage_register(subpage, start, end, phys_section_add(&d->map, section));

 }

@@ -791,7 +799,7 @@ static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *sec

     hwaddr start_addr = section->offset_within_address_space;

     ram_addr_t size = section->size;

     hwaddr addr;

-    uint16_t section_index = phys_section_add(section);

+    uint16_t section_index = phys_section_add(&d->map, section);

     assert(size);

@@ -1619,7 +1627,7 @@ static uint64_t subpage_read(void *opaque, hwaddr addr,

            mmio, len, addr, idx);

 #endif

-    section = &phys_sections[mmio->sub_section[idx]];

+    section = &mmio->as->dispatch->map.sections[mmio->sub_section[idx]];

     addr += mmio->base;

     addr -= section->offset_within_address_space;

     addr += section->offset_within_region;

@@ -1638,7 +1646,7 @@ static void subpage_write(void *opaque, hwaddr addr,

            __func__, mmio, len, addr, idx, value);

 #endif

-    section = &phys_sections[mmio->sub_section[idx]];

+    section = &mmio->as->dispatch->map.sections[mmio->sub_section[idx]];

     addr += mmio->base;

     addr -= section->offset_within_address_space;

     addr += section->offset_within_region;

@@ -1696,10 +1704,10 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,

     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,

            mmio, start, end, idx, eidx, memory);

 #endif

-    if (memory_region_is_ram(phys_sections[section].mr)) {

-        MemoryRegionSection new_section = phys_sections[section];

+    if (memory_region_is_ram(mmio->as->dispatch->map.sections[section].mr)) {

+        MemoryRegionSection new_section = mmio->as->dispatch->map.sections[section];

         new_section.mr = &io_mem_subpage_ram;

-        section = phys_section_add(&new_section);

+        section = phys_section_add(&mmio->as->dispatch->map, &new_section);

     }

     for (; idx <= eidx; idx++) {

         mmio->sub_section[idx] = section;

@@ -1708,12 +1716,13 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,

     return 0;

 }

-static subpage_t *subpage_init(hwaddr base)

+static subpage_t *subpage_init(AddressSpace *as, hwaddr base)

 {

     subpage_t *mmio;

     mmio = g_malloc0(sizeof(subpage_t));

+    mmio->as = as;

     mmio->base = base;

     memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,

                           "subpage", TARGET_PAGE_SIZE);

@@ -1727,7 +1736,7 @@ static subpage_t *subpage_init(hwaddr base)

     return mmio;

 }

-static uint16_t dummy_section(MemoryRegion *mr)

+static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)

 {

     MemoryRegionSection section = {

         .mr = mr,

@@ -1736,12 +1745,13 @@ static uint16_t dummy_section(MemoryRegion *mr)

         .size = UINT64_MAX,

     };

-    return phys_section_add(&section);

+    return phys_section_add(map, &section);

 }

 MemoryRegion *iotlb_to_region(hwaddr index)

 {

-    return phys_sections[index & ~TARGET_PAGE_MASK].mr;

+    return address_space_memory.dispatch->map.sections[

+           index & ~TARGET_PAGE_MASK].mr;

 }

 static void io_mem_init(void)

@@ -1761,23 +1771,19 @@ static void io_mem_init(void)

 static void mem_begin(MemoryListener *listener)

 {

     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);

+    uint16_t n;

     destroy_all_mappings(d);

     d->phys_map.ptr = PHYS_MAP_NODE_NIL;

-}

-

-static void core_begin(MemoryListener *listener)

-{

-    uint16_t n;

-    phys_sections_clear();

-    n = dummy_section(&io_mem_unassigned);

+    phys_sections_clear(&d->map);

+    n = dummy_section(&d->map, &io_mem_unassigned);

     assert(n == PHYS_SECTION_UNASSIGNED);

-    n = dummy_section(&io_mem_notdirty);

+    n = dummy_section(&d->map, &io_mem_notdirty);

     assert(n == PHYS_SECTION_NOTDIRTY);

-    n = dummy_section(&io_mem_rom);

+    n = dummy_section(&d->map, &io_mem_rom);

     assert(n == PHYS_SECTION_ROM);

-    n = dummy_section(&io_mem_watch);

+    n = dummy_section(&d->map, &io_mem_watch);

     assert(n == PHYS_SECTION_WATCH);

 }

@@ -1822,7 +1828,6 @@ static void io_region_del(MemoryListener *listener,

 }

 static MemoryListener core_memory_listener = {

-    .begin = core_begin,

     .log_global_start = core_log_global_start,

     .log_global_stop = core_log_global_stop,

     .priority = 1,

@@ -1840,7 +1845,7 @@ static MemoryListener tcg_memory_listener = {

 void address_space_init_dispatch(AddressSpace *as)

 {

-    AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);

+    AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);

     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };

     d->listener = (MemoryListener) {

@@ -1849,6 +1854,7 @@ void address_space_init_dispatch(AddressSpace *as)

         .region_nop = mem_add,

         .priority = 0,

     };

+    d->as = as;

     as->dispatch = d;

     memory_listener_register(&d->listener, as);

 }

@@ -1858,7 +1864,7 @@ void address_space_destroy_dispatch(AddressSpace *as)

     AddressSpaceDispatch *d = as->dispatch;

     memory_listener_unregister(&d->listener);

-    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);

+    destroy_l2_mapping(&d->map, &d->phys_map, P_L2_LEVELS - 1);

     g_free(d);

     as->dispatch = NULL;

 }

@@ -2446,7 +2452,7 @@ void stl_phys_notdirty(hwaddr addr, uint32_t val)

     if (!memory_region_is_ram(section->mr) || section->readonly) {

         addr = memory_region_section_addr(section, addr);

         if (memory_region_is_ram(section->mr)) {

-            section = &phys_sections[PHYS_SECTION_ROM];

+            section = &address_space_memory.dispatch->map.sections[PHYS_SECTION_ROM];

         }

         io_mem_write(section->mr, addr, val, 4);

     } else {

@@ -2479,7 +2485,7 @@ void stq_phys_notdirty(hwaddr addr, uint64_t val)

     if (!memory_region_is_ram(section->mr) || section->readonly) {

         addr = memory_region_section_addr(section, addr);

         if (memory_region_is_ram(section->mr)) {

-            section = &phys_sections[PHYS_SECTION_ROM];

+            section = &address_space_memory.dispatch->map.sections[PHYS_SECTION_ROM];

         }

 #ifdef TARGET_WORDS_BIGENDIAN

         io_mem_write(section->mr, addr, val >> 32, 4);

@@ -2508,7 +2514,7 @@ static inline void stl_phys_internal(hwaddr addr, uint32_t val,

     if (!memory_region_is_ram(section->mr) || section->readonly) {

         addr = memory_region_section_addr(section, addr);

         if (memory_region_is_ram(section->mr)) {

-            section = &phys_sections[PHYS_SECTION_ROM];

+            section = &address_space_memory.dispatch->map.sections[PHYS_SECTION_ROM];

         }

 #if defined(TARGET_WORDS_BIGENDIAN)

         if (endian == DEVICE_LITTLE_ENDIAN) {

@@ -2575,7 +2581,7 @@ static inline void stw_phys_internal(hwaddr addr, uint32_t val,

     if (!memory_region_is_ram(section->mr) || section->readonly) {

         addr = memory_region_section_addr(section, addr);

         if (memory_region_is_ram(section->mr)) {

-            section = &phys_sections[PHYS_SECTION_ROM];

+            section = &address_space_memory.dispatch->map.sections[PHYS_SECTION_ROM];

         }

 #if defined(TARGET_WORDS_BIGENDIAN)

         if (endian == DEVICE_LITTLE_ENDIAN) {

-- 

1.7.1