WIP pmm2

Change-Id: I5a2414fbfaf9de15019a02ad822790387ffdc5ea

kernel/include/kernel/vm/page.h

@@ -10,6 +10,7 @@
 #include <list.h>
 #include <magenta/compiler.h>
 #include <stdint.h>
+#include <sys/types.h>
 
 // forward declare
 class VmObject;
@@ -17,37 +18,45 @@ class VmObject;
 #define VM_PAGE_OBJECT_PIN_COUNT_BITS 5
 #define VM_PAGE_OBJECT_MAX_PIN_COUNT ((1ul << VM_PAGE_OBJECT_PIN_COUNT_BITS) - 1)
 
-// core per page structure
+// core per page structure allocated at pmm arena creation time
 typedef struct vm_page {
+    struct list_node node;
+    paddr_t paddr;
+    // offset 0x18
+
     struct {
         uint32_t flags : 8;
         uint32_t state : 3;
     };
     uint32_t map_count;
+    // offset: 0x20
 
     union {
         struct {
             // in allocated/just freed state, use a linked list to hold the page in a queue
             struct list_node node;
+            // offset: 0x30
         } free;
         struct {
             // attached to a vm object
             uint64_t offset; // unused currently
+            // offset: 0x28
             VmObject* obj; // unused currently
 
+            // offset: 0x30
             uint8_t pin_count : VM_PAGE_OBJECT_PIN_COUNT_BITS;
             // If true, one pin slot is used by the VmObject to keep a run
             // contiguous.
             bool contiguous_pin : 1;
         } object;
 
-        uint8_t pad[24]; // pad out to 32 bytes
+        uint8_t pad[0x38 - 0x20]; // pad out to 0x38 bytes
     };
 } vm_page_t;
 
 // pmm will maintain pages of this size
 #define VM_PAGE_STRUCT_SIZE (sizeof(vm_page_t))
-static_assert(sizeof(vm_page_t) == 32, "");
+static_assert(sizeof(vm_page_t) == 0x38, "");
 
 enum vm_page_state {
     VM_PAGE_STATE_FREE,

kernel/include/kernel/vm/pmm.h

@@ -22,45 +22,48 @@ typedef struct pmm_arena_info {
     size_t size;
 } pmm_arena_info_t;
 
-#define PMM_ARENA_FLAG_KMAP (0x1) // this arena is already mapped and useful for kallocs
+#define PMM_ARENA_FLAG_KMAP   (0x1) // this arena is mapped into the kernel and useful for kallocs
+#define PMM_ARENA_FLAG_LO_MEM (0x2) // this arena is contained within architecturally-defined 'low memory'
 
 // Add a pre-filled memory arena to the physical allocator.
-status_t pmm_add_arena(const pmm_arena_info_t* arena) __NONNULL((1));
+// The arena data will be copied.
+status_t pmm_add_arena(const pmm_arena_info_t* arena);
 
 // flags for allocation routines below
-#define PMM_ALLOC_FLAG_ANY (0x0)  // no restrictions on which arena to allocate from
-#define PMM_ALLOC_FLAG_KMAP (0x1) // allocate only from arenas marked KMAP
+#define PMM_ALLOC_FLAG_ANY    (0x0) // no restrictions on which arena to allocate from
+#define PMM_ALLOC_FLAG_KMAP   (0x1) // allocate only from arenas marked KMAP
+#define PMM_ALLOC_FLAG_LO_MEM (0x2) // allocate only from arenas marked LO_MEM
 
 // Allocate count pages of physical memory, adding to the tail of the passed list.
 // The list must be initialized.
 // Returns the number of pages allocated.
-size_t pmm_alloc_pages(size_t count, uint alloc_flags, struct list_node* list) __NONNULL((3));
+size_t pmm_alloc_pages(size_t count, uint alloc_flags, list_node* list);
 
 // Allocate a single page of physical memory.
 vm_page_t* pmm_alloc_page(uint alloc_flags, paddr_t* pa);
 
 // Allocate a specific range of physical pages, adding to the tail of the passed list.
 // Returns the number of pages allocated.
-size_t pmm_alloc_range(paddr_t address, size_t count, struct list_node* list);
+size_t pmm_alloc_range(paddr_t address, size_t count, list_node* list);
 
 // Allocate a run of contiguous pages, aligned on log2 byte boundary (0-31)
 // If the optional physical address pointer is passed, return the address.
 // If the optional list is passed, append the allocate page structures to the tail of the list.
 size_t pmm_alloc_contiguous(size_t count, uint alloc_flags, uint8_t align_log2, paddr_t* pa,
-                            struct list_node* list);
+                            list_node* list);
 
 // Free a list of physical pages.
 // Returns the number of pages freed.
-size_t pmm_free(struct list_node* list) __NONNULL((1));
+size_t pmm_free(list_node* list);
 
 // Helper routine for the above.
-size_t pmm_free_page(vm_page_t* page) __NONNULL((1));
+size_t pmm_free_page(vm_page_t* page);
 
 // Return count of unallocated physical pages in system
-size_t pmm_count_free_pages(void);
+uint64_t pmm_count_free_pages();
 
 // Return amount of physical memory in system, in bytes.
-size_t pmm_count_total_bytes(void);
+uint64_t pmm_count_total_bytes();
 
 // Counts the number of pages in every state. For every page in every arena,
 // increments the corresponding VM_PAGE_STATE_*-indexed entry of
@@ -70,7 +73,7 @@ void pmm_count_total_states(size_t state_count[_VM_PAGE_STATE_COUNT]);
 // Allocate a run of pages out of the kernel area and return the pointer in kernel space.
 // If the optional list is passed, append the allocate page structures to the tail of the list.
 // If the optional physical address pointer is passed, return the address.
-void* pmm_alloc_kpages(size_t count, struct list_node* list, paddr_t* pa);
+void* pmm_alloc_kpages(size_t count, list_node* list, paddr_t* pa);
 
 // Same as above but a single page at a time
 void* pmm_alloc_kpage(paddr_t* pa, vm_page_t** p);

kernel/kernel/vm/pmm.cpp

@@ -25,6 +25,7 @@
 #include <trace.h>
 
 #include "pmm_arena.h"
+#include "pmm_node.h"
 #include "vm_priv.h"
 
 #include <magenta/thread_annotations.h>
@@ -33,203 +34,53 @@
 
 #define LOCAL_TRACE MAX(VM_GLOBAL_TRACE, 0)
 
-// the main arena list
-static Mutex arena_lock;
-static mxtl::DoublyLinkedList<PmmArena*> arena_list TA_GUARDED(arena_lock);
-static size_t arena_cumulative_size TA_GUARDED(arena_lock);
+// The (currently) one and only pmm node
+static PmmNode pmm_node;
 
 #if PMM_ENABLE_FREE_FILL
 static void pmm_enforce_fill(uint level) {
-    for (auto& a : arena_list) {
-        a.EnforceFill();
-    }
+    pmm_node.EnforceFill();
 }
 LK_INIT_HOOK(pmm_fill, &pmm_enforce_fill, LK_INIT_LEVEL_VM);
 #endif
 
-// We don't need to hold the arena lock while executing this, since it is
-// only accesses values that are set once during system initialization.
-paddr_t vm_page_to_paddr(const vm_page_t* page) TA_NO_THREAD_SAFETY_ANALYSIS {
-    for (const auto& a : arena_list) {
-        // LTRACEF("testing page %p against arena %p\n", page, &a);
-        if (a.page_belongs_to_arena(page)) {
-            return a.page_address_from_arena(page);
-        }
-    }
-    return -1;
+paddr_t vm_page_to_paddr(const vm_page_t* page) {
+    return page->paddr;
 }
 
-// We don't need to hold the arena lock while executing this, since it is
-// only accesses values that are set once during system initialization.
-vm_page_t* paddr_to_vm_page(paddr_t addr) TA_NO_THREAD_SAFETY_ANALYSIS {
-    for (auto& a : arena_list) {
-        if (a.address_in_arena(addr)) {
-            size_t index = (addr - a.base()) / PAGE_SIZE;
-            return a.get_page(index);
-        }
-    }
-    return nullptr;
+vm_page_t* paddr_to_vm_page(paddr_t addr) {
+    return pmm_node.PaddrToPage(addr);
 }
 
-// We disable thread safety analysis here, since this function is only called
-// during early boot before threading exists.
-status_t pmm_add_arena(const pmm_arena_info_t* info) TA_NO_THREAD_SAFETY_ANALYSIS {
-    LTRACEF("arena %p name '%s' base %#" PRIxPTR " size %#zx\n", info, info->name, info->base, info->size);
-
-    // Make sure we're in early boot (ints disabled and no active CPUs according
-    // to the scheduler).
-    DEBUG_ASSERT(mp_get_active_mask() == 0);
-    DEBUG_ASSERT(arch_ints_disabled());
-
-    DEBUG_ASSERT(IS_PAGE_ALIGNED(info->base));
-    DEBUG_ASSERT(IS_PAGE_ALIGNED(info->size));
-    DEBUG_ASSERT(info->size > 0);
-
-    // allocate a c++ arena object
-    PmmArena* arena = new (boot_alloc_mem(sizeof(PmmArena))) PmmArena(info);
-
-    // walk the arena list and add arena based on priority order
-    for (auto& a : arena_list) {
-        if (a.priority() > arena->priority()) {
-            arena_list.insert(a, arena);
-            goto done_add;
-        }
-    }
-
-    // walked off the end, add it to the end of the list
-    arena_list.push_back(arena);
-
-done_add:
-    // tell the arena to allocate a page array
-    arena->BootAllocArray();
-
-    arena_cumulative_size += info->size;
-
-    return MX_OK;
+status_t pmm_add_arena(const pmm_arena_info_t* info) {
+    return pmm_node.AddArena(info);
 }
 
 vm_page_t* pmm_alloc_page(uint alloc_flags, paddr_t* pa) {
-    AutoLock al(&arena_lock);
-
-    /* walk the arenas in order until we find one with a free page */
-    for (auto& a : arena_list) {
-        /* skip the arena if it's not KMAP and the KMAP only allocation flag was passed */
-        if (alloc_flags & PMM_ALLOC_FLAG_KMAP) {
-            if ((a.flags() & PMM_ARENA_FLAG_KMAP) == 0)
-                continue;
-        }
-
-        // try to allocate the page out of the arena
-        vm_page_t* page = a.AllocPage(pa);
-        if (page)
-            return page;
-    }
-
-    LTRACEF("failed to allocate page\n");
-    return nullptr;
+    return pmm_node.AllocPage(alloc_flags, pa);
 }
 
-size_t pmm_alloc_pages(size_t count, uint alloc_flags, struct list_node* list) {
-    LTRACEF("count %zu\n", count);
-
-    /* list must be initialized prior to calling this */
-    DEBUG_ASSERT(list);
-
-    if (count == 0)
-        return 0;
-
-    AutoLock al(&arena_lock);
-
-    /* walk the arenas in order, allocating as many pages as we can from each */
-    size_t allocated = 0;
-    for (auto& a : arena_list) {
-        DEBUG_ASSERT(count > allocated);
-
-        /* skip the arena if it's not KMAP and the KMAP only allocation flag was passed */
-        if (alloc_flags & PMM_ALLOC_FLAG_KMAP) {
-            if ((a.flags() & PMM_ARENA_FLAG_KMAP) == 0)
-                continue;
-        }
-
-        // ask the arena to allocate some pages
-        allocated += a.AllocPages(count - allocated, list);
-        DEBUG_ASSERT(allocated <= count);
-        if (allocated == count)
-            break;
-    }
-
-    return allocated;
+size_t pmm_alloc_pages(size_t count, uint alloc_flags, list_node* list) {
+    return pmm_node.AllocPages(count, alloc_flags, list);
 }
 
-size_t pmm_alloc_range(paddr_t address, size_t count, struct list_node* list) {
-    LTRACEF("address %#" PRIxPTR ", count %zu\n", address, count);
-
-    uint allocated = 0;
-    if (count == 0)
-        return 0;
-
-    address = ROUNDDOWN(address, PAGE_SIZE);
-
-    AutoLock al(&arena_lock);
-
-    /* walk through the arenas, looking to see if the physical page belongs to it */
-    for (auto& a : arena_list) {
-        while (allocated < count && a.address_in_arena(address)) {
-            vm_page_t* page = a.AllocSpecific(address);
-            if (!page)
-                break;
-
-            if (list)
-                list_add_tail(list, &page->free.node);
-
-            allocated++;
-            address += PAGE_SIZE;
-        }
-
-        if (allocated == count)
-            break;
-    }
-
-    return allocated;
+size_t pmm_alloc_range(paddr_t address, size_t count, list_node* list) {
+    return pmm_node.AllocRange(address, count, list);
 }
 
 size_t pmm_alloc_contiguous(size_t count, uint alloc_flags, uint8_t alignment_log2, paddr_t* pa,
-                            struct list_node* list) {
-    LTRACEF("count %zu, align %u\n", count, alignment_log2);
-
-    if (count == 0)
-        return 0;
-    if (alignment_log2 < PAGE_SIZE_SHIFT)
-        alignment_log2 = PAGE_SIZE_SHIFT;
-
-    AutoLock al(&arena_lock);
-
-    for (auto& a : arena_list) {
-        /* skip the arena if it's not KMAP and the KMAP only allocation flag was passed */
-        if (alloc_flags & PMM_ALLOC_FLAG_KMAP) {
-            if ((a.flags() & PMM_ARENA_FLAG_KMAP) == 0)
-                continue;
-        }
-
-        size_t allocated = a.AllocContiguous(count, alignment_log2, pa, list);
-        if (allocated > 0) {
-            DEBUG_ASSERT(allocated == count);
-            return allocated;
-        }
-    }
-
-    LTRACEF("couldn't find run\n");
-    return 0;
+                            list_node* list) {
+    return pmm_node.AllocContiguous(count, alloc_flags, alignment_log2, pa, list);
 }
 
 /* physically allocate a run from arenas marked as KMAP */
-void* pmm_alloc_kpages(size_t count, struct list_node* list, paddr_t* _pa) {
+void* pmm_alloc_kpages(size_t count, list_node* list, paddr_t* _pa) {
     LTRACEF("count %zu\n", count);
 
     paddr_t pa;
     /* fast path for single count allocations */
     if (count == 1) {
-        vm_page_t* p = pmm_alloc_page(PMM_ALLOC_FLAG_KMAP, &pa);
+        vm_page_t* p = pmm_node.AllocPage(PMM_ALLOC_FLAG_KMAP, &pa);
         if (!p)
             return nullptr;
 
@@ -237,7 +88,7 @@ void* pmm_alloc_kpages(size_t count, struct list_node* list, paddr_t* _pa) {
             list_add_tail(list, &p->free.node);
         }
     } else {
-        size_t alloc_count = pmm_alloc_contiguous(count, PMM_ALLOC_FLAG_KMAP, PAGE_SIZE_SHIFT, &pa, list);
+        size_t alloc_count = pmm_node.AllocContiguous(count, PMM_ALLOC_FLAG_KMAP, PAGE_SIZE_SHIFT, &pa, list);
         if (alloc_count == 0)
             return nullptr;
     }
@@ -256,7 +107,7 @@ void* pmm_alloc_kpage(paddr_t* _pa, vm_page_t** _p) {
     LTRACE_ENTRY;
 
     paddr_t pa;
-    vm_page_t* p = pmm_alloc_page(PMM_ALLOC_FLAG_KMAP, &pa);
+    vm_page_t* p = pmm_node.AllocPage(PMM_ALLOC_FLAG_KMAP, &pa);
     if (!p)
         return nullptr;
 
@@ -275,7 +126,7 @@ size_t pmm_free_kpages(void* _ptr, size_t count) {
 
     uint8_t* ptr = (uint8_t*)_ptr;
 
-    struct list_node list;
+    list_node list;
     list_initialize(&list);
 
     while (count > 0) {
@@ -288,38 +139,15 @@ size_t pmm_free_kpages(void* _ptr, size_t count) {
         count--;
     }
 
-    return pmm_free(&list);
+    return pmm_node.Free(&list);
 }
 
-size_t pmm_free(struct list_node* list) {
-    LTRACEF("list %p\n", list);
-
-    DEBUG_ASSERT(list);
-
-    AutoLock al(&arena_lock);
-
-    uint count = 0;
-    while (!list_is_empty(list)) {
-        vm_page_t* page = list_remove_head_type(list, vm_page_t, free.node);
-
-        DEBUG_ASSERT(!page_is_free(page));
-
-        /* see which arena this page belongs to and add it */
-        for (auto& a : arena_list) {
-            if (a.FreePage(page) >= 0) {
-                count++;
-                break;
-            }
-        }
-    }
-
-    LTRACEF("returning count %u\n", count);
-
-    return count;
+size_t pmm_free(list_node* list) {
+    return pmm_node.Free(list);
 }
 
 size_t pmm_free_page(vm_page_t* page) {
-    struct list_node list;
+    list_node list;
     list_initialize(&list);
 
     list_add_head(&list, &page->free.node);
@@ -327,61 +155,24 @@ size_t pmm_free_page(vm_page_t* page) {
     return pmm_free(&list);
 }
 
-static size_t pmm_count_free_pages_locked() TA_REQ(arena_lock) {
-    size_t free = 0u;
-    for (const auto& a : arena_list) {
-        free += a.free_count();
-    }
-    return free;
+uint64_t pmm_count_free_pages() {
+    return pmm_node.CountFreePages();
 }
 
-size_t pmm_count_free_pages() {
-    AutoLock al(&arena_lock);
-    return pmm_count_free_pages_locked();
-}
-
-static void pmm_dump_free() TA_REQ(arena_lock) {
-    auto megabytes_free = pmm_count_free_pages_locked() / 256u;
-    printf(" %zu free MBs\n", megabytes_free);
-}
-
-static size_t pmm_count_total_bytes_locked() TA_REQ(arena_lock) {
-    return arena_cumulative_size;
-}
-
-size_t pmm_count_total_bytes() {
-    AutoLock al(&arena_lock);
-    return pmm_count_total_bytes_locked();
+uint64_t pmm_count_total_bytes() {
+    return pmm_node.CountTotalBytes();
 }
 
 void pmm_count_total_states(size_t state_count[_VM_PAGE_STATE_COUNT]) {
-    // TODO(MG-833): This is extremely expensive, holding a global lock
-    // and touching every page/arena. We should keep a running count instead.
-    AutoLock al(&arena_lock);
-    for (auto& a : arena_list) {
-        a.CountStates(state_count);
-    }
+    pmm_node.CountTotalStates(state_count);
 }
 
-extern "C" enum handler_return pmm_dump_timer(struct timer* t, lk_time_t now, void*) TA_REQ(arena_lock) {
+extern "C" enum handler_return pmm_dump_timer(struct timer* t, lk_time_t now, void*) {
     timer_set_oneshot(t, now + LK_SEC(1), &pmm_dump_timer, nullptr);
-    pmm_dump_free();
+    pmm_node.DumpFree();
     return INT_NO_RESCHEDULE;
 }
 
-// No lock analysis here, as we want to just go for it in the panic case without the lock.
-static void arena_dump(bool is_panic) TA_NO_THREAD_SAFETY_ANALYSIS {
-    if (!is_panic) {
-        arena_lock.Acquire();
-    }
-    for (auto& a : arena_list) {
-        a.Dump(false, false);
-    }
-    if (!is_panic) {
-        arena_lock.Release();
-    }
-}
-
 static int cmd_pmm(int argc, const cmd_args* argv, uint32_t flags) {
     bool is_panic = flags & CMD_FLAG_PANIC;
 
@@ -390,23 +181,15 @@ static int cmd_pmm(int argc, const cmd_args* argv, uint32_t flags) {
         printf("not enough arguments\n");
     usage:
         printf("usage:\n");
-        printf("%s arenas\n", argv[0].str);
+        printf("%s dump\n", argv[0].str);
         if (!is_panic) {
-            printf("%s alloc <count>\n", argv[0].str);
-            printf("%s alloc_range <address> <count>\n", argv[0].str);
-            printf("%s alloc_kpages <count>\n", argv[0].str);
-            printf("%s alloc_contig <count> <alignment>\n", argv[0].str);
-            printf("%s dump_alloced\n", argv[0].str);
-            printf("%s free_alloced\n", argv[0].str);
             printf("%s free\n", argv[0].str);
         }
         return MX_ERR_INTERNAL;
     }
 
-    static struct list_node allocated = LIST_INITIAL_VALUE(allocated);
-
-    if (!strcmp(argv[1].str, "arenas")) {
-        arena_dump(is_panic);
+    if (!strcmp(argv[1].str, "dump")) {
+        pmm_node.Dump(is_panic);
     } else if (is_panic) {
         // No other operations will work during a panic.
         printf("Only the \"arenas\" command is available during a panic.\n");
@@ -424,89 +207,6 @@ static int cmd_pmm(int argc, const cmd_args* argv, uint32_t flags) {
             timer_cancel(&timer);
             show_mem = false;
         }
-    } else if (!strcmp(argv[1].str, "alloc")) {
-        if (argc < 3)
-            goto notenoughargs;
-
-        struct list_node list;
-        list_initialize(&list);
-
-        size_t count = pmm_alloc_pages((uint)argv[2].u, 0, &list);
-        printf("alloc returns %zu\n", count);
-
-        vm_page_t* p;
-        list_for_every_entry (&list, p, vm_page_t, free.node) {
-            paddr_t paddr;
-            {
-                DEBUG_ASSERT(!is_panic);
-                AutoLock al(&arena_lock);
-                paddr = vm_page_to_paddr(p);
-            };
-            printf("\tpage %p, address %#" PRIxPTR "\n", p, paddr);
-        }
-
-        /* add the pages to the local allocated list */
-        struct list_node* node;
-        while ((node = list_remove_head(&list))) {
-            list_add_tail(&allocated, node);
-        }
-    } else if (!strcmp(argv[1].str, "dump_alloced")) {
-        vm_page_t* page;
-
-        list_for_every_entry (&allocated, page, vm_page_t, free.node) { dump_page(page); }
-    } else if (!strcmp(argv[1].str, "alloc_range")) {
-        if (argc < 4)
-            goto notenoughargs;
-
-        struct list_node list;
-        list_initialize(&list);
-
-        size_t count = pmm_alloc_range(argv[2].u, (uint)argv[3].u, &list);
-        printf("alloc returns %zu\n", count);
-
-        vm_page_t* p;
-        list_for_every_entry (&list, p, vm_page_t, free.node) {
-            paddr_t paddr;
-            {
-                DEBUG_ASSERT(!is_panic);
-                AutoLock al(&arena_lock);
-                paddr = vm_page_to_paddr(p);
-            }
-            printf("\tpage %p, address %#" PRIxPTR "\n", p, paddr);
-        }
-
-        /* add the pages to the local allocated list */
-        struct list_node* node;
-        while ((node = list_remove_head(&list))) {
-            list_add_tail(&allocated, node);
-        }
-    } else if (!strcmp(argv[1].str, "alloc_kpages")) {
-        if (argc < 3)
-            goto notenoughargs;
-
-        paddr_t pa;
-        void* ptr = pmm_alloc_kpages((uint)argv[2].u, nullptr, &pa);
-        printf("pmm_alloc_kpages returns %p pa %#" PRIxPTR "\n", ptr, pa);
-    } else if (!strcmp(argv[1].str, "alloc_contig")) {
-        if (argc < 4)
-            goto notenoughargs;
-
-        struct list_node list;
-        list_initialize(&list);
-
-        paddr_t pa;
-        size_t ret = pmm_alloc_contiguous((uint)argv[2].u, 0, (uint8_t)argv[3].u, &pa, &list);
-        printf("pmm_alloc_contiguous returns %zu, address %#" PRIxPTR "\n", ret, pa);
-        printf("address %% align = %#" PRIxPTR "\n", static_cast<uintptr_t>(pa % argv[3].u));
-
-        /* add the pages to the local allocated list */
-        struct list_node* node;
-        while ((node = list_remove_head(&list))) {
-            list_add_tail(&allocated, node);
-        }
-    } else if (!strcmp(argv[1].str, "free_alloced")) {
-        size_t err = pmm_free(&allocated);
-        printf("pmm_free returns %zu\n", err);
     } else {
         printf("unknown command\n");
         goto usage;

kernel/kernel/vm/pmm_arena.cpp

@@ -5,14 +5,15 @@
 // https://opensource.org/licenses/MIT
 #include "pmm_arena.h"
 
-#include "vm_priv.h"
-
 #include <err.h>
 #include <inttypes.h>
 #include <pretty/sizes.h>
 #include <string.h>
 #include <trace.h>
 
+#include "pmm_node.h"
+#include "vm_priv.h"
+
 #define LOCAL_TRACE MAX(VM_GLOBAL_TRACE, 0)
 
 PmmArena::PmmArena(const pmm_arena_info_t* info)
@@ -20,34 +21,7 @@ PmmArena::PmmArena(const pmm_arena_info_t* info)
 
 PmmArena::~PmmArena() {}
 
-#if PMM_ENABLE_FREE_FILL
-void PmmArena::EnforceFill() {
-    DEBUG_ASSERT(!enforce_fill_);
-
-    vm_page_t* page;
-    list_for_every_entry (&free_list_, page, vm_page_t, free.node) {
-        FreeFill(page);
-    }
-
-    enforce_fill_ = true;
-}
-
-void PmmArena::FreeFill(vm_page_t* page) {
-    paddr_t paddr = page_address_from_arena(page);
-    void* kvaddr = paddr_to_kvaddr(paddr);
-    memset(kvaddr, PMM_FREE_FILL_BYTE, PAGE_SIZE);
-}
-
-void PmmArena::CheckFreeFill(vm_page_t* page) {
-    paddr_t paddr = page_address_from_arena(page);
-    uint8_t* kvaddr = static_cast<uint8_t*>(paddr_to_kvaddr(paddr));
-    for (size_t j = 0; j < PAGE_SIZE; ++j) {
-        ASSERT(!enforce_fill_ || *(kvaddr + j) == PMM_FREE_FILL_BYTE);
-    }
-}
-#endif // PMM_ENABLE_FREE_FILL
-
-void PmmArena::BootAllocArray() {
+void PmmArena::BootAllocArray(PmmNode *node) {
     /* allocate an array of pages to back this one */
     size_t page_count = size() / PAGE_SIZE;
     size_t size = page_count * VM_PAGE_STRUCT_SIZE;
@@ -61,43 +35,20 @@ void PmmArena::BootAllocArray() {
     page_array_ = (vm_page_t*)raw_page_array;
 
     /* add them to the free list */
+    list_node list;
+    list_initialize(&list);
     for (size_t i = 0; i < page_count; i++) {
         auto& p = page_array_[i];
+        p.paddr = base() + i * PAGE_SIZE;
+        LTRACEF_LEVEL(2, "p %p, paddr 0x%lx\n", &p, p.paddr);
 
-        list_add_tail(&free_list_, &p.free.node);
+        list_add_tail(&list, &p.node);
     }
 
-    free_count_ += page_count;
+    node->AddFreePages(&list);
 }
 
-vm_page_t* PmmArena::AllocPage(paddr_t* pa) {
-    vm_page_t* page = list_remove_head_type(&free_list_, vm_page_t, free.node);
-    if (!page)
-        return nullptr;
-
-    DEBUG_ASSERT(free_count_ > 0);
-
-    free_count_--;
-
-    DEBUG_ASSERT(page_is_free(page));
-
-    page->state = VM_PAGE_STATE_ALLOC;
-#if PMM_ENABLE_FREE_FILL
-    CheckFreeFill(page);
-#endif
-
-    if (pa) {
-        /* compute the physical address of the page based on its offset into the arena */
-        *pa = page_address_from_arena(page);
-        LTRACEF("pa %#" PRIxPTR ", page %p\n", *pa, page);
-    }
-
-    LTRACEF("allocating page %p, pa %#" PRIxPTR "\n", page, page_address_from_arena(page));
-
-    return page;
-}
-
-vm_page_t* PmmArena::AllocSpecific(paddr_t pa) {
+vm_page_t* PmmArena::FindSpecific(paddr_t pa) {
     if (!address_in_arena(pa))
         return nullptr;
 
@@ -105,52 +56,10 @@ vm_page_t* PmmArena::AllocSpecific(paddr_t pa) {
 
     DEBUG_ASSERT(index < size() / PAGE_SIZE);
 
-    vm_page_t* page = get_page(index);
-    if (!page_is_free(page)) {
-        /* we hit an allocated page */
-        return nullptr;
-    }
-
-    list_delete(&page->free.node);
-
-    page->state = VM_PAGE_STATE_ALLOC;
-
-    DEBUG_ASSERT(free_count_ > 0);
-
-    free_count_--;
-
-    return page;
+    return get_page(index);
 }
 
-size_t PmmArena::AllocPages(size_t count, list_node* list) {
-    size_t allocated = 0;
-
-    while (allocated < count) {
-        vm_page_t* page = list_remove_head_type(&free_list_, vm_page_t, free.node);
-        if (!page)
-            return allocated;
-
-        LTRACEF("allocating page %p, pa %#" PRIxPTR "\n", page, page_address_from_arena(page));
-
-        DEBUG_ASSERT(free_count_ > 0);
-
-        free_count_--;
-
-        DEBUG_ASSERT(page_is_free(page));
-#if PMM_ENABLE_FREE_FILL
-        CheckFreeFill(page);
-#endif
-
-        page->state = VM_PAGE_STATE_ALLOC;
-        list_add_tail(list, &page->free.node);
-
-        allocated++;
-    }
-
-    return allocated;
-}
-
-size_t PmmArena::AllocContiguous(size_t count, uint8_t alignment_log2, paddr_t* pa, struct list_node* list) {
+vm_page_t* PmmArena::FindFreeContiguous(size_t count, uint8_t alignment_log2) {
     /* walk the list starting at alignment boundaries.
      * calculate the starting offset into this arena, based on the
      * base address of the arena to handle the case where the arena
@@ -183,54 +92,13 @@ retry:
         }
 
         /* we found a run */
-        LTRACEF("found run from pn %" PRIuPTR " to %" PRIuPTR "\n", start, start + count);
+        p = &page_array_[start];
+        LTRACEF("found run from pa %#" PRIxPTR " to %#" PRIxPTR "\n", p->paddr, p->paddr + count * PAGE_SIZE);
 
-        /* remove the pages from the run out of the free list */
-        for (paddr_t i = start; i < start + count; i++) {
-            p = &page_array_[i];
-            DEBUG_ASSERT(page_is_free(p));
-            DEBUG_ASSERT(list_in_list(&p->free.node));
-
-            list_delete(&p->free.node);
-            p->state = VM_PAGE_STATE_ALLOC;
-
-            DEBUG_ASSERT(free_count_ > 0);
-
-            free_count_--;
-
-#if PMM_ENABLE_FREE_FILL
-            CheckFreeFill(p);
-#endif
-
-            if (list)
-                list_add_tail(list, &p->free.node);
-        }
-
-        if (pa)
-            *pa = base() + start * PAGE_SIZE;
-
-        return count;
+        return p;
     }
 
-    return 0;
-}
-
-status_t PmmArena::FreePage(vm_page_t* page) {
-    LTRACEF("page %p\n", page);
-    if (!page_belongs_to_arena(page))
-        return MX_ERR_NOT_FOUND;
-
-    DEBUG_ASSERT(page->state != VM_PAGE_STATE_OBJECT || page->object.pin_count == 0);
-
-#if PMM_ENABLE_FREE_FILL
-    FreeFill(page);
-#endif
-
-    page->state = VM_PAGE_STATE_FREE;
-
-    list_add_head(&free_list_, &page->free.node);
-    free_count_++;
-    return MX_OK;
+    return nullptr;
 }
 
 void PmmArena::CountStates(size_t state_count[_VM_PAGE_STATE_COUNT]) const {
@@ -239,11 +107,11 @@ void PmmArena::CountStates(size_t state_count[_VM_PAGE_STATE_COUNT]) const {
     }
 }
 
-void PmmArena::Dump(bool dump_pages, bool dump_free_ranges) {
+void PmmArena::Dump(bool dump_pages, bool dump_free_ranges) const {
     char pbuf[16];
     printf("arena %p: name '%s' base %#" PRIxPTR " size %s (0x%zx) priority %u flags 0x%x\n", this, name(), base(),
            format_size(pbuf, sizeof(pbuf), size()), size(), priority(), flags());
-    printf("\tpage_array %p, free_count %zu\n", page_array_, free_count_);
+    printf("\tpage_array %p\n", page_array_);
 
     /* dump all of the pages */
     if (dump_pages) {

kernel/kernel/vm/pmm_arena.h

@@ -5,14 +5,14 @@
 // https://opensource.org/licenses/MIT
 #pragma once
 
+#include <mxtl/canary.h>
 #include <mxtl/intrusive_double_list.h>
 #include <mxtl/macros.h>
 
 #include <kernel/vm/pmm.h>
 #include <trace.h>
 
-#define PMM_ENABLE_FREE_FILL 0
-#define PMM_FREE_FILL_BYTE 0x42
+class PmmNode;
 
 class PmmArena : public mxtl::DoublyLinkedListable<PmmArena*> {
 public:
@@ -22,13 +22,8 @@ public:
     DISALLOW_COPY_ASSIGN_AND_MOVE(PmmArena);
 
     // set up the per page structures, allocated out of the boot time allocator
-    void BootAllocArray();
-
-#if PMM_ENABLE_FREE_FILL
-    void EnforceFill();
-#endif
-
-    void Dump(bool dump_pages, bool dump_free_ranges);
+    // link them to the free list in the PmmNode
+    void BootAllocArray(PmmNode *);
 
     // accessors
     const pmm_arena_info_t& info() const { return info_; }
@@ -37,7 +32,6 @@ public:
     size_t size() const { return info_.size; }
     unsigned int flags() const { return info_.flags; }
     unsigned int priority() const { return info_.priority; }
-    size_t free_count() const { return free_count_; };
 
     // Counts the number of pages in every state. For each page in the arena,
     // increments the corresponding VM_PAGE_STATE_*-indexed entry of
@@ -46,46 +40,26 @@ public:
 
     vm_page_t* get_page(size_t index) { return &page_array_[index]; }
 
-    // main allocation routines
-    vm_page_t* AllocPage(paddr_t* pa);
-    vm_page_t* AllocSpecific(paddr_t pa);
-    size_t AllocPages(size_t count, list_node* list);
-    size_t AllocContiguous(size_t count, uint8_t alignment_log2, paddr_t* pa, struct list_node* list);
-    status_t FreePage(vm_page_t* page);
+    // find a free run of contiguous pages
+    vm_page_t* FindFreeContiguous(size_t count, uint8_t alignment_log2);
+
+    // return a pointer to a specific page
+    vm_page_t* FindSpecific(paddr_t pa);
 
     // helpers
     bool page_belongs_to_arena(const vm_page* page) const {
-        uintptr_t page_addr = reinterpret_cast<uintptr_t>(page);
-        uintptr_t page_array_base = reinterpret_cast<uintptr_t>(page_array_);
-
-        return ((page_addr >= page_array_base) &&
-                (page_addr < (page_array_base + (info_.size / PAGE_SIZE) * VM_PAGE_STRUCT_SIZE)));
-    }
-
-    paddr_t page_address_from_arena(const vm_page* page) const {
-        uintptr_t page_addr = reinterpret_cast<uintptr_t>(page);
-        uintptr_t page_array_base = reinterpret_cast<uintptr_t>(page_array_);
-
-        return ((paddr_t)(((page_addr - page_array_base) / VM_PAGE_STRUCT_SIZE) * PAGE_SIZE + info_.base));
+        return (page->paddr >= base() && page->paddr < (base() + size()));
     }
 
     bool address_in_arena(paddr_t address) const {
         return (address >= info_.base && address <= info_.base + info_.size - 1);
     }
 
+    void Dump(bool dump_pages, bool dump_free_ranges) const;
+
 private:
-#if PMM_ENABLE_FREE_FILL
-    void FreeFill(vm_page_t* page);
-    void CheckFreeFill(vm_page_t* page);
-#endif
+    mxtl::Canary<mxtl::magic("PARN")> canary_;
 
     const pmm_arena_info_t info_;
     vm_page_t* page_array_ = nullptr;
-
-    size_t free_count_ = 0;
-    list_node free_list_ = LIST_INITIAL_VALUE(free_list_);
-
-#if PMM_ENABLE_FREE_FILL
-    bool enforce_fill_ = false;
-#endif
 };

kernel/kernel/vm/pmm_node.cpp

@@ -0,0 +1,322 @@
+// Copyright 2017 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+#include "pmm_node.h"
+
+#include <kernel/mp.h>
+#include <mxcpp/new.h>
+#include <trace.h>
+
+#include "vm_priv.h"
+
+#define LOCAL_TRACE MAX(VM_GLOBAL_TRACE, 0)
+
+PmmNode::PmmNode() {
+}
+
+PmmNode::~PmmNode() {
+}
+
+// We disable thread safety analysis here, since this function is only called
+// during early boot before threading exists.
+status_t PmmNode::AddArena(const pmm_arena_info_t* info) TA_NO_THREAD_SAFETY_ANALYSIS {
+    LTRACEF("arena %p name '%s' base %#" PRIxPTR " size %#zx\n", info, info->name, info->base, info->size);
+
+    // Make sure we're in early boot (ints disabled and no active CPUs according
+    // to the scheduler).
+    DEBUG_ASSERT(mp_get_active_mask() == 0);
+    DEBUG_ASSERT(arch_ints_disabled());
+
+    DEBUG_ASSERT(IS_PAGE_ALIGNED(info->base));
+    DEBUG_ASSERT(IS_PAGE_ALIGNED(info->size));
+    DEBUG_ASSERT(info->size > 0);
+
+    // allocate a c++ arena object
+    PmmArena* arena = new (boot_alloc_mem(sizeof(PmmArena))) PmmArena(info);
+
+    // walk the arena list and add arena based on priority order
+    for (auto& a : arena_list_) {
+        if (a.priority() > arena->priority()) {
+            arena_list_.insert(a, arena);
+            goto done_add;
+        }
+    }
+
+    // walked off the end, add it to the end of the list
+    arena_list_.push_back(arena);
+
+done_add:
+    // tell the arena to allocate a page array and all the pages to us
+    arena->BootAllocArray(this);
+
+    arena_cumulative_size_ += info->size;
+
+    return MX_OK;
+}
+
+void PmmNode::AddFreePages(list_node *list) {
+    LTRACEF("list %p\n", list);
+
+    vm_page_t *temp, *page;
+    list_for_every_entry_safe(list, page, temp, vm_page_t, node) {
+        list_delete(&page->node);
+        list_add_tail(&free_list_, &page->node);
+        free_count_++;
+    }
+
+    LTRACEF("free count now %" PRIu64 "\n", free_count_);
+}
+
+vm_page_t* PmmNode::AllocPage(uint alloc_flags, paddr_t* pa) {
+    AutoLock al(&lock_);
+
+    vm_page_t* page = list_remove_head_type(&free_list_, vm_page_t, node);
+    if (!page)
+        return nullptr;
+
+    DEBUG_ASSERT(free_count_ > 0);
+
+    free_count_--;
+
+    DEBUG_ASSERT(page_is_free(page));
+
+    page->state = VM_PAGE_STATE_ALLOC;
+#if PMM_ENABLE_FREE_FILL
+    CheckFreeFill(page);
+#endif
+
+    if (pa) {
+        *pa = page->paddr;
+    }
+
+    LTRACEF("allocating page %p, pa %#" PRIxPTR "\n", page, page->paddr);
+
+    return page;
+}
+
+size_t PmmNode::AllocPages(size_t count, uint alloc_flags, list_node* list) {
+    LTRACEF("count %zu\n", count);
+
+    /* list must be initialized prior to calling this */
+    DEBUG_ASSERT(list);
+
+    if (count == 0)
+        return 0;
+
+    AutoLock al(&lock_);
+
+    size_t allocated = 0;
+    while (allocated < count) {
+        vm_page_t* page = list_remove_head_type(&free_list_, vm_page_t, node);
+        if (!page)
+            return allocated;
+
+        LTRACEF("allocating page %p, pa %#" PRIxPTR "\n", page, page->paddr);
+
+        DEBUG_ASSERT(free_count_ > 0);
+
+        free_count_--;
+
+        DEBUG_ASSERT(page_is_free(page));
+#if PMM_ENABLE_FREE_FILL
+        CheckFreeFill(page);
+#endif
+
+        page->state = VM_PAGE_STATE_ALLOC;
+        list_add_tail(list, &page->free.node);
+
+        allocated++;
+    }
+
+    return allocated;
+}
+
+size_t PmmNode::AllocRange(paddr_t address, size_t count, list_node* list) {
+    LTRACEF("address %#" PRIxPTR ", count %zu\n", address, count);
+
+    size_t allocated = 0;
+    if (count == 0)
+        return 0;
+
+    address = ROUNDDOWN(address, PAGE_SIZE);
+
+    AutoLock al(&lock_);
+
+    /* walk through the arenas, looking to see if the physical page belongs to it */
+    for (auto& a : arena_list_) {
+        while (allocated < count && a.address_in_arena(address)) {
+            vm_page_t* page = a.FindSpecific(address);
+            if (!page)
+                break;
+
+            if (!page_is_free(page))
+                break;
+
+            list_delete(&page->node);
+
+            page->state = VM_PAGE_STATE_ALLOC;
+
+            if (list)
+                list_add_tail(list, &page->free.node);
+
+            allocated++;
+            address += PAGE_SIZE;
+            free_count_--;
+        }
+
+        if (allocated == count)
+            break;
+    }
+
+    LTRACEF("returning allocated count %zu\n", allocated);
+
+    return allocated;
+}
+
+size_t PmmNode::AllocContiguous(const size_t count, uint alloc_flags, uint8_t alignment_log2,
+                                paddr_t* pa, list_node* list) {
+    LTRACEF("count %zu, align %u\n", count, alignment_log2);
+
+    if (count == 0)
+        return 0;
+    if (alignment_log2 < PAGE_SIZE_SHIFT)
+        alignment_log2 = PAGE_SIZE_SHIFT;
+
+    AutoLock al(&lock_);
+
+    for (auto& a : arena_list_) {
+        vm_page_t* p = a.FindFreeContiguous(count, alignment_log2);
+        if (!p)
+            continue;
+
+        if (pa)
+            *pa = p->paddr;
+
+        /* remove the pages from the run out of the free list */
+        for (size_t i = 0; i < count; i++, p++) {
+            DEBUG_ASSERT_MSG(page_is_free(p), "p %p state %u\n", p, p->state);
+            DEBUG_ASSERT(list_in_list(&p->node));
+
+            list_delete(&p->node);
+            p->state = VM_PAGE_STATE_ALLOC;
+
+            DEBUG_ASSERT(free_count_ > 0);
+
+            free_count_--;
+
+#if PMM_ENABLE_FREE_FILL
+            CheckFreeFill(p);
+#endif
+
+            if (list)
+                list_add_tail(list, &p->free.node);
+        }
+
+        return count;
+    }
+
+    LTRACEF("couldn't find run\n");
+    return 0;
+}
+
+size_t PmmNode::Free(list_node* list) {
+    LTRACEF("list %p\n", list);
+
+    DEBUG_ASSERT(list);
+
+    AutoLock al(&lock_);
+
+    uint count = 0;
+    while (!list_is_empty(list)) {
+        vm_page_t* page = list_remove_head_type(list, vm_page_t, free.node);
+
+        DEBUG_ASSERT(page->state != VM_PAGE_STATE_OBJECT || page->object.pin_count == 0);
+        DEBUG_ASSERT(!page_is_free(page));
+
+#if PMM_ENABLE_FREE_FILL
+        FreeFill(page);
+#endif
+
+        // remove it from its old queue
+        if (list_in_list(&page->node))
+            list_delete(&page->node);
+
+        // mark it free
+        page->state = VM_PAGE_STATE_FREE;
+
+        // add it to the free queue
+        list_add_head(&free_list_, &page->node);
+
+        free_count_++;
+        count++;
+    }
+
+    LTRACEF("returning count %u\n", count);
+
+    return count;
+}
+
+uint64_t PmmNode::CountFreePages() const {
+    return free_count_;
+}
+
+uint64_t PmmNode::CountTotalBytes() const {
+    return arena_cumulative_size_;
+}
+
+void PmmNode::CountTotalStates(size_t state_count[_VM_PAGE_STATE_COUNT]) const {
+    // TODO(MG-833): This is extremely expensive, holding a global lock
+    // and touching every page/arena. We should keep a running count instead.
+    AutoLock al(&lock_);
+    for (auto& a : arena_list_) {
+        a.CountStates(state_count);
+    }
+}
+
+void PmmNode::DumpFree() const TA_NO_THREAD_SAFETY_ANALYSIS {
+    auto megabytes_free = CountFreePages() / 256u;
+    printf(" %zu free MBs\n", megabytes_free);
+}
+
+// No lock analysis here, as we want to just go for it in the panic case without the lock.
+void PmmNode::Dump(bool is_panic) const TA_NO_THREAD_SAFETY_ANALYSIS {
+    if (!is_panic) {
+        lock_.Acquire();
+    }
+    printf("pmm node %p: free_count %zu\n", this, free_count_);
+    for (auto& a : arena_list_) {
+        a.Dump(false, false);
+    }
+    if (!is_panic) {
+        lock_.Release();
+    }
+}
+
+#if PMM_ENABLE_FREE_FILL
+void PmmNode::EnforceFill() {
+    DEBUG_ASSERT(!enforce_fill_);
+
+    vm_page_t* page;
+    list_for_every_entry (&free_list_, page, vm_page_t, node) {
+        FreeFill(page);
+    }
+
+    enforce_fill_ = true;
+}
+
+void PmmNode::FreeFill(vm_page_t* page) {
+    void* kvaddr = paddr_to_kvaddr(page->paddr);
+    DEBUG_ASSERT(is_kernel_address((vaddr_t)kvaddr));
+    memset(kvaddr, PMM_FREE_FILL_BYTE, PAGE_SIZE);
+}
+
+void PmmNode::CheckFreeFill(vm_page_t* page) {
+    uint8_t* kvaddr = static_cast<uint8_t*>(paddr_to_kvaddr(page->paddr));
+    for (size_t j = 0; j < PAGE_SIZE; ++j) {
+        ASSERT(!enforce_fill_ || *(kvaddr + j) == PMM_FREE_FILL_BYTE);
+    }
+}
+#endif // PMM_ENABLE_FREE_FILL
+

kernel/kernel/vm/pmm_node.h

@@ -0,0 +1,92 @@
+// Copyright 2017 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+#pragma once
+
+#include <mxtl/canary.h>
+#include <mxtl/intrusive_double_list.h>
+
+#include <kernel/mutex.h>
+#include <kernel/vm/pmm.h>
+
+#include "pmm_arena.h"
+
+#define PMM_ENABLE_FREE_FILL 0
+#define PMM_FREE_FILL_BYTE 0x42
+
+// per numa node collection of pmm arenas and worker threads
+class PmmNode {
+public:
+    PmmNode();
+    ~PmmNode();
+
+    DISALLOW_COPY_ASSIGN_AND_MOVE(PmmNode);
+
+    paddr_t PageToPaddr(const vm_page_t* page) TA_NO_THREAD_SAFETY_ANALYSIS;
+    vm_page_t* PaddrToPage(paddr_t addr) TA_NO_THREAD_SAFETY_ANALYSIS;
+
+    // main allocator routines
+    vm_page_t* AllocPage(uint alloc_flags, paddr_t* pa);
+    size_t AllocPages(size_t count, uint alloc_flags, list_node* list);
+    size_t AllocRange(paddr_t address, size_t count, list_node* list);
+    size_t AllocContiguous(size_t count, uint alloc_flags, uint8_t alignment_log2, paddr_t* pa, list_node* list);
+    size_t Free(list_node* list);
+
+    uint64_t CountFreePages() const;
+    uint64_t CountTotalBytes() const;
+    void CountTotalStates(size_t state_count[_VM_PAGE_STATE_COUNT]) const;
+
+    // printf free and overall state of the internal arenas
+    // NOTE: both functions skip mutexes and can be called inside timer or crash context
+    // though the data they return may be questionable
+    void DumpFree() const TA_NO_THREAD_SAFETY_ANALYSIS;
+    void Dump(bool is_panic) const TA_NO_THREAD_SAFETY_ANALYSIS;
+
+#if PMM_ENABLE_FREE_FILL
+    void EnforceFill() TA_NO_THREAD_SAFETY_ANALYSIS;
+#endif
+
+    status_t AddArena(const pmm_arena_info_t* info);
+
+    // add new pages to the free queue. used when boostrapping a PmmArena
+    void AddFreePages(list_node *list);
+
+private:
+    mxtl::Canary<mxtl::magic("PNOD")> canary_;
+
+    mutable Mutex lock_;
+
+    uint64_t arena_cumulative_size_ = 0;
+    uint64_t free_count_ = 0;
+
+    mxtl::DoublyLinkedList<PmmArena*> arena_list_ TA_GUARDED(lock_);
+
+    // page queues
+    list_node free_list_ TA_GUARDED(lock_) = LIST_INITIAL_VALUE(free_list_);
+    list_node inactive_list_ TA_GUARDED(lock_) = LIST_INITIAL_VALUE(inactive_list_);
+    list_node active_list_ TA_GUARDED(lock_) = LIST_INITIAL_VALUE(active_list_);
+    list_node modified_list_ TA_GUARDED(lock_) = LIST_INITIAL_VALUE(modified_list_);
+    list_node wired_list_ TA_GUARDED(lock_) = LIST_INITIAL_VALUE(wired_list_);
+
+#if PMM_ENABLE_FREE_FILL
+    void FreeFill(vm_page_t* page);
+    void CheckFreeFill(vm_page_t* page);
+
+    bool enforce_fill_ = false;
+#endif
+
+};
+
+// We don't need to hold the arena lock while executing this, since it is
+// only accesses values that are set once during system initialization.
+inline vm_page_t* PmmNode::PaddrToPage(paddr_t addr) TA_NO_THREAD_SAFETY_ANALYSIS {
+    for (auto& a : arena_list_) {
+        if (a.address_in_arena(addr)) {
+            size_t index = (addr - a.base()) / PAGE_SIZE;
+            return a.get_page(index);
+        }
+    }
+    return nullptr;
+}

kernel/kernel/vm/rules.mk

@@ -20,6 +20,7 @@ MODULE_SRCS += \
     $(LOCAL_DIR)/page.cpp \
     $(LOCAL_DIR)/pmm.cpp \
     $(LOCAL_DIR)/pmm_arena.cpp \
+    $(LOCAL_DIR)/pmm_node.cpp \
     $(LOCAL_DIR)/vm.cpp \
     $(LOCAL_DIR)/vm_address_region.cpp \
     $(LOCAL_DIR)/vm_address_region_or_mapping.cpp \