diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 33b92b3961f199..546e30b877c9a7 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -10,8 +10,6 @@ #include #include -#include -#include #include #include #include @@ -443,66 +441,22 @@ void __init xen_add_extra_mem(unsigned long start_pfn, unsigned long n_pfns) } #ifdef CONFIG_XEN_UNPOPULATED_ALLOC -/* - * How much of the extra memory dom0 keeps on the balloon, to grow into up - * to the ceiling Xen set for it (dom0_mem=...,max:). Without a max: Xen - * reports UINT_MAX and dom0 keeps nothing, so growing stays opt-in. The - * last memory section always stays with the mapping pool, so that the - * grant table and other early users never fall back to holes in the - * physical map before PCI devices have claimed their MMIO windows. - */ -static unsigned long __init xen_dom0_growth_pages(unsigned long extra_pages) -{ - domid_t domid = DOMID_SELF; - long max_pages, cur_pages; - - if (!xen_initial_domain() || extra_pages <= PAGES_PER_SECTION) - return 0; - - max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid); - cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid); - if (cur_pages <= 0 || max_pages <= cur_pages || max_pages >= UINT_MAX) - return 0; - - return min_t(unsigned long, max_pages - cur_pages, - extra_pages - PAGES_PER_SECTION); -} - int __init arch_xen_unpopulated_init(struct resource **res) { - unsigned long extra_pages = 0, growth_pages; unsigned int i; if (!xen_domain()) return -ENODEV; - /* Must be set strictly before calling xen_free_unpopulated_pages(). */ *res = &iomem_resource; - for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) - extra_pages += xen_extra_mem[i].n_pfns; - growth_pages = xen_dom0_growth_pages(extra_pages); - /* - * Initialize with pages from the extra memory regions (see - * arch/x86/xen/setup.c), less what dom0 keeps to grow into. Every - * region gives up the same fraction: on PVH that spreads dom0's - * growth over the NUMA nodes the regions belong to, and leaves each - * node's pool its share of the rest. + * Initialize with the extra memory regions (see + * arch/x86/xen/setup.c). */ for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { - struct xen_memory_region *region = &xen_extra_mem[i]; - unsigned long keep = 0, j; - - if (growth_pages) - keep = mult_frac(region->n_pfns, growth_pages, - extra_pages); - - for (j = keep; j < region->n_pfns; j++) { - struct page *pg = pfn_to_page(region->start_pfn + j); - - xen_free_unpopulated_pages(1, &pg); - } + xen_unpopulated_add_range(xen_extra_mem[i].start_pfn, + xen_extra_mem[i].n_pfns); /* * Account for the region being in the physmap but unpopulated. @@ -510,9 +464,9 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_unpopulated_pages += region->n_pfns - keep; - /* Only the part dom0 keeps is added to the balloon driver. */ - region->n_pfns = keep; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; + /* Zero so region is not also added to the balloon driver. */ + xen_extra_mem[i].n_pfns = 0; } return 0; diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 4f1813964af9b1..e7168805814b8c 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -355,6 +355,7 @@ config XEN_FRONT_PGDIR_SHBUF config XEN_UNPOPULATED_ALLOC bool "Use unpopulated memory ranges for guest mappings" depends on ZONE_DEVICE + select GENERIC_ALLOCATOR default XEN_BACKEND || XEN_GNTDEV || XEN_DOM0 help Use unpopulated memory ranges in order to create mappings for guest diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 1f2997cdd6c074..6bbb0b0887da58 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -632,6 +632,48 @@ static enum bp_state balloon_deflate(unsigned long nr_pages) return increase_reservation(nr_pages); } +/* + * Grow dom0 into 2 MiB blocks of the unpopulated allocator's address space, + * rather than hotplugging memory at the first free physical address, which + * can belong to a PCI device that has not claimed it yet. The blocks are + * parked on the balloon only for this call and deflated like any others, and + * whatever Xen did not populate goes straight back, so the balloon never + * holds address space grant mappings could use. + */ +static enum bp_state balloon_grow_dom0(unsigned long nr_pages) +{ + unsigned long before = balloon_stats.current_pages; + struct page *heads[BALLOON_BLOCK_BATCH], *page; + unsigned int nr = 1, i; + enum bp_state state; + + BUILD_BUG_ON(XEN_UNPOPULATED_BLOCK_PAGES != BALLOON_BLOCK_NR_PAGES); + + /* + * Take as many blocks as balloon_deflate() populates whole, or one for + * it to populate a page at a time. + */ + if (balloon_blocks_usable()) + nr = clamp(nr_pages / BALLOON_BLOCK_NR_PAGES, 1UL, + ARRAY_SIZE(heads)); + nr = xen_alloc_unpopulated_blocks(heads, nr); + if (!nr) { + pr_warn_once("no room left for dom0 to grow into\n"); + return BP_ECANCELED; + } + + for (i = 0; i < nr; i++) + balloon_append_block(heads[i]); + + state = balloon_deflate(nr_pages); + balloon_stats.total_pages += balloon_stats.current_pages - before; + + while ((page = balloon_retrieve(false))) + xen_free_unpopulated_pages(1, &page); + + return state; +} + static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) { enum bp_state state = BP_DONE; @@ -830,6 +872,8 @@ static int balloon_thread(void *unused) if (credit > 0) { if (balloon_is_inflated()) balloon_state = balloon_deflate(credit); + else if (xen_initial_domain()) + balloon_state = balloon_grow_dom0(credit); else balloon_state = reserve_additional_memory(); } @@ -968,6 +1012,31 @@ void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages) } EXPORT_SYMBOL(xen_free_ballooned_pages); +/* + * How many pages dom0 can still grow by: the 2 MiB blocks the unpopulated + * allocator has to spare, within Xen's ceiling for dom0 (dom0_mem=...,max:) + * when one is set. Without a max: Xen reports UINT_MAX. + */ +unsigned long balloon_room_pages(void) +{ + domid_t domid = DOMID_SELF; + long max_pages, cur_pages; + unsigned long room; + + if (!xen_initial_domain()) + return 0; + + room = xen_unpopulated_spare_blocks() * XEN_UNPOPULATED_BLOCK_PAGES; + + max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid); + cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid); + if (max_pages > 0 && max_pages < UINT_MAX && cur_pages > 0) + room = min_t(unsigned long, room, + max(max_pages - cur_pages, 0L)); + + return room; +} + #define BALLOON_OOM_DEFLATE_BATCH 512UL /* pages per OOM event (~2 MiB) */ static int balloon_oom_notify(struct notifier_block *nb, unsigned long dummy, @@ -1024,26 +1093,10 @@ static struct notifier_block balloon_oom_nb = { .notifier_call = balloon_oom_notify, }; -static void __init balloon_append_range(unsigned long pfn, - unsigned long end_pfn) -{ - while (pfn < end_pfn) { - if (balloon_blocks_usable() && - IS_ALIGNED(pfn, BALLOON_BLOCK_NR_PAGES) && - end_pfn - pfn >= BALLOON_BLOCK_NR_PAGES) { - balloon_append_block(pfn_to_page(pfn)); - pfn += BALLOON_BLOCK_NR_PAGES; - } else { - balloon_append(pfn_to_page(pfn)); - pfn++; - } - } -} - static int __init balloon_add_regions(bool append) { unsigned long start_pfn, pages; - unsigned long extra_pfn_end; + unsigned long pfn, extra_pfn_end; unsigned int i; for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { @@ -1060,7 +1113,8 @@ static int __init balloon_add_regions(bool append) */ extra_pfn_end = min(max_pfn, start_pfn + pages); - balloon_append_range(start_pfn, extra_pfn_end); + for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) + balloon_append(pfn_to_page(pfn)); /* * There are two different use-cases depending on how the diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index e0a2b5cd1423de..ff7211c17e6f05 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include #include #include +#include #include #include @@ -24,10 +26,41 @@ * * A single mutex covers all per-node lists. Alloc/free are * connect-time events on the I/O backends and not contention-sensitive. + * + * The boot-time unbacked address space, which arch_xen_unpopulated_init() + * hands over with xen_unpopulated_add_range(), is not ZONE_DEVICE but RAM + * pfns with nothing behind them. It is kept in a gen_pool per node, whose + * addresses are pfns, so that a page given back joins its neighbours again. + * Mappings take pages from page_list[] first, then from the gen_pool. + * + * dom0's balloon takes aligned 2 MiB blocks from the gen_pools, which Xen can + * populate as one extent each, and leaves each node at least + * MAPPING_FLOOR_PAGES free pages. */ static DEFINE_MUTEX(list_lock); static struct page *page_list[MAX_NUMNODES]; static unsigned int list_count[MAX_NUMNODES]; +static struct gen_pool *boot_pool[MAX_NUMNODES]; + +/* + * Each allocation searches a gen_pool chunk from its first page, and skips a + * chunk with too few free pages without searching it. Boot-time address + * space goes into the pools in chunks of at most this many pages, so that + * chunks used up are skipped whole. + */ +#define BOOT_CHUNK_PAGES (SZ_1G >> PAGE_SHIFT) + +/* + * Free pages of each node that dom0's balloon leaves to mappings, so that + * they rarely need fill_list() to carve a region out of address space that + * may belong to a device. + */ +#define MAPPING_FLOOR_PAGES PAGES_PER_SECTION + +/* Align blocks on the pfn, not on the offset into the gen_pool chunk. */ +static struct genpool_data_align block_align = { + .align = XEN_UNPOPULATED_BLOCK_PAGES, +}; static struct resource *target_resource; @@ -180,6 +213,51 @@ static int fill_list(unsigned int nr_pages, int node) return ret; } +/* Put @pg on the pool of its own node. Called with list_lock held. */ +static void push_page(struct page *pg) +{ + int node = xen_unpopulated_clamp_node(page_to_nid(pg)); + + if (!is_zone_device_page(pg)) { + gen_pool_free(boot_pool[node], page_to_pfn(pg), 1); + return; + } + + pg->zone_device_data = page_list[node]; + page_list[node] = pg; + list_count[node]++; +} + +/* + * Take a page off the pool of @node: from page_list[] first, then from the + * boot-time address space. Returns NULL when the pool is empty. + */ +static struct page *take_page(int node) +{ + struct page *pg = page_list[node]; + unsigned long pfn = 0; + + if (pg) { + page_list[node] = pg->zone_device_data; + list_count[node]--; + return pg; + } + + if (boot_pool[node]) + pfn = gen_pool_alloc(boot_pool[node], 1); + return pfn ? pfn_to_page(pfn) : NULL; +} + +/* Free pages in the pool of @node, boot-time ones included. */ +static unsigned long pool_pages(int node) +{ + unsigned long nr = list_count[node]; + + if (boot_pool[node]) + nr += gen_pool_avail(boot_pool[node]); + return nr; +} + /* * Take @nr_pages from the pool of @node, or from any node's pool when @any is * set. Called with list_lock held. Never carves a new region. @@ -187,23 +265,21 @@ static int fill_list(unsigned int nr_pages, int node) static int pop_pages(unsigned int nr_pages, struct page **pages, int node, bool any) { - unsigned int i, total = 0; + unsigned long total = 0; + unsigned int i; for (i = 0; i < MAX_NUMNODES; i++) - total += list_count[i]; - if ((any ? total : list_count[node]) < nr_pages) + total += pool_pages(i); + if ((any ? total : pool_pages(node)) < nr_pages) return -ENOMEM; for (i = 0; i < nr_pages; i++) { int src = node; struct page *pg; - while (!page_list[src]) + while (!(pg = take_page(src))) src = (src + 1) % MAX_NUMNODES; - pg = page_list[src]; - page_list[src] = pg->zone_device_data; - list_count[src]--; pages[i] = pg; #ifdef CONFIG_XEN_HAVE_PVMMU @@ -213,14 +289,8 @@ static int pop_pages(unsigned int nr_pages, struct page **pages, int node, if (ret < 0) { unsigned int j; - for (j = 0; j <= i; j++) { - int dst = xen_unpopulated_clamp_node( - page_to_nid(pages[j])); - - pages[j]->zone_device_data = page_list[dst]; - page_list[dst] = pages[j]; - list_count[dst]++; - } + for (j = 0; j <= i; j++) + push_page(pages[j]); return ret; } } @@ -283,8 +353,9 @@ EXPORT_SYMBOL(xen_alloc_unpopulated_pages_node); */ int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) { - unsigned int i, total = 0; + unsigned long total = 0; int node = numa_node_id(); + unsigned int i; int ret; if (!target_resource) @@ -292,7 +363,7 @@ int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) mutex_lock(&list_lock); for (i = 0; i < MAX_NUMNODES; i++) - total += list_count[i]; + total += pool_pages(i); ret = 0; if (total < nr_pages) ret = fill_list(nr_pages - total, node); @@ -324,16 +395,141 @@ void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages) } mutex_lock(&list_lock); - for (i = 0; i < nr_pages; i++) { - int node = xen_unpopulated_clamp_node(page_to_nid(pages[i])); + for (i = 0; i < nr_pages; i++) + push_page(pages[i]); + mutex_unlock(&list_lock); +} +EXPORT_SYMBOL(xen_free_unpopulated_pages); - pages[i]->zone_device_data = page_list[node]; - page_list[node] = pages[i]; - list_count[node]++; +/** + * xen_unpopulated_add_range - hand over boot-time unbacked address space + * @start_pfn: first pfn of the range + * @nr_pages: number of pages in the range + * + * The pages must have a struct page and nothing behind them, as the extra + * memory regions of a Xen guest do. They go to the gen_pool of their node in + * chunks of at most BOOT_CHUNK_PAGES, and a chunk never spans two nodes. + */ +void __init xen_unpopulated_add_range(unsigned long start_pfn, + unsigned long nr_pages) +{ + unsigned long pfn, end, end_pfn = start_pfn + nr_pages; + + mutex_lock(&list_lock); + for (pfn = start_pfn; pfn < end_pfn; pfn = end) { + int node = page_to_nid(pfn_to_page(pfn)); + + /* + * Freed pages go back to the pool of their own node, so a + * chunk must not hold pages of two nodes. When the chunk ends + * on another node, move its end back to the last page of this + * node, which assumes no other node's pages come before it. + */ + end = min(ALIGN(pfn + 1, BOOT_CHUNK_PAGES), end_pfn); + while (page_to_nid(pfn_to_page(end - 1)) != node) + end--; + + if (!boot_pool[node]) + boot_pool[node] = gen_pool_create(0, node); + if (!boot_pool[node] || + gen_pool_add(boot_pool[node], pfn, end - pfn, node)) + pr_warn("Cannot keep pfns %#lx-%#lx for mappings\n", + pfn, end - 1); } mutex_unlock(&list_lock); } -EXPORT_SYMBOL(xen_free_unpopulated_pages); + +/* + * Take a 2 MiB block off the boot-time address space of @node, if @node keeps + * MAPPING_FLOOR_PAGES free pages after it. Returns NULL otherwise. + */ +static struct page *take_block(int node) +{ + unsigned long pfn = 0; + + if (boot_pool[node] && + pool_pages(node) >= MAPPING_FLOOR_PAGES + XEN_UNPOPULATED_BLOCK_PAGES) + pfn = gen_pool_alloc_algo(boot_pool[node], + XEN_UNPOPULATED_BLOCK_PAGES, + gen_pool_first_fit_align, &block_align); + return pfn ? pfn_to_page(pfn) : NULL; +} + +/** + * xen_alloc_unpopulated_blocks - take 2 MiB blocks for dom0's balloon + * @heads: first page of each block taken + * @nr_blocks: number of blocks wanted + * + * Blocks come from the boot-time address space of the lowest node that can + * spare one and still keep MAPPING_FLOOR_PAGES free pages. Give back the + * pages Xen did not populate with xen_free_unpopulated_pages(). + * + * Returns the number of blocks taken, which may be less than @nr_blocks. + */ +unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks) +{ + unsigned int nr = 0; + int node; + + mutex_lock(&list_lock); + for (node = 0; node < MAX_NUMNODES && nr < nr_blocks; node++) { + struct page *head; + + while (nr < nr_blocks && (head = take_block(node))) + heads[nr++] = head; + } + mutex_unlock(&list_lock); + + return nr; +} + +/* Add the free 2 MiB blocks of @chunk to *@data. */ +static void count_free_blocks(struct gen_pool *pool, + struct gen_pool_chunk *chunk, void *data) +{ + unsigned long nr_pages = chunk->end_addr - chunk->start_addr + 1; + unsigned long *count = data; + unsigned long b, e; + + for_each_clear_bitrange(b, e, chunk->bits, nr_pages) { + unsigned long first = ALIGN(chunk->start_addr + b, + XEN_UNPOPULATED_BLOCK_PAGES); + unsigned long end = ALIGN_DOWN(chunk->start_addr + e, + XEN_UNPOPULATED_BLOCK_PAGES); + + if (end > first) + *count += (end - first) / XEN_UNPOPULATED_BLOCK_PAGES; + } +} + +/** + * xen_unpopulated_spare_blocks - 2 MiB blocks dom0's balloon could still take + * + * Each node spares the free 2 MiB blocks of its boot-time address space, up to + * what it can give and still keep MAPPING_FLOOR_PAGES free pages. + */ +unsigned long xen_unpopulated_spare_blocks(void) +{ + unsigned long nr = 0; + int node; + + mutex_lock(&list_lock); + for (node = 0; node < MAX_NUMNODES; node++) { + unsigned long blocks = 0, nr_free = pool_pages(node); + + if (!boot_pool[node] || nr_free <= MAPPING_FLOOR_PAGES) + continue; + + gen_pool_for_each_chunk(boot_pool[node], count_free_blocks, + &blocks); + nr += min(blocks, (nr_free - MAPPING_FLOOR_PAGES) / + XEN_UNPOPULATED_BLOCK_PAGES); + } + mutex_unlock(&list_lock); + + return nr; +} static int __init unpopulated_init(void) { diff --git a/drivers/xen/xen-balloon.c b/drivers/xen/xen-balloon.c index 5ed11aa71c60f8..53159d3224ab84 100644 --- a/drivers/xen/xen-balloon.c +++ b/drivers/xen/xen-balloon.c @@ -146,6 +146,7 @@ BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(balloon_stats.current_pages)); BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_low)); BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_high)); BALLOON_SHOW(blocks, "%lu\n", balloon_stats.balloon_blocks); +BALLOON_SHOW(room_kb, "%lu\n", PAGES2KB(balloon_room_pages())); static DEVICE_ULONG_ATTR(schedule_delay, 0444, balloon_stats.schedule_delay); static DEVICE_ULONG_ATTR(max_schedule_delay, 0644, balloon_stats.max_schedule_delay); @@ -225,6 +226,7 @@ static struct attribute *balloon_info_attrs[] = { &dev_attr_low_kb.attr, &dev_attr_high_kb.attr, &dev_attr_blocks.attr, + &dev_attr_room_kb.attr, NULL }; diff --git a/include/xen/balloon.h b/include/xen/balloon.h index 4702026337e554..00a44b9d75162d 100644 --- a/include/xen/balloon.h +++ b/include/xen/balloon.h @@ -27,6 +27,7 @@ struct balloon_stats { extern struct balloon_stats balloon_stats; void balloon_set_new_target(unsigned long target); +unsigned long balloon_room_pages(void); int xen_alloc_ballooned_pages(unsigned int nr_pages, struct page **pages); void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages); diff --git a/include/xen/xen.h b/include/xen/xen.h index ccf9bfa338131c..c1eec9f7329558 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -68,12 +68,19 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, extern u64 xen_saved_max_mem_size; #endif +/* Pages in a 2 MiB block of unbacked address space, one order-9 Xen extent. */ +#define XEN_UNPOPULATED_BLOCK_PAGES (1UL << (21 - PAGE_SHIFT)) + #ifdef CONFIG_XEN_UNPOPULATED_ALLOC extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages_node(unsigned int nr_pages, struct page **pages, int node); int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); +void xen_unpopulated_add_range(unsigned long start_pfn, unsigned long nr_pages); +unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks); +unsigned long xen_unpopulated_spare_blocks(void); #include int arch_xen_unpopulated_init(struct resource **res); #else @@ -94,6 +101,15 @@ static inline void xen_free_unpopulated_pages(unsigned int nr_pages, { xen_free_ballooned_pages(nr_pages, pages); } +static inline unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks) +{ + return 0; +} +static inline unsigned long xen_unpopulated_spare_blocks(void) +{ + return 0; +} #endif /*