xen/dom0: refactor dom0 memory growth to avoid splitting based on max

arch/x86/xen/enlighten.c

@@ -10,8 +10,6 @@
 
 #include <xen/xen.h>
 #include <xen/features.h>
-#include <xen/interface/xen.h>
-#include <xen/interface/memory.h>
 #include <xen/interface/sched.h>
 #include <xen/interface/version.h>
 #include <xen/page.h>
@@ -443,76 +441,32 @@ void __init xen_add_extra_mem(unsigned long start_pfn, unsigned long n_pfns)
 }
 
 #ifdef CONFIG_XEN_UNPOPULATED_ALLOC
-/*
- * How much of the extra memory dom0 keeps on the balloon, to grow into up
- * to the ceiling Xen set for it (dom0_mem=...,max:). Without a max: Xen
- * reports UINT_MAX and dom0 keeps nothing, so growing stays opt-in. The
- * last memory section always stays with the mapping pool, so that the
- * grant table and other early users never fall back to holes in the
- * physical map before PCI devices have claimed their MMIO windows.
- */
-static unsigned long __init xen_dom0_growth_pages(unsigned long extra_pages)
-{
-	domid_t domid = DOMID_SELF;
-	long max_pages, cur_pages;
-
-	if (!xen_initial_domain() || extra_pages <= PAGES_PER_SECTION)
-		return 0;
-
-	max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid);
-	cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid);
-	if (cur_pages <= 0 || max_pages <= cur_pages || max_pages >= UINT_MAX)
-		return 0;
-
-	return min_t(unsigned long, max_pages - cur_pages,
-		     extra_pages - PAGES_PER_SECTION);
-}
-
 int __init arch_xen_unpopulated_init(struct resource **res)
 {
-	unsigned long extra_pages = 0, growth_pages;
 	unsigned int i;
 
 	if (!xen_domain())
 		return -ENODEV;
 
-	/* Must be set strictly before calling xen_free_unpopulated_pages(). */
 	*res = &iomem_resource;
 
-	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++)
-		extra_pages += xen_extra_mem[i].n_pfns;
-	growth_pages = xen_dom0_growth_pages(extra_pages);
-
 	/*
-	 * Initialize with pages from the extra memory regions (see
-	 * arch/x86/xen/setup.c), less what dom0 keeps to grow into. Every
-	 * region gives up the same fraction: on PVH that spreads dom0's
-	 * growth over the NUMA nodes the regions belong to, and leaves each
-	 * node's pool its share of the rest.
+	 * Initialize with the extra memory regions (see
+	 * arch/x86/xen/setup.c).
 	 */
 	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
-		struct xen_memory_region *region = &xen_extra_mem[i];
-		unsigned long keep = 0, j;
-
-		if (growth_pages)
-			keep = mult_frac(region->n_pfns, growth_pages,
-					 extra_pages);
-
-		for (j = keep; j < region->n_pfns; j++) {
-			struct page *pg = pfn_to_page(region->start_pfn + j);
-
-			xen_free_unpopulated_pages(1, &pg);
-		}
+		xen_unpopulated_add_range(xen_extra_mem[i].start_pfn,
+					  xen_extra_mem[i].n_pfns);
 
 		/*
 		 * Account for the region being in the physmap but unpopulated.
 		 * The value in xen_released_pages is used by the balloon
 		 * driver to know how much of the physmap is unpopulated and
 		 * set an accurate initial memory target.
 		 */
-		xen_unpopulated_pages += region->n_pfns - keep;
-		/* Only the part dom0 keeps is added to the balloon driver. */
-		region->n_pfns = keep;
+		xen_unpopulated_pages += xen_extra_mem[i].n_pfns;
+		/* Zero so region is not also added to the balloon driver. */
+		xen_extra_mem[i].n_pfns = 0;
 	}
 
 	return 0;

drivers/xen/Kconfig

@@ -355,6 +355,7 @@ config XEN_FRONT_PGDIR_SHBUF
 config XEN_UNPOPULATED_ALLOC
 	bool "Use unpopulated memory ranges for guest mappings"
 	depends on ZONE_DEVICE
+	select GENERIC_ALLOCATOR
 	default XEN_BACKEND || XEN_GNTDEV || XEN_DOM0
 	help
 	  Use unpopulated memory ranges in order to create mappings for guest

drivers/xen/balloon.c

@@ -632,6 +632,48 @@ static enum bp_state balloon_deflate(unsigned long nr_pages)
 	return increase_reservation(nr_pages);
 }
 
+/*
+ * Grow dom0 into 2 MiB blocks of the unpopulated allocator's address space,
+ * rather than hotplugging memory at the first free physical address, which
+ * can belong to a PCI device that has not claimed it yet. The blocks are
+ * parked on the balloon only for this call and deflated like any others, and
+ * whatever Xen did not populate goes straight back, so the balloon never
+ * holds address space grant mappings could use.
+ */
+static enum bp_state balloon_grow_dom0(unsigned long nr_pages)
+{
+	unsigned long before = balloon_stats.current_pages;
+	struct page *heads[BALLOON_BLOCK_BATCH], *page;
+	unsigned int nr = 1, i;
+	enum bp_state state;
+
+	BUILD_BUG_ON(XEN_UNPOPULATED_BLOCK_PAGES != BALLOON_BLOCK_NR_PAGES);
+
+	/*
+	 * Take as many blocks as balloon_deflate() populates whole, or one for
+	 * it to populate a page at a time.
+	 */
+	if (balloon_blocks_usable())
+		nr = clamp(nr_pages / BALLOON_BLOCK_NR_PAGES, 1UL,
+			   ARRAY_SIZE(heads));
+	nr = xen_alloc_unpopulated_blocks(heads, nr);
+	if (!nr) {
+		pr_warn_once("no room left for dom0 to grow into\n");
+		return BP_ECANCELED;
+	}
+
+	for (i = 0; i < nr; i++)
+		balloon_append_block(heads[i]);
+
+	state = balloon_deflate(nr_pages);
+	balloon_stats.total_pages += balloon_stats.current_pages - before;
+
+	while ((page = balloon_retrieve(false)))
+		xen_free_unpopulated_pages(1, &page);
+
+	return state;
+}
+
 static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
 {
 	enum bp_state state = BP_DONE;
@@ -830,6 +872,8 @@ static int balloon_thread(void *unused)
 		if (credit > 0) {
 			if (balloon_is_inflated())
 				balloon_state = balloon_deflate(credit);
+			else if (xen_initial_domain())
+				balloon_state = balloon_grow_dom0(credit);
 			else
 				balloon_state = reserve_additional_memory();
 		}
@@ -968,6 +1012,31 @@ void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages)
 }
 EXPORT_SYMBOL(xen_free_ballooned_pages);
 
+/*
+ * How many pages dom0 can still grow by: the 2 MiB blocks the unpopulated
+ * allocator has to spare, within Xen's ceiling for dom0 (dom0_mem=...,max:)
+ * when one is set. Without a max: Xen reports UINT_MAX.
+ */
+unsigned long balloon_room_pages(void)
+{
+	domid_t domid = DOMID_SELF;
+	long max_pages, cur_pages;
+	unsigned long room;
+
+	if (!xen_initial_domain())
+		return 0;
+
+	room = xen_unpopulated_spare_blocks() * XEN_UNPOPULATED_BLOCK_PAGES;
+
+	max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid);
+	cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid);
+	if (max_pages > 0 && max_pages < UINT_MAX && cur_pages > 0)
+		room = min_t(unsigned long, room,
+			     max(max_pages - cur_pages, 0L));
+
+	return room;
+}
+
 #define BALLOON_OOM_DEFLATE_BATCH 512UL /* pages per OOM event (~2 MiB) */
 
 static int balloon_oom_notify(struct notifier_block *nb, unsigned long dummy,
@@ -1024,26 +1093,10 @@ static struct notifier_block balloon_oom_nb = {
 	.notifier_call = balloon_oom_notify,
 };
 
-static void __init balloon_append_range(unsigned long pfn,
-					unsigned long end_pfn)
-{
-	while (pfn < end_pfn) {
-		if (balloon_blocks_usable() &&
-		    IS_ALIGNED(pfn, BALLOON_BLOCK_NR_PAGES) &&
-		    end_pfn - pfn >= BALLOON_BLOCK_NR_PAGES) {
-			balloon_append_block(pfn_to_page(pfn));
-			pfn += BALLOON_BLOCK_NR_PAGES;
-		} else {
-			balloon_append(pfn_to_page(pfn));
-			pfn++;
-		}
-	}
-}
-
 static int __init balloon_add_regions(bool append)
 {
 	unsigned long start_pfn, pages;
-	unsigned long extra_pfn_end;
+	unsigned long pfn, extra_pfn_end;
 	unsigned int i;
 
 	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
@@ -1060,7 +1113,8 @@ static int __init balloon_add_regions(bool append)
 		 */
 		extra_pfn_end = min(max_pfn, start_pfn + pages);
 
-		balloon_append_range(start_pfn, extra_pfn_end);
+		for (pfn = start_pfn; pfn < extra_pfn_end; pfn++)
+			balloon_append(pfn_to_page(pfn));
 
 		/*
 		 * There are two different use-cases depending on how the