8388561: [s390x] Prevent accidental page table expansion

Reviewed-by: amitkumar, rtoyonaga

src/hotspot/cpu/s390/compressedKlass_s390.cpp

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026, IBM Corp. All rights reserved.
+ * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,14 +25,13 @@
  */
 
 #include "oops/compressedKlass.hpp"
+#include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size_t size, bool aslr, bool optimize_for_zero_base) {
 
   char* result = nullptr;
 
-  uintptr_t tried_below = 0;
-
   // First, attempt to allocate < 4GB. We do this unconditionally:
   // - if optimize_for_zero_base, a <4GB mapping start allows us to use base=0 shift=0
   // - if !optimize_for_zero_base, a <4GB mapping start allows us to use algfi
@@ -44,7 +44,9 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size
 
   // Failing that, aim for a base that is 4G-aligned; such a base can be set with aih.
   if (result == nullptr) {
-    result = reserve_address_space_for_16bit_move(size, aslr);
+    constexpr uintptr_t from = nth_bit<uintptr_t>(32);
+    const uintptr_t to = os::vm_page_table_expansion_point(); // prevent accidentally expanding the page table
+    result = reserve_address_space_X(from, to, size, nth_bit<uintptr_t>(32), aslr);
   }
 
   return result;

src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp

@@ -477,3 +477,10 @@ int os::extra_bang_size_in_bytes() {
 }
 
 void os::setup_fpu() {}
+
+uintptr_t os::vm_page_table_expansion_point() {
+  // On s390x, page table will dynamically expand based on user demand
+  // (eg mmap probing with high addresses). First expansion happens
+  // at 2^42 (4TB).
+  return nth_bit<uintptr_t>(42);
+}

src/hotspot/share/memory/memoryReserver.cpp

@@ -471,10 +471,13 @@ static char** get_attach_addresses_for_disjoint_mode() {
   }
   uint start = i;
 
-  // Avoid more steps than requested.
+  // Avoid more steps than requested, and avoid expanding the page table
+  // by over-eager probing.
   i = 0;
   while (addresses[start+i] != 0) {
-    if (i == HeapSearchSteps) {
+    if (i == HeapSearchSteps ||
+       (addresses[start+i] >= os::vm_page_table_expansion_point()))
+    {
       addresses[start+i] = 0;
       break;
     }

src/hotspot/share/runtime/os.cpp

@@ -1994,6 +1994,13 @@ static void shuffle_fisher_yates(T* arr, unsigned num, FastRandom& frand) {
   }
 }
 
+#ifndef S390
+// Default implementation: page table never expands.
+uintptr_t os::vm_page_table_expansion_point() {
+  return align_down(UINTPTR_MAX, os::vm_allocation_granularity());
+}
+#endif
+
 // Helper for os::attempt_reserve_memory_between
 // Given an array of things, do a hemisphere split such that the resulting
 // order is: [first, last, first + 1, last - 1, ...]

src/hotspot/share/runtime/os.hpp

@@ -503,6 +503,11 @@ class os: AllStatic {
   // Returns the lowest address the process is allowed to map against.
   static size_t vm_min_address();
 
+  // Some kernels (e.g. s390x) can dynamically expand the page table. This function returns
+  // the lowest user space address that will expand the page table for the first time.
+  // We typically want to avoid expanding the page table unless it is really necessary.
+  static uintptr_t vm_page_table_expansion_point();
+
   // Returns an upper limit beyond which reserve_memory() calls are guaranteed
   // to fail. It is not guaranteed that reserving less memory than this will
   // succeed, however.

test/hotspot/jtreg/runtime/CompressedOops/CompressedCPUSpecificClassSpaceReservation.java

@@ -43,6 +43,9 @@ import jdk.test.lib.process.ProcessTools;
 import jtreg.SkippedException;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class CompressedCPUSpecificClassSpaceReservation {
     // Note: windows: On windows, we currently have the issue that os::reserve_memory_aligned relies on
@@ -68,7 +71,11 @@ public class CompressedCPUSpecificClassSpaceReservation {
         final String tryReserveForUnscaled = "reserve_between (range [0x0000000000000000-0x0000000100000000)";
         final String tryReserveBelow4G = "reserve_between (range [0x0000000000000000-0x0000000100000000)";
         final String tryReserveForZeroBased = "reserve_between (range [0x0000000100000000-0x0000000800000000)";
-        final String tryReserveFor16bitMoveIntoQ3 = "reserve_between (range [0x0000000100000000-0x0001000000000000)";
+        // Failing zero-based allocation, platforms will often attempt allocation suitable for a disjointed move:
+        // an insert of the base address bits into the register holding the nK. That requires the base to not
+        // intersect with nK bits (for simplicity, we always assume nK range of 32bits).
+        // The upper limit of this reservation attempt is platform-dependent, though.
+        final String tryReserveFor16bitMoveIntoQ3Regex = "reserve_between.*0x0000000100000000-0x\\d{8}00000000.*alignment 0x100000000";
         if (Platform.isAArch64()) {
             if (CDS) {
                 output.shouldNotContain(tryReserveForUnscaled);
@@ -77,7 +84,7 @@ public class CompressedCPUSpecificClassSpaceReservation {
             }
             output.shouldContain("Trying to reserve at an EOR-compatible address");
             output.shouldNotContain(tryReserveForZeroBased);
-            output.shouldContain(tryReserveFor16bitMoveIntoQ3);
+            output.shouldMatch(tryReserveFor16bitMoveIntoQ3Regex);
         } else if (Platform.isPPC()) {
             if (CDS) {
                 output.shouldNotContain(tryReserveForUnscaled);
@@ -86,7 +93,7 @@ public class CompressedCPUSpecificClassSpaceReservation {
                 output.shouldContain(tryReserveForUnscaled);
                 output.shouldContain(tryReserveForZeroBased);
             }
-            output.shouldContain(tryReserveFor16bitMoveIntoQ3);
+            output.shouldMatch(tryReserveFor16bitMoveIntoQ3Regex);
         } else if (Platform.isRISCV64()) {
             output.shouldContain(tryReserveForUnscaled); // unconditionally
             // bits 32..44
@@ -100,7 +107,7 @@ public class CompressedCPUSpecificClassSpaceReservation {
             } else {
                 output.shouldContain(tryReserveForZeroBased);
             }
-            output.shouldContain(tryReserveFor16bitMoveIntoQ3);
+            output.shouldMatch(tryReserveFor16bitMoveIntoQ3Regex);
         } else if (Platform.isX64()) {
             output.shouldContain(tryReserveBelow4G);
             if (CDS) {
@@ -119,6 +126,24 @@ public class CompressedCPUSpecificClassSpaceReservation {
             output.shouldContain("CDS archive(s) not mapped");
         }
         output.shouldContain("Compressed class space mapped at:");
+
+        // S390: Make very sure every reserve_between attempt we do (which we do
+        // for class space only, currently) never probes beyond 2^42 to avoid
+        // page table expansion
+        if (Platform.isS390x()) {
+            Pattern pat = Pattern.compile(".*reserve_between \\(range \\[0x[0-9a-f]{16}-0x([0-9a-f]{16})\\).*");
+            List<Matcher> matches = output.matchersForAllMatchingLinesStdout(pat);
+            if (matches.size() == 0) {
+                throw new RuntimeException("Expected matches");
+            }
+            for (Matcher mat : matches) {
+                long address_to = Long.parseLong(mat.group(1), 16);
+                if (address_to > Math.powExact(2L, 42)) {
+                    System.out.println(mat.group(0));
+                    throw new RuntimeException("Address space probing beyond 2^42?");
+                }
+            }
+        }
     }
 
     public static void main(String[] args) throws Exception {

test/lib/jdk/test/lib/process/OutputAnalyzer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -563,6 +563,32 @@ public final class OutputAnalyzer {
         return firstMatch(pattern, 0);
     }
 
+    private List<Matcher> matchersForAllMatchingLinesInternal(Pattern needleRegex, List<String> haystack) {
+        List<Matcher> matchingMatchers = haystack.stream().
+                map(needleRegex::matcher).filter(Matcher::matches).collect(Collectors.toList());
+        return matchingMatchers;
+    }
+
+    /**
+     * Given a regular expression, matches the expression against every *line* of stderr output
+     * and returns a list of all matching matchers.
+     * @param needleRegex
+     * @return List of matchers
+     */
+    public  List<Matcher> matchersForAllMatchingLinesStderr(Pattern needleRegex) {
+        return matchersForAllMatchingLinesInternal(needleRegex, stderrAsLines());
+    }
+
+    /**
+     * Given a regular expression, matches the expression against every *line* of stdout output
+     * and returns a list of all matching matchers.
+     * @param needleRegex
+     * @return List of matchers
+     */
+    public  List<Matcher> matchersForAllMatchingLinesStdout(Pattern needleRegex) {
+        return matchersForAllMatchingLinesInternal(needleRegex, stdoutAsLines());
+    }
+
     /**
      * Verify the exit value of the process
      *