diff --git a/test/hotspot/jtreg/gc/TestTransparentHugePagesHeap.java b/test/hotspot/jtreg/gc/TestTransparentHugePagesHeap.java index 2c2e19b87c4..68909ef1c98 100644 --- a/test/hotspot/jtreg/gc/TestTransparentHugePagesHeap.java +++ b/test/hotspot/jtreg/gc/TestTransparentHugePagesHeap.java @@ -49,11 +49,7 @@ * @run driver TestTransparentHugePagesHeap Serial */ -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.Scanner; @@ -62,6 +58,8 @@ import jdk.test.lib.os.linux.HugePageConfiguration; import jdk.test.lib.process.OutputAnalyzer; import jdk.test.lib.process.ProcessTools; import jdk.test.lib.Platform; +import jdk.test.lib.os.linux.Smaps; +import jdk.test.lib.os.linux.Smaps.Range; import jtreg.SkippedException; @@ -100,82 +98,37 @@ public class TestTransparentHugePagesHeap { public static void main(String args[]) throws Exception { // Extract the heap start from pagesize logging - BigInteger heapStart = extractHeapStartFromLog(); + String heapStart = extractHeapStartFromLog(); - Path smaps = makeSmapsCopy(); + Smaps smaps = Smaps.parseSelf(); - final Pattern addressRangePattern = Pattern.compile("([0-9a-f]*?)-([0-9a-f]*?) .*"); - final Pattern thpEligiblePattern = Pattern.compile("THPeligible:\\s+(\\d)\\s*"); - - Scanner smapsFile = new Scanner(smaps); - while (smapsFile.hasNextLine()) { - Matcher addressRangeMatcher = addressRangePattern.matcher(smapsFile.nextLine()); - if (!addressRangeMatcher.matches()) { - continue; - } - - // Found an address range line in the smaps file - - BigInteger addressStart = new BigInteger(addressRangeMatcher.group(1), 16); - BigInteger addressEnd = new BigInteger(addressRangeMatcher.group(2), 16); - - // Linux sometimes merges adjacent VMAs so we can't search for a range that - // exactly matches the heap range. Instead we look for the first range that - // contains the start of the heap and verify that that range is THP eligible. - - if (addressStart.compareTo(heapStart) > 0 || heapStart.compareTo(addressEnd) >= 0) { - continue; - } - - // Found a range that contains the start of the heap, verify that it is THP eligible. - - while (smapsFile.hasNextLine()) { - Matcher m = thpEligiblePattern.matcher(smapsFile.nextLine()); - if (!m.matches()) { - continue; - } - - // Found the THPeligible line - - if (m.group(1).equals("1")) { - // Success - THPeligible is 1, heap can be backed by huge pages - return; - } - - throw new RuntimeException("The address range 0x" + addressStart.toString(16) - + "-0x" + addressEnd.toString(16) - + " that contains the heap start" + heapStart - + " is not THPeligible"); - } - - throw new RuntimeException("Couldn't find THPeligible in the smaps file"); + Range range = smaps.getRange(heapStart); + if (range == null) { + throw new AssertionError("Could not find heap section in smaps file. No memory range found for heap start: " + heapStart); } - throw new RuntimeException("Could not find an address range containing the heap start " + heapStart + " in the smaps file"); + if (!range.isTransparentHuge()) { + // Failed to verify THP for heap + throw new RuntimeException("The address range 0x" + range.getStart().toString(16) + + "-0x" + range.getEnd().toString(16) + + " that contains the heap start" + heapStart + + " is not THPeligible"); + } } - private static BigInteger extractHeapStartFromLog() throws Exception { + private static String extractHeapStartFromLog() throws Exception { // [0.041s][info][pagesize] Heap: min=128M max=128M base=0x0000ffff5c600000 size=128M page_size=2M final Pattern heapAddress = Pattern.compile(".* Heap: .*base=0x([0-9A-Fa-f]*).*"); Scanner logFile = new Scanner(Paths.get("thp-" + ProcessHandle.current().pid() + ".log")); while (logFile.hasNextLine()) { - String line = logFile.nextLine(); - - Matcher m = heapAddress.matcher(line); + Matcher m = heapAddress.matcher(logFile.nextLine()); if (m.matches()) { - return new BigInteger(m.group(1), 16); + return m.group(1); } } throw new RuntimeException("Failed to find heap start"); } - - private static Path makeSmapsCopy() throws Exception { - Path src = Paths.get("/proc/self/smaps"); - Path dest = Paths.get("smaps-copy-" + ProcessHandle.current().pid() + ".txt"); - Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING); - return dest; - } } } diff --git a/test/hotspot/jtreg/runtime/os/TestTracePageSizes.java b/test/hotspot/jtreg/runtime/os/TestTracePageSizes.java index d325468b2bb..b0f07f71188 100644 --- a/test/hotspot/jtreg/runtime/os/TestTracePageSizes.java +++ b/test/hotspot/jtreg/runtime/os/TestTracePageSizes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -104,135 +104,21 @@ */ import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.LinkedList; import java.util.Scanner; import java.util.regex.Matcher; import java.util.regex.Pattern; import jdk.test.lib.Platform; +import jdk.test.lib.os.linux.Smaps; +import jdk.test.lib.os.linux.Smaps.Range; import jtreg.SkippedException; // Check that page sizes logged match what is recorded in /proc/self/smaps. // For transparent huge pages the matching is best effort since we can't // know for sure what the underlying page size is. public class TestTracePageSizes { - // Store address ranges with known page size. - private static LinkedList ranges = new LinkedList<>(); private static boolean debug; private static int run; - // Copy smaps locally - // (To minimize chances of concurrent modification when parsing, as well as helping with error analysis) - private static Path copySmaps() throws Exception { - Path p1 = Paths.get("/proc/self/smaps"); - Path p2 = Paths.get("smaps-copy-" + ProcessHandle.current().pid() + "-" + (run++) + ".txt"); - Files.copy(p1, p2, StandardCopyOption.REPLACE_EXISTING); - debug("Copied " + p1 + " to " + p2 + "..."); - return p2; - } - - // Parse /proc/self/smaps. - private static void parseSmaps() throws Exception { - // We can override the smaps file to parse to pass in a pre-fetched one - String smapsFileToParse = System.getProperty("smaps-file"); - if (smapsFileToParse != null) { - parseSmaps(Paths.get(smapsFileToParse)); - } else { - Path smapsCopy = copySmaps(); - parseSmaps(smapsCopy); - } - } - - static class SmapsParser { - // This is a simple smaps parser; it will recognize smaps section start lines - // (e.g. "40fa00000-439b80000 rw-p 00000000 00:00 0 ") and look for keywords inside the section. - // Section will be finished and written into a RangeWithPageSize when either the next section is found - // or the end of file is encountered. - static final Pattern SECTION_START_PATT = Pattern.compile("^([a-f0-9]+)-([a-f0-9]+) [\\-rwpsx]{4}.*"); - static final Pattern KERNEL_PAGESIZE_PATT = Pattern.compile("^KernelPageSize:\\s*(\\d*) kB"); - static final Pattern THP_ELIGIBLE_PATT = Pattern.compile("^THPeligible:\\s+(\\d*)"); - static final Pattern VMFLAGS_PATT = Pattern.compile("^VmFlags: ([\\w\\? ]*)"); - String start; - String end; - String ps; - String thpEligible; - String vmFlags; - int lineno; - - void reset() { - start = null; - end = null; - ps = null; - thpEligible = null; - vmFlags = null; - } - - public void finish() { - if (start != null) { - RangeWithPageSize range = new RangeWithPageSize(start, end, ps, thpEligible, vmFlags); - ranges.add(range); - debug("Added range: " + range); - reset(); - } - } - - void eatNext(String line) { - // For better debugging experience call finish here before the debug() call. - Matcher matSectionStart = SECTION_START_PATT.matcher(line); - if (matSectionStart.matches()) { - finish(); - } - - debug("" + (lineno++) + " " + line); - - if (matSectionStart.matches()) { - start = matSectionStart.group(1); - end = matSectionStart.group(2); - ps = null; - vmFlags = null; - return; - } else { - Matcher matKernelPageSize = KERNEL_PAGESIZE_PATT.matcher(line); - if (matKernelPageSize.matches()) { - ps = matKernelPageSize.group(1); - return; - } - Matcher matTHPEligible = THP_ELIGIBLE_PATT.matcher(line); - if (matTHPEligible.matches()) { - thpEligible = matTHPEligible.group(1); - return; - } - Matcher matVmFlags = VMFLAGS_PATT.matcher(line); - if (matVmFlags.matches()) { - vmFlags = matVmFlags.group(1); - return; - } - } - } - } - - // Parse /proc/self/smaps - private static void parseSmaps(Path smapsFileToParse) throws Exception { - debug("Parsing: " + smapsFileToParse.getFileName() + "..."); - SmapsParser parser = new SmapsParser(); - Files.lines(smapsFileToParse).forEach(parser::eatNext); - parser.finish(); - } - - // Search for a range including the given address. - private static RangeWithPageSize getRange(String addr) { - long laddr = Long.decode(addr); - for (RangeWithPageSize range : ranges) { - if (range.includes(laddr)) { - return range; - } - } - return null; - } - // Helper to get the page size in KB given a page size parsed // from log_info(pagesize) output. private static long pageSizeInKB(String pageSize) { @@ -272,7 +158,7 @@ public class TestTracePageSizes { } // Parse /proc/self/smaps to compare with values logged in the VM. - parseSmaps(); + Smaps smaps = Smaps.parseSelf(); // Setup patters for the JVM page size logging. String traceLinePatternString = ".*base=(0x[0-9A-Fa-f]*).* page_size=(\\d+[BKMG]).*"; @@ -289,7 +175,7 @@ public class TestTracePageSizes { String address = trace.group(1); String pageSize = trace.group(2); - RangeWithPageSize range = getRange(address); + Range range = smaps.getRange(address); if (range == null) { debug("Could not find range for: " + line); throw new AssertionError("No memory range found for address: " + address); @@ -325,75 +211,3 @@ public class TestTracePageSizes { } } } - -// Class used to store information about memory ranges parsed -// from /proc/self/smaps. The file contain a lot of information -// about the different mappings done by an application, but the -// lines we care about are: -// 700000000-73ea00000 rw-p 00000000 00:00 0 -// ... -// KernelPageSize: 4 kB -// ... -// VmFlags: rd wr mr mw me ac sd -// -// We use the VmFlags to know what kind of huge pages are used. -// For transparent huge pages the KernelPageSize field will not -// report the large page size. -class RangeWithPageSize { - private long start; - private long end; - private long pageSize; - private boolean thpEligible; - private boolean vmFlagHG; - private boolean vmFlagHT; - private boolean isTHP; - - public RangeWithPageSize(String start, String end, String pageSize, String thpEligible, String vmFlags) { - // Note: since we insist on kernels >= 3.8, all the following information should be present - // (none of the input strings be null). - this.start = Long.parseUnsignedLong(start, 16); - this.end = Long.parseUnsignedLong(end, 16); - this.pageSize = Long.parseLong(pageSize); - this.thpEligible = thpEligible == null ? false : (Integer.parseInt(thpEligible) == 1); - - vmFlagHG = false; - vmFlagHT = false; - // Check if the vmFlags line include: - // * ht - Meaning the range is mapped using explicit huge pages. - // * hg - Meaning the range is madvised huge. - for (String flag : vmFlags.split(" ")) { - if (flag.equals("ht")) { - vmFlagHT = true; - } else if (flag.equals("hg")) { - vmFlagHG = true; - } - } - - // When the THP policy is 'always' instead of 'madvise, the vmFlagHG property is false, - // therefore also check thpEligible. If this is still causing problems in the future, - // we might have to check the AnonHugePages field. - - isTHP = vmFlagHG || this.thpEligible; - } - - public long getPageSize() { - return pageSize; - } - - public boolean isTransparentHuge() { - return isTHP; - } - - public boolean isExplicitHuge() { - return vmFlagHT; - } - - public boolean includes(long addr) { - return start <= addr && addr < end; - } - - public String toString() { - return "[" + Long.toHexString(start) + ", " + Long.toHexString(end) + ") " + - "pageSize=" + pageSize + "KB isTHP=" + isTHP + " isHUGETLB=" + vmFlagHT; - } -} diff --git a/test/lib/jdk/test/lib/os/linux/Smaps.java b/test/lib/jdk/test/lib/os/linux/Smaps.java new file mode 100644 index 00000000000..11cdfe34319 --- /dev/null +++ b/test/lib/jdk/test/lib/os/linux/Smaps.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.test.lib.os.linux; + +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Smaps { + + // List of memory ranges + private List ranges; + + protected Smaps(List ranges) { + this.ranges = ranges; + } + + // Search for a range including the given address. + public Range getRange(String addr) { + BigInteger laddr = new BigInteger(addr.substring(2), 16); + for (Range range : ranges) { + if (range.includes(laddr)) { + return range; + } + } + + return null; + } + + public static Smaps parseSelf() throws Exception { + return parse(Path.of("/proc/self/smaps")); + } + + public static Smaps parse(Path smaps) throws Exception { + return new Parser(smaps).parse(); + } + + // This is a simple smaps parser; it will recognize smaps section start lines + // (e.g. "40fa00000-439b80000 rw-p 00000000 00:00 0 ") and look for keywords inside the section. + // Section will be finished and written into a RangeWithPageSize when either the next section is found + // or the end of file is encountered. + private static class Parser { + + private static final Pattern SECTION_START_PATT = Pattern.compile("^([a-f0-9]+)-([a-f0-9]+) [\\-rwpsx]{4}.*"); + private static final Pattern KERNEL_PAGESIZE_PATT = Pattern.compile("^KernelPageSize:\\s*(\\d*) kB"); + private static final Pattern THP_ELIGIBLE_PATT = Pattern.compile("^THPeligible:\\s+(\\d*)"); + private static final Pattern VMFLAGS_PATT = Pattern.compile("^VmFlags: ([\\w\\? ]*)"); + + String start; + String end; + String ps; + String thpEligible; + String vmFlags; + + List ranges; + Path smaps; + + Parser(Path smaps) { + this.ranges = new LinkedList(); + this.smaps = smaps; + reset(); + } + + void reset() { + start = null; + end = null; + ps = null; + thpEligible = null; + vmFlags = null; + } + + public void finish() { + if (start != null) { + Range range = new Range(start, end, ps, thpEligible, vmFlags); + ranges.add(range); + reset(); + } + } + + public void eatNext(String line) { + // For better debugging experience call finish here before the debug() call. + Matcher matSectionStart = SECTION_START_PATT.matcher(line); + if (matSectionStart.matches()) { + finish(); + } + + if (matSectionStart.matches()) { + start = matSectionStart.group(1); + end = matSectionStart.group(2); + ps = null; + vmFlags = null; + return; + } else { + Matcher matKernelPageSize = KERNEL_PAGESIZE_PATT.matcher(line); + if (matKernelPageSize.matches()) { + ps = matKernelPageSize.group(1); + return; + } + Matcher matTHPEligible = THP_ELIGIBLE_PATT.matcher(line); + if (matTHPEligible.matches()) { + thpEligible = matTHPEligible.group(1); + return; + } + Matcher matVmFlags = VMFLAGS_PATT.matcher(line); + if (matVmFlags.matches()) { + vmFlags = matVmFlags.group(1); + return; + } + } + } + + // Copy smaps locally + // (To minimize chances of concurrent modification when parsing, as well as helping with error analysis) + private Path copySmaps() throws Exception { + Path copy = Paths.get("smaps-copy-" + ProcessHandle.current().pid() + "-" + System.nanoTime() + ".txt"); + Files.copy(smaps, copy, StandardCopyOption.REPLACE_EXISTING); + return copy; + } + + // Parse /proc/self/smaps + public Smaps parse() throws Exception { + Path smapsCopy = copySmaps(); + Files.lines(smapsCopy).forEach(this::eatNext); + + // Finish up the last range + this.finish(); + + // Return a Smaps object with the parsed ranges + return new Smaps(ranges); + } + } + + // Class used to store information about memory ranges parsed + // from /proc/self/smaps. The file contain a lot of information + // about the different mappings done by an application, but the + // lines we care about are: + // 700000000-73ea00000 rw-p 00000000 00:00 0 + // ... + // KernelPageSize: 4 kB + // ... + // THPeligible: 0 + // ... + // VmFlags: rd wr mr mw me ac sd + // + // We use the VmFlags to know what kind of huge pages are used. + // For transparent huge pages the KernelPageSize field will not + // report the large page size. + public static class Range { + + private BigInteger start; + private BigInteger end; + private long pageSize; + private boolean thpEligible; + private boolean vmFlagHG; + private boolean vmFlagHT; + private boolean isTHP; + + public Range(String start, String end, String pageSize, String thpEligible, String vmFlags) { + // Note: since we insist on kernels >= 3.8, all the following information should be present + // (none of the input strings be null). + this.start = new BigInteger(start, 16); + this.end = new BigInteger(end, 16); + this.pageSize = Long.parseLong(pageSize); + this.thpEligible = thpEligible == null ? false : (Integer.parseInt(thpEligible) == 1); + + vmFlagHG = false; + vmFlagHT = false; + // Check if the vmFlags line include: + // * ht - Meaning the range is mapped using explicit huge pages. + // * hg - Meaning the range is madvised huge. + for (String flag : vmFlags.split(" ")) { + if (flag.equals("ht")) { + vmFlagHT = true; + } else if (flag.equals("hg")) { + vmFlagHG = true; + } + } + + // When the THP policy is 'always' instead of 'madvise, the vmFlagHG property is false, + // therefore also check thpEligible. If this is still causing problems in the future, + // we might have to check the AnonHugePages field. + + isTHP = vmFlagHG || this.thpEligible; + } + + public BigInteger getStart() { + return start; + } + + public BigInteger getEnd() { + return end; + } + + public long getPageSize() { + return pageSize; + } + + public boolean isTransparentHuge() { + return isTHP; + } + + public boolean isExplicitHuge() { + return vmFlagHT; + } + + public boolean includes(BigInteger addr) { + boolean isGreaterThanOrEqualStart = start.compareTo(addr) <= 0; + boolean isLessThanEnd = addr.compareTo(end) < 0; + + return isGreaterThanOrEqualStart && isLessThanEnd; + } + + public String toString() { + return "[" + start.toString(16) + ", " + end.toString(16) + ") " + + "pageSize=" + pageSize + "KB isTHP=" + isTHP + " isHUGETLB=" + vmFlagHT; + } + } +}