diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/TestFramework.java b/test/hotspot/jtreg/compiler/lib/ir_framework/TestFramework.java index 85c52ef33da..09e291ce5a4 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/TestFramework.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/TestFramework.java @@ -327,8 +327,10 @@ public class TestFramework { for (Scenario scenario : scenarios) { int scenarioIndex = scenario.getIndex(); - TestFormat.checkNoThrow(scenarioIndices.add(scenarioIndex), - "Cannot define two scenarios with the same index " + scenarioIndex); + if (!scenarioIndices.add(scenarioIndex)) { + TestFormat.failNoThrow("Cannot define two scenarios with the same index " + scenarioIndex); + continue; + } this.scenarios.add(scenario); } TestFormat.throwIfAnyFailures(); @@ -336,9 +338,12 @@ public class TestFramework { } /** - * Add the cross-product (cartesian product) of sets of flags as Scenarios. Unlike when when constructing + * Add the cross-product (cartesian product) of sets of flags as Scenarios. Unlike when constructing * scenarios directly a string can contain multiple flags separated with a space. This allows grouping - * flags that have to be specified togeher. Further, an empty string in a set stands in for "no flag". + * flags that have to be specified together. Further, an empty string in a set stands in for "no flag". + *

+ * Passing a single set will create a scenario for each of the provided flags in the set (i.e. the same as + * passing an additional set with an empty string only). *

* Example: * 

@@ -355,7 +360,7 @@ public class TestFramework {
      *     Scenario(5, "-Xbatch -XX:-TieredCompilation", "-XX:+UseNewCode2")
      *
* - * @param sets sets of flags to generate the cross product for. + * @param flagSets sets of flags to generate the cross product for. * @return the same framework instance. */ @SafeVarargs @@ -376,7 +381,7 @@ public class TestFramework { Stream> crossProduct = Arrays.stream(flagSets) .reduce( - Stream.of(Collections.emptyList()), // Initialize Stream> acc with a Stream containing an empty list of Strings. + Stream.of(Collections.emptyList()), // Initialize Stream> acc with a Stream containing an empty list of Strings. (Stream> acc, Set set) -> acc.flatMap(lAcc -> // For each List> lAcc in acc... set.stream().map(flag -> { // ...and each flag in the current set... @@ -384,19 +389,19 @@ public class TestFramework { newList.add(flag); // ...and append the flag. return newList; }) // This results in one List> for each lAcc... - ), // ...that get flattend into one big List>. - (a, b) -> Stream.concat(a, b)); // combiner; if any reduction steps are executed in parallel, just concat two streams. + ), // ...that get flattened into one big List>. + Stream::concat); // combiner; if any reduction steps are executed in parallel, just concat two streams. Scenario[] newScenarios = crossProduct .map(flags -> new Scenario( // For each List flags in crossProduct create a new Scenario. idx.getAndIncrement(), flags.stream() // Process flags - .map(s -> Set.of(s.split("[ ]"))) // Split muliple flags in the same string into separate strings. + .map(s -> Set.of(s.split("[ ]"))) // Split multiple flags in the same string into separate strings. .flatMap(Collection::stream) // Flatten the Stream> into Stream>. .filter(s -> !s.isEmpty()) // Remove empty string flags. - .collect(Collectors.toList()) + .toList() .toArray(new String[0]))) - .collect(Collectors.toList()).toArray(new Scenario[0]); + .toList().toArray(new Scenario[0]); return addScenarios(newScenarios); } diff --git a/test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestScenariosCrossProduct.java b/test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestScenariosCrossProduct.java index 496fcbddb0f..46813bbff78 100644 --- a/test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestScenariosCrossProduct.java +++ b/test/hotspot/jtreg/testlibrary_tests/ir_framework/tests/TestScenariosCrossProduct.java @@ -23,15 +23,22 @@ package ir_framework.tests; -import java.util.Set; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import compiler.lib.ir_framework.*; -import compiler.lib.ir_framework.shared.TestRunException; import compiler.lib.ir_framework.shared.TestFormatException; +import compiler.lib.ir_framework.shared.TestRunException; import jdk.test.lib.Asserts; /* * @test + * @bug 8365262 8369232 * @requires vm.debug == true & vm.compMode != "Xint" & vm.compiler2.enabled & vm.flagless * @summary Test cross product scenarios with the framework. * @library /test/lib /testlibrary_tests / @@ -39,29 +46,20 @@ import jdk.test.lib.Asserts; */ public class TestScenariosCrossProduct { - static void hasNFailures(String s, int count) { - if (!s.matches("The following scenarios have failed: (#[0-9](, )?){" + count + "}. Please check stderr for more information.")) { - throw new RuntimeException("Expected " + count + " failures in \"" + s + "\""); - } - } public static void main(String[] args) { - // Test argument handling - try { - TestFramework t = new TestFramework(); - t.addCrossProductScenarios((Set[]) null); - Asserts.fail("Should have thrown exception"); - } catch (TestFormatException e) {} - try { - TestFramework t = new TestFramework(); - t.addCrossProductScenarios(Set.of("foo", "bar"), null); - Asserts.fail("Should have thrown exception"); - } catch (TestFormatException e) {} + expectFormatFailure((Set[]) null); + expectFormatFailure(Set.of("foo", "bar"), null); + try { TestFramework t = new TestFramework(); t.addCrossProductScenarios(Set.of("blub"), Set.of("foo", null)); - Asserts.fail("Should have thrown exception"); - } catch (NullPointerException e) {} // Set.of prevents null elements + shouldHaveThrown(); + } catch (NullPointerException _) { + // Expected: Set.of prevents null elements + } + + try { TestFramework t = new TestFramework(); t.addCrossProductScenarios(); @@ -70,95 +68,291 @@ public class TestScenariosCrossProduct { } // Single set should test all flags in the set by themselves. - try { - TestFramework t1 = new TestFramework(); - t1.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=51", - "-XX:TLABRefillWasteFraction=53", - "-XX:TLABRefillWasteFraction=64")); - t1.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 3); - } + new TestCase() + .inputFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=51", + "-XX:TLABRefillWasteFraction=53", + "-XX:TLABRefillWasteFraction=64") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=51"), + Set.of("-XX:TLABRefillWasteFraction=53"), + Set.of("-XX:TLABRefillWasteFraction=64") + )) + .run(); // The cross product of a set with one element and a set with three elements is three sets. - try { - TestFramework t2 = new TestFramework(); - t2.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=53"), - Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2", "-XX:+UseNewCode3")); - t2.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 3); - } + new TestCase() + .inputFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=53"), + Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2", "-XX:+UseNewCode3") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode3") + )) + .run(); + // The cross product of two sets with two elements is four sets. - try { - TestFramework t3 = new TestFramework(); - t3.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=53", "-XX:TLABRefillWasteFraction=64"), - Set.of("-XX:+UseNewCode", "-XX:-UseNewCode")); - t3.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 4); - } + new TestCase() + .inputFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:TLABRefillWasteFraction=64"), + Set.of("-XX:+UseNewCode", "-XX:-UseNewCode") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:-UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=64", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=64", "-XX:-UseNewCode") + )) + .run(); + // Test with a pair of flags. - try { - TestFramework t4 = new TestFramework(); - t4.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=50 -XX:+UseNewCode", "-XX:TLABRefillWasteFraction=40"), - Set.of("-XX:+UseNewCode2")); - t4.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 1); - } + new TestCase() + .inputFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=50 -XX:+UseNewCode", "-XX:TLABRefillWasteFraction=40"), + Set.of("-XX:+UseNewCode2") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=50", "-XX:+UseNewCode", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=40", "-XX:+UseNewCode2") + )) + .run(); - // Test with an empty string. All 6 scenarios fail because 64 is the default value for TLABRefillWasteFraction. - try { - TestFramework t5 = new TestFramework(); - t5.addCrossProductScenarios(Set.of("", "-XX:TLABRefillWasteFraction=51", "-XX:TLABRefillWasteFraction=53"), - Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2")); - t5.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 6); - } + // Test with an empty string, resulting in 6 scenarios. + new TestCase() + .inputFlags(Set.of( + Set.of("", "-XX:TLABRefillWasteFraction=51", "-XX:TLABRefillWasteFraction=53"), + Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:+UseNewCode"), + Set.of("-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode2") + )) + .run(); + // Test with 3 input sets which equals to 2x2x2 = 8 scenarios. + new TestCase() + .inputFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=51", + "-XX:TLABRefillWasteFraction=53"), + Set.of("-XX:+UseNewCode", + "-XX:-UseNewCode"), + Set.of("-XX:+UseNewCode2", + "-XX:-UseNewCode2") + ) + ) + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:-UseNewCode", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:-UseNewCode", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode", "-XX:-UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode", "-XX:-UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:-UseNewCode", "-XX:-UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:-UseNewCode", "-XX:-UseNewCode2") + )) + .run(); + + TestFramework testFramework = new TestFramework(); + testFramework.addScenarios(new Scenario(0, "-XX:TLABRefillWasteFraction=50", "-XX:+UseNewCode")); + testFramework.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=51", "-XX:TLABRefillWasteFraction=53"), + Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2")); try { - TestFramework t6 = new TestFramework(); - t6.addScenarios(new Scenario(0, "-XX:TLABRefillWasteFraction=50", "-XX:+UseNewCode")); // failPair - t6.addCrossProductScenarios(Set.of("-XX:TLABRefillWasteFraction=51", "-XX:TLABRefillWasteFraction=53"), - Set.of("-XX:+UseNewCode", "-XX:+UseNewCode2")); - try { - t6.addScenarios(new Scenario(4, "-XX:+UseNewCode3")); // fails because index 4 is already used - Asserts.fail("Should have thrown exception"); - } catch (TestFormatException e) {} - t6.addScenarios(new Scenario(5, "-XX:+UseNewCode3")); // fail default - t6.start(); - Asserts.fail("Should have thrown exception"); - } catch (TestRunException e) { - hasNFailures(e.getMessage(), 6); + testFramework.addScenarios(new Scenario(4, "-XX:+UseNewCode3")); // fails because index 4 is already used + shouldHaveThrown(); + } catch (TestFormatException _) { + // Expected. + } + testFramework.addScenarios(new Scenario(5, "-XX:+UseNewCode3")); + + new TestCase() + .expectedScenariosWithFlags(Set.of( + Set.of("-XX:TLABRefillWasteFraction=50", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=51", "-XX:+UseNewCode2"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode"), + Set.of("-XX:TLABRefillWasteFraction=53", "-XX:+UseNewCode2"), + Set.of("-XX:+UseNewCode3") + )) + .runWithPreAddedScenarios(testFramework); + + runEndToEndTest(); + } + + private static void expectFormatFailure(Set... flagSets) { + TestFramework testFramework = new TestFramework(); + try { + testFramework.addCrossProductScenarios(flagSets); + shouldHaveThrown(); + } catch (TestFormatException _) { + // Expected. } } - @Test - @IR(applyIf = {"TLABRefillWasteFraction", "64"}, counts = {IRNode.CALL, "1"}) - public void failDefault() { + private static void shouldHaveThrown() { + Asserts.fail("Should have thrown exception"); + } + + static class TestCase { + private Set> inputFlags; + private Set> expectedScenariosWithFlags; + + public TestCase inputFlags(Set> inputFlags) { + this.inputFlags = inputFlags; + return this; + } + + public TestCase expectedScenariosWithFlags(Set> expectedScenariosWithFlags) { + this.expectedScenariosWithFlags = expectedScenariosWithFlags; + return this; + } + + public void run() { + TestFramework testFramework = new TestFramework(); + testFramework.addCrossProductScenarios(inputFlags.toArray(new Set[0])); + runWithPreAddedScenarios(testFramework); + } + + public void runWithPreAddedScenarios(TestFramework testFramework) { + List scenariosFromCrossProduct = getScenarios(testFramework); + assertScenarioCount(expectedScenariosWithFlags.size(), scenariosFromCrossProduct); + assertScenariosWithFlags(scenariosFromCrossProduct, expectedScenariosWithFlags); + assertSameResultWhenManuallyAdding(scenariosFromCrossProduct, expectedScenariosWithFlags); + } + + private static void assertScenarioCount(int expectedCount, List scenarios) { + Asserts.assertEQ(expectedCount, scenarios.size(), "Scenario count is off"); + } + + /** + * Check that the added scenarios to the IR framework with TestFramework.addCrossProductScenarios() + * (i.e. 'scenariosFromCrossProduct') match the expected flag combos (i.e. 'expectedScenariosWithFlags'). + */ + private static void assertScenariosWithFlags(List scenariosFromCrossProduct, + Set> expectedScenariosWithFlags) { + for (Set expectedScenarioFlags : expectedScenariosWithFlags) { + if (scenariosFromCrossProduct.stream() + .map(Scenario::getFlags) + .map(Set::copyOf) + .anyMatch(flags -> flags.equals(expectedScenarioFlags))) { + continue; + } + System.err.println("Scenarios from cross product:"); + for (Scenario s : scenariosFromCrossProduct) { + System.err.println(Arrays.toString(s.getFlags().toArray())); + } + throw new RuntimeException("Could not find a scenario with the provided flags: " + Arrays.toString(expectedScenarioFlags.toArray())); + } + } + + /** + * Add scenarios for the provided flag sets in 'expectedScenariosWithFlags' by using TestFramework.addScenarios(). + * We should end up with the same scenarios as if we added them with TestFramework.addCrossProductScenarios(). + * This is verified by this method by comparing the flags of the scenarios, ignoring scenario indices. + */ + private static void assertSameResultWhenManuallyAdding(List scenariosFromCrossProduct, + Set> expectedScenariosWithFlags) { + List expectedScenarios = getScenariosWithFlags(expectedScenariosWithFlags); + List fetchedScenarios = addScenariosAndFetchFromFramework(expectedScenarios); + assertSameScenarios(scenariosFromCrossProduct, fetchedScenarios); + } + + private static List getScenariosWithFlags(Set> expectedScenariosWithFlags) { + List expecedScenarioList = new ArrayList<>(); + int index = -1; // Use some different indices - should not matter what we choose. + for (Set expectedScenarioFlags : expectedScenariosWithFlags) { + expecedScenarioList.add(new Scenario(index--, expectedScenarioFlags.toArray(new String[0]))); + } + return expecedScenarioList; + } + + private static List addScenariosAndFetchFromFramework(List expecedScenarioList) { + TestFramework testFramework = new TestFramework(); + testFramework.addScenarios(expecedScenarioList.toArray(new Scenario[0])); + return getScenarios(testFramework); + } + + private static void assertSameScenarios(List scenariosFromCrossProduct, + List expectedScenarios) { + assertScenariosWithFlags(scenariosFromCrossProduct, fetchFlags(expectedScenarios)); + } + + private static Set> fetchFlags(List scenarios) { + return scenarios.stream() + .map(scenario -> new HashSet<>(scenario.getFlags())) + .collect(Collectors.toSet()); + } + } + + private static List getScenarios(TestFramework testFramework) { + Field field; + try { + field = TestFramework.class.getDeclaredField("scenarios"); + field.setAccessible(true); + return (List)field.get(testFramework); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Also run a simple end-to-end test to sanity check the API method. We capture the stderr to fetch the + * scenario flags. + */ + private static void runEndToEndTest() { + TestFramework testFramework = new TestFramework(); + + // Capture stderr + PrintStream originalErr = System.err; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + System.setErr(printStream); + + try { + testFramework + .addCrossProductScenarios(Set.of("-XX:+UseNewCode", "-XX:-UseNewCode"), + Set.of("-XX:+UseNewCode2", "-XX:-UseNewCode2")) + .addFlags() + .start(); + shouldHaveThrown(); + } catch (TestRunException e) { + // Expected. + System.setErr(originalErr); + Asserts.assertTrue(e.getMessage().contains("The following scenarios have failed: #0, #1, #2, #3.")); + String stdErr = outputStream.toString(); + Asserts.assertTrue(stdErr.contains("Scenario flags: [-XX:+UseNewCode, -XX:+UseNewCode2]")); + Asserts.assertTrue(stdErr.contains("Scenario flags: [-XX:-UseNewCode, -XX:-UseNewCode2]")); + Asserts.assertTrue(stdErr.contains("Scenario flags: [-XX:+UseNewCode, -XX:-UseNewCode2]")); + Asserts.assertTrue(stdErr.contains("Scenario flags: [-XX:-UseNewCode, -XX:+UseNewCode2]")); + Asserts.assertEQ(4, scenarioCount(stdErr)); + } + } + + public static int scenarioCount(String stdErr) { + Pattern pattern = Pattern.compile("Scenario flags"); + Matcher matcher = pattern.matcher(stdErr); + int count = 0; + while (matcher.find()) { + count++; + } + return count; } @Test - @IR(applyIf = {"TLABRefillWasteFraction", "51"}, counts = {IRNode.CALL, "1"}) - public void fail1() { - } - - @Test - @IR(applyIf = {"TLABRefillWasteFraction", "53"}, counts = {IRNode.CALL, "1"}) - public void fail2() { - } - - @Test - @IR(applyIfAnd = {"TLABRefillWasteFraction", "50", "UseNewCode", "true"}, counts = {IRNode.CALL, "1"}) - public void failPair() { + public void endToEndTest() { + throw new RuntimeException("executed test"); } }