diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 7356f5a1913..b8ddbfe5120 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -6227,15 +6227,21 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR // Since IEEE 754 floating point format represents mantissa in 1.0 format // hence biased exponent can be used to compute leading zero count as per // following formula:- - // LZCNT = 32 - (biased_exp - 127) + // LZCNT = 31 - (biased_exp - 127) // Special handling has been introduced for Zero, Max_Int and -ve source values. // Broadcast 0xFF vpcmpeqd(xtmp1, xtmp1, xtmp1, vec_enc); vpsrld(xtmp1, xtmp1, 24, vec_enc); + // Remove the bit to the right of the highest set bit ensuring that the conversion to float cannot round up to a higher + // power of 2, which has a higher exponent than the input. This transformation is valid as only the highest set bit + // contributes to the leading number of zeros. + vpsrld(xtmp2, src, 1, vec_enc); + vpandn(xtmp3, xtmp2, src, vec_enc); + // Extract biased exponent. - vcvtdq2ps(dst, src, vec_enc); + vcvtdq2ps(dst, xtmp3, vec_enc); vpsrld(dst, dst, 23, vec_enc); vpand(dst, dst, xtmp1, vec_enc); @@ -6244,7 +6250,7 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR // Exponent = biased_exp - 127 vpsubd(dst, dst, xtmp1, vec_enc); - // Exponent = Exponent + 1 + // Exponent_plus_one = Exponent + 1 vpsrld(xtmp3, xtmp1, 6, vec_enc); vpaddd(dst, dst, xtmp3, vec_enc); @@ -6257,7 +6263,7 @@ void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMR vpslld(xtmp1, xtmp3, 5, vec_enc); // Exponent is 32 if corresponding source lane contains max_int value. vpcmpeqd(xtmp2, dst, xtmp1, vec_enc); - // LZCNT = 32 - exponent + // LZCNT = 32 - exponent_plus_one vpsubd(dst, xtmp1, dst, vec_enc); // Replace LZCNT with a value 1 if corresponding source lane diff --git a/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java b/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java index db0b563d1fe..3a273c7be8c 100644 --- a/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java +++ b/test/hotspot/jtreg/compiler/vectorization/TestNumberOfContinuousZeros.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2022, 2023, Arm Limited. All rights reserved. + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +24,7 @@ /** * @test +* @bug 8297172 8331993 8349637 * @key randomness * @summary Test vectorization of numberOfTrailingZeros/numberOfLeadingZeros for Long * @requires vm.compiler2.enabled @@ -30,16 +32,20 @@ * (os.simpleArch == "aarch64" & vm.cpu.features ~= ".*sve.*") | * (os.simpleArch == "riscv64" & vm.cpu.features ~= ".*zvbb.*") * @library /test/lib / +* @modules jdk.incubator.vector * @run driver compiler.vectorization.TestNumberOfContinuousZeros */ package compiler.vectorization; +import jdk.incubator.vector.*; import compiler.lib.ir_framework.*; import java.util.Random; import jdk.test.lib.Asserts; +import jdk.test.lib.Utils; public class TestNumberOfContinuousZeros { + private static final int[] SPECIAL = { 0x01FFFFFF, 0x03FFFFFE, 0x07FFFFFC, 0x0FFFFFF8, 0x1FFFFFF0, 0x3FFFFFE0, 0xFFFFFFFF }; private long[] inputLong; private int[] outputLong; private int[] inputInt; @@ -47,8 +53,8 @@ public class TestNumberOfContinuousZeros { private static final int LEN = 1024; private Random rng; - public static void main(String args[]) { - TestFramework.run(); + public static void main(String[] args) { + TestFramework.runWithFlags("--add-modules=jdk.incubator.vector"); } public TestNumberOfContinuousZeros() { @@ -56,7 +62,7 @@ public class TestNumberOfContinuousZeros { outputLong = new int[LEN]; inputInt = new int[LEN]; outputInt = new int[LEN]; - rng = new Random(42); + rng = Utils.getRandomInstance(); for (int i = 0; i < LEN; ++i) { inputLong[i] = rng.nextLong(); inputInt[i] = rng.nextInt(); @@ -119,5 +125,80 @@ public class TestNumberOfContinuousZeros { Asserts.assertEquals(outputInt[i], Integer.numberOfLeadingZeros(inputInt[i])); } } + + @Setup + static Object[] setupSpecialIntArray() { + int[] res = new int[LEN]; + + for (int i = 0; i < LEN; i++) { + res[i] = SPECIAL[i % SPECIAL.length]; + } + + return new Object[] { res }; + } + + @Test + @IR(counts = {IRNode.COUNT_LEADING_ZEROS_VI, "> 0"}) + @Arguments(setup = "setupSpecialIntArray") + public Object[] testSpecialIntLeadingZeros(int[] ints) { + int[] res = new int[LEN]; + + for (int i = 0; i < LEN; ++i) { + res[i] = Integer.numberOfLeadingZeros(ints[i]); + } + + return new Object[] { ints, res }; + } + + @Check(test = "testSpecialIntLeadingZeros") + public void checkSpecialIntLeadingZeros(Object[] vals) { + int[] in = (int[]) vals[0]; + int[] out = (int[]) vals[1]; + + for (int i = 0; i < LEN; ++i) { + int value = Integer.numberOfLeadingZeros(in[i]); + + if (out[i] != value) { + throw new IllegalStateException("Expected lzcnt(" + in[i] + ") to be " + value + " but got " + out[i]); + } + } + } + + private static final VectorSpecies SPECIES = IntVector.SPECIES_PREFERRED; + + @Test + @IR(counts = {IRNode.COUNT_LEADING_ZEROS_VI, "> 0"}) + @Arguments(setup = "setupSpecialIntArray") + public Object[] checkSpecialIntLeadingZerosVector(int[] ints) { + int[] res = new int[LEN]; + + for (int i = 0; i < ints.length; i += SPECIES.length()) { + IntVector av = IntVector.fromArray(SPECIES, ints, i); + av.lanewise(VectorOperators.LEADING_ZEROS_COUNT).intoArray(res, i); + } + + return new Object[] { ints, res }; + } + + @Check(test = "checkSpecialIntLeadingZerosVector") + public void checkSpecialIntLeadingZerosVector(Object[] vals) { + int[] ints = (int[]) vals[0]; + int[] res = (int[]) vals[1]; + + // Verification + + int[] check = new int[LEN]; + + for (int i = 0; i < ints.length; i += SPECIES.length()) { + IntVector av = IntVector.fromArray(SPECIES, ints, i); + av.lanewise(VectorOperators.LEADING_ZEROS_COUNT).intoArray(check, i); + } + + for (int i = 0; i < LEN; i++) { + if (res[i] != check[i]) { + throw new IllegalStateException("Expected " + check[i] + " but got " + res[i]); + } + } + } } diff --git a/test/hotspot/jtreg/compiler/vectorization/TestVectorZeroCount.java b/test/hotspot/jtreg/compiler/vectorization/TestVectorZeroCount.java new file mode 100644 index 00000000000..a29ade54008 --- /dev/null +++ b/test/hotspot/jtreg/compiler/vectorization/TestVectorZeroCount.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* @test + * @bug 8349637 + * @summary Ensure that vectorization of numberOfLeadingZeros and numberOfTrailingZeros outputs correct values + * @library /test/lib / + * @run main/othervm compiler.vectorization.TestVectorZeroCount + */ + +package compiler.vectorization; + +import java.util.Random; + +import jdk.test.lib.Utils; + +public class TestVectorZeroCount { + private static final int SIZE = 1024; + private static final Random RANDOM = Utils.getRandomInstance(); + + private static final int[] INT_VALUES = new int[SIZE]; + private static final int[] INT_EXPECTED_LEADING = new int[SIZE]; + private static final int[] INT_RESULT_LEADING = new int[SIZE]; + private static final int[] INT_EXPECTED_TRAILING = new int[SIZE]; + private static final int[] INT_RESULT_TRAILING = new int[SIZE]; + + private static final long[] LONG_VALUES = new long[SIZE]; + private static final long[] LONG_EXPECTED_LEADING = new long[SIZE]; + private static final long[] LONG_RESULT_LEADING = new long[SIZE]; + private static final long[] LONG_EXPECTED_TRAILING = new long[SIZE]; + private static final long[] LONG_RESULT_TRAILING = new long[SIZE]; + + private static int intCounter = Integer.MIN_VALUE; + private static int longIterations = 100_000_000; + + public static boolean testInt() { + boolean done = false; + + // Non-vectorized loop as baseline (not vectorized because source array is initialized) + for (int i = 0; i < SIZE; ++i) { + INT_VALUES[i] = intCounter++; + if (intCounter == Integer.MAX_VALUE) { + done = true; + } + INT_EXPECTED_LEADING[i] = Integer.numberOfLeadingZeros(INT_VALUES[i]); + INT_EXPECTED_TRAILING[i] = Integer.numberOfTrailingZeros(INT_VALUES[i]); + } + // Vectorized loop + for (int i = 0; i < SIZE; ++i) { + INT_RESULT_LEADING[i] = Integer.numberOfLeadingZeros(INT_VALUES[i]); + } + for (int i = 0; i < SIZE; ++i) { + INT_RESULT_TRAILING[i] = Integer.numberOfTrailingZeros(INT_VALUES[i]); + } + + // Compare results + for (int i = 0; i < SIZE; ++i) { + if (INT_RESULT_LEADING[i] != INT_EXPECTED_LEADING[i]) { + throw new RuntimeException("Unexpected result for Integer.numberOfLeadingZeros(" + INT_VALUES[i] + "): " + INT_RESULT_LEADING[i] + ", expected " + INT_EXPECTED_LEADING[i]); + } + if (INT_RESULT_TRAILING[i] != INT_EXPECTED_TRAILING[i]) { + throw new RuntimeException("Unexpected result for Integer.numberOfTrailingZeros(" + INT_VALUES[i] + "): " + INT_RESULT_TRAILING[i] + ", expected " + INT_EXPECTED_TRAILING[i]); + } + } + return done; + } + + public static boolean testLong() { + boolean done = false; + + // Non-vectorized loop as baseline (not vectorized because source array is initialized) + for (int i = 0; i < SIZE; ++i) { + // Use random values because the long range is too large to iterate over it + LONG_VALUES[i] = RANDOM.nextLong(); + if (longIterations-- == 0) { + done = true; + } + LONG_EXPECTED_LEADING[i] = Long.numberOfLeadingZeros(LONG_VALUES[i]); + LONG_EXPECTED_TRAILING[i] = Long.numberOfTrailingZeros(LONG_VALUES[i]); + } + // Vectorized loop + for (int i = 0; i < SIZE; ++i) { + LONG_RESULT_LEADING[i] = Long.numberOfLeadingZeros(LONG_VALUES[i]); + } + for (int i = 0; i < SIZE; ++i) { + LONG_RESULT_TRAILING[i] = Long.numberOfTrailingZeros(LONG_VALUES[i]); + } + + // Compare results + for (int i = 0; i < SIZE; ++i) { + if (LONG_RESULT_LEADING[i] != LONG_EXPECTED_LEADING[i]) { + throw new RuntimeException("Unexpected result for Long.numberOfLeadingZeros(" + LONG_VALUES[i] + "): " + LONG_RESULT_LEADING[i] + ", expected " + LONG_EXPECTED_LEADING[i]); + } + if (LONG_RESULT_TRAILING[i] != LONG_EXPECTED_TRAILING[i]) { + throw new RuntimeException("Unexpected result for Long.numberOfTrailingZeros(" + LONG_VALUES[i] + "): " + LONG_RESULT_TRAILING[i] + ", expected " + LONG_EXPECTED_TRAILING[i]); + } + } + return done; + } + + public static void main(String[] args) { + // Run twice to make sure compiled code is used from the beginning + for (int i = 0; i < 2; ++i) { + while (!testLong()) ; + while (!testInt()) ; + } + } +}