From 436d291a1c679f892e5164ddf7d04ca59d6ca254 Mon Sep 17 00:00:00 2001 From: Eric Fang Date: Wed, 15 Apr 2026 08:24:51 +0000 Subject: [PATCH] 8370863: VectorAPI: Optimize the VectorMaskCast chain in specific patterns Reviewed-by: xgong, vlivanov, galder --- src/hotspot/share/opto/phaseX.cpp | 7 +- src/hotspot/share/opto/vectornode.cpp | 33 +- src/hotspot/share/opto/vectornode.hpp | 1 + .../TestVectorLongToMaskNodeIdealization.java | 8 +- .../vectorapi/VectorMaskCastIdentityTest.java | 104 +++--- .../vectorapi/VectorMaskCastTest.java | 188 +++++++---- .../vectorapi/VectorMaskToLongTest.java | 33 +- .../VectorStoreMaskIdentityTest.java | 317 ++++++++++++++++++ .../vector/VectorStoreMaskBenchmark.java | 84 +++++ 9 files changed, 654 insertions(+), 121 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/vectorapi/VectorStoreMaskIdentityTest.java create mode 100644 test/micro/org/openjdk/bench/jdk/incubator/vector/VectorStoreMaskBenchmark.java diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 0fecc14f31a..a4d6a6c33d0 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -2123,7 +2123,12 @@ void PhaseIterGVN::verify_Identity_for(Node* n) { if (n->is_Vector()) { // Found with tier1-3. Not investigated yet. - // The observed issue was with AndVNode::Identity + // The observed issue was with AndVNode::Identity and + // VectorStoreMaskNode::Identity (see JDK-8370863). + // + // Found with: + // compiler/vectorapi/VectorStoreMaskIdentityTest.java + // -XX:CompileThreshold=100 -XX:-TieredCompilation -XX:VerifyIterativeGVN=1110 return; } diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index b07f0234492..dbadc18da01 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1047,6 +1047,20 @@ Node* VectorNode::Ideal(PhaseGVN* phase, bool can_reshape) { return nullptr; } +// Traverses a chain of VectorMaskCast and returns the first non VectorMaskCast node. +// +// Due to the unique nature of vector masks, for specific IR patterns, +// VectorMaskCast does not affect the output results. For example: +// (VectorStoreMask (VectorMaskCast* (VectorLoadMask x))) => (x) +// x remains to be a bool vector with no changes. +// This function can be used to eliminate the VectorMaskCast in such patterns. +Node* VectorNode::uncast_mask(Node* n) { + while (n->Opcode() == Op_VectorMaskCast) { + n = n->in(1); + } + return n; +} + // Return initial Pack node. Additional operands added with add_opd() calls. PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) { const TypeVect* vt = TypeVect::make(bt, vlen); @@ -1495,10 +1509,12 @@ Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) { Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) { // Identity transformation on boolean vectors. - // VectorStoreMask (VectorLoadMask bv) elem_size ==> bv + // VectorStoreMask (VectorMaskCast* VectorLoadMask bv) elem_size ==> bv // vector[n]{bool} => vector[n]{t} => vector[n]{bool} - if (in(1)->Opcode() == Op_VectorLoadMask) { - return in(1)->in(1); + Node* in1 = VectorNode::uncast_mask(in(1)); + if (in1->Opcode() == Op_VectorLoadMask) { + assert(length() == in1->as_Vector()->length(), "vector length must match"); + return in1->in(1); } return this; } @@ -1959,11 +1975,12 @@ Node* VectorMaskOpNode::Ideal(PhaseGVN* phase, bool can_reshape) { } Node* VectorMaskCastNode::Identity(PhaseGVN* phase) { - Node* in1 = in(1); - // VectorMaskCast (VectorMaskCast x) => x - if (in1->Opcode() == Op_VectorMaskCast && - vect_type()->eq(in1->in(1)->bottom_type())) { - return in1->in(1); + // (VectorMaskCast+ x) => (x) + // If the types of the input and output nodes in a VectorMaskCast chain are + // exactly the same, the intermediate VectorMaskCast nodes can be eliminated. + Node* n = VectorNode::uncast_mask(this); + if (vect_type()->eq(n->bottom_type())) { + return n; } return this; } diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index c4fff06e771..de866898302 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -195,6 +195,7 @@ class VectorNode : public TypeNode { static bool is_scalar_op_that_returns_int_but_vector_op_returns_long(int opc); static bool is_reinterpret_opcode(int opc); + static Node* uncast_mask(Node* n); static void trace_new_vector(Node* n, const char* context) { #ifdef ASSERT diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorLongToMaskNodeIdealization.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorLongToMaskNodeIdealization.java index 706e21554bb..f9e50fbd23b 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorLongToMaskNodeIdealization.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorLongToMaskNodeIdealization.java @@ -46,7 +46,7 @@ import static compiler.lib.template_framework.Template.let; /* * @test - * @bug 8277997 8378968 + * @bug 8277997 8378968 8380290 * @key randomness * @summary Testing some optimizations in VectorLongToMaskNode::Ideal * For now: VectorMask.fromLong(.., mask.toLong()) @@ -93,7 +93,7 @@ public class TestVectorLongToMaskNodeIdealization { IRNode.VECTOR_STORE_MASK, "> 0", // Not yet optimized away IRNode.VECTOR_LONG_TO_MASK, "= 0", // Optimized away IRNode.VECTOR_MASK_TO_LONG, "= 0", // Optimized away - IRNode.VECTOR_MASK_CAST, "> 0", // Not yet optimized away + IRNode.VECTOR_MASK_CAST, "= 0", // Optimized away IRNode.VECTOR_BLEND_I, IRNode.VECTOR_SIZE_4, "> 0", IRNode.XOR_VI, IRNode.VECTOR_SIZE_4, "> 0", IRNode.STORE_VECTOR, "> 0"}, @@ -168,7 +168,7 @@ public class TestVectorLongToMaskNodeIdealization { IRNode.VECTOR_STORE_MASK, "> 0", // Not yet optimized away IRNode.VECTOR_LONG_TO_MASK, "= 0", // Optimized away IRNode.VECTOR_MASK_TO_LONG, "= 0", // Optimized away - IRNode.VECTOR_MASK_CAST, "> 0", // Not yet optimized away: Cast Z->Z, see JDK-8379866 + IRNode.VECTOR_MASK_CAST, "= 0", // Optimized away IRNode.VECTOR_BLEND_I, IRNode.VECTOR_SIZE_4, "> 0", IRNode.XOR_VI, IRNode.VECTOR_SIZE_4, "> 0", IRNode.STORE_VECTOR, "> 0"}, @@ -219,7 +219,7 @@ public class TestVectorLongToMaskNodeIdealization { IRNode.VECTOR_STORE_MASK, "= 0", IRNode.VECTOR_LONG_TO_MASK, "= 0", // Optimized away IRNode.VECTOR_MASK_TO_LONG, "= 0", // Optimized away - IRNode.VECTOR_MASK_CAST, "> 0", // Not yet optimized Z->Z, see JDK-8379866 + IRNode.VECTOR_MASK_CAST, "= 0", // Optimized away IRNode.VECTOR_BLEND_I, IRNode.VECTOR_SIZE_4, "> 0", IRNode.XOR_VI, IRNode.VECTOR_SIZE_4, "> 0", IRNode.STORE_VECTOR, "> 0"}, diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastIdentityTest.java index e4f166f510a..e1c20da76ec 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastIdentityTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * Copyright (c) 2025, 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,10 +23,10 @@ /* * @test -* @bug 8356760 +* @bug 8356760 8370863 * @key randomness * @library /test/lib / -* @summary Optimize VectorMask.fromLong for all-true/all-false cases +* @summary test VectorMaskCast Identity() optimizations * @modules jdk.incubator.vector * * @run driver compiler.vectorapi.VectorMaskCastIdentityTest @@ -50,70 +50,88 @@ public class VectorMaskCastIdentityTest { } @Test - @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) - public static int testTwoCastToDifferentType() { - // The types before and after the two casts are not the same, so the cast cannot be eliminated. - VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mr, 0); - VectorMask mDouble128 = mFloat64.cast(DoubleVector.SPECIES_128); - VectorMask mInt64 = mDouble128.cast(IntVector.SPECIES_64); - return mInt64.trueCount(); + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, "> 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static int testOneCastToSameType() { + // The types before and after the cast sequence are the same, + // so the casts will be eliminated. + VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); + mInt128 = mInt128.cast(IntVector.SPECIES_128); + // Insert a not() to prevent the casts being optimized by the optimization: + // (VectorStoreMask (VectorMaskCast* (VectorLoadMask x))) => x + return mInt128.not().trueCount(); } - @Run(test = "testTwoCastToDifferentType") - public static void testTwoCastToDifferentType_runner() { - int count = testTwoCastToDifferentType(); - VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mr, 0); - Asserts.assertEquals(count, mFloat64.trueCount()); + @Run(test = "testOneCastToSameType") + public static void testOneCastToSameType_runner() { + int count = testOneCastToSameType(); + VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); + Asserts.assertEquals(count, mInt128.not().trueCount()); } @Test - @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 2" }, applyIfCPUFeatureOr = {"avx2", "true"}) - public static int testTwoCastToDifferentType2() { - // The types before and after the two casts are not the same, so the cast cannot be eliminated. - VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); - VectorMask mDouble256 = mInt128.cast(DoubleVector.SPECIES_256); - VectorMask mShort64 = mDouble256.cast(ShortVector.SPECIES_64); - return mShort64.trueCount(); - } - - @Run(test = "testTwoCastToDifferentType2") - public static void testTwoCastToDifferentType2_runner() { - int count = testTwoCastToDifferentType2(); - VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); - Asserts.assertEquals(count, mInt128.trueCount()); - } - - @Test - @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, "> 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) public static int testTwoCastToSameType() { - // The types before and after the two casts are the same, so the cast will be eliminated. + // The types before and after the cast sequence are the same, + // so the casts will be eliminated. VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); VectorMask mFloat128 = mInt128.cast(FloatVector.SPECIES_128); VectorMask mInt128_2 = mFloat128.cast(IntVector.SPECIES_128); - return mInt128_2.trueCount(); + return mInt128_2.not().trueCount(); } @Run(test = "testTwoCastToSameType") public static void testTwoCastToSameType_runner() { int count = testTwoCastToSameType(); VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); - Asserts.assertEquals(count, mInt128.trueCount()); + Asserts.assertEquals(count, mInt128.not().trueCount()); } @Test - @IR(counts = { IRNode.VECTOR_MASK_CAST, "= 1" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, "> 0", + IRNode.VECTOR_MASK_CAST, "= 1" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) public static int testOneCastToDifferentType() { - // The types before and after the only cast are different, the cast will not be eliminated. - VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mr, 0).not(); - VectorMask mInt128 = mFloat128.cast(IntVector.SPECIES_128); - return mInt128.trueCount(); + // The types before and after the cast sequence are different, + // so the casts will not be eliminated. + VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); + VectorMask mShort64 = mInt128.cast(ShortVector.SPECIES_64); + return mShort64.not().trueCount(); } @Run(test = "testOneCastToDifferentType") public static void testOneCastToDifferentType_runner() { int count = testOneCastToDifferentType(); - VectorMask mInt128 = VectorMask.fromArray(FloatVector.SPECIES_128, mr, 0).not(); - Asserts.assertEquals(count, mInt128.trueCount()); + VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mr, 0); + Asserts.assertEquals(count, mInt128.not().trueCount()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, "> 0", + IRNode.VECTOR_MASK_CAST, "= 2" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx2", "true", "rvv", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static int testTwoCastToDifferentType() { + // The types before and after the cast sequence are different, so the + // casts are not eliminated. We should probably be able to eliminate + // the intermediate cast, so that we only need a cast from short to int. + VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mr, 0); + VectorMask mFloat128 = mShort64.cast(FloatVector.SPECIES_128); + VectorMask mInt128 = mFloat128.cast(IntVector.SPECIES_128); + return mInt128.not().trueCount(); + } + + @Run(test = "testTwoCastToDifferentType") + public static void testTwoCastToDifferentType_runner() { + int count = testTwoCastToDifferentType(); + VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mr, 0); + Asserts.assertEquals(count, mShort64.not().trueCount()); } public static void main(String[] args) { diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java index 1f346cffd2f..c75ea6c7a66 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, 2023, Arm Limited. All rights reserved. + * Copyright (c) 2025, 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -76,13 +77,15 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testByte64ToShort128(VectorMask v) { - return v.cast(ShortVector.SPECIES_128); + // A not operation is introduced to prevent the cast from being optimized away. + return v.not().cast(ShortVector.SPECIES_128); } @Run(test = "testByte64ToShort128") public static void testByte64ToShort128_runner() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); VectorMask res = testByte64ToShort128(mByte64); + mByte64 = mByte64.not(); Asserts.assertEquals(res.toString(), mByte64.toString()); Asserts.assertEquals(res.trueCount(), mByte64.trueCount()); } @@ -90,13 +93,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testByte64ToInt256(VectorMask v) { - return v.cast(IntVector.SPECIES_256); + return v.not().cast(IntVector.SPECIES_256); } @Run(test = "testByte64ToInt256") public static void testByte64ToInt256_runner() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); VectorMask res = testByte64ToInt256(mByte64); + mByte64 = mByte64.not(); Asserts.assertEquals(res.toString(), mByte64.toString()); Asserts.assertEquals(res.trueCount(), mByte64.trueCount()); } @@ -104,13 +108,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testByte64ToFloat256(VectorMask v) { - return v.cast(FloatVector.SPECIES_256); + return v.not().cast(FloatVector.SPECIES_256); } @Run(test = "testByte64ToFloat256") public static void testByte64ToFloat256_runner() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); VectorMask res = testByte64ToFloat256(mByte64); + mByte64 = mByte64.not(); Asserts.assertEquals(res.toString(), mByte64.toString()); Asserts.assertEquals(res.trueCount(), mByte64.trueCount()); } @@ -118,13 +123,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testByte64ToLong512(VectorMask v) { - return v.cast(LongVector.SPECIES_512); + return v.not().cast(LongVector.SPECIES_512); } @Run(test = "testByte64ToLong512") public static void testByte64ToLong512_runner() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); VectorMask res = testByte64ToLong512(mByte64); + mByte64 = mByte64.not(); Asserts.assertEquals(res.toString(), mByte64.toString()); Asserts.assertEquals(res.trueCount(), mByte64.trueCount()); } @@ -132,13 +138,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testByte64ToDouble512(VectorMask v) { - return v.cast(DoubleVector.SPECIES_512); + return v.not().cast(DoubleVector.SPECIES_512); } @Run(test = "testByte64ToDouble512") public static void testByte64ToDouble512_runner() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); VectorMask res = testByte64ToDouble512(mByte64); + mByte64 = mByte64.not(); Asserts.assertEquals(res.toString(), mByte64.toString()); Asserts.assertEquals(res.trueCount(), mByte64.trueCount()); } @@ -146,13 +153,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testByte128ToShort256(VectorMask v) { - return v.cast(ShortVector.SPECIES_256); + return v.not().cast(ShortVector.SPECIES_256); } @Run(test = "testByte128ToShort256") public static void testByte128ToShort256_runner() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); VectorMask res = testByte128ToShort256(mByte128); + mByte128 = mByte128.not(); Asserts.assertEquals(res.toString(), mByte128.toString()); Asserts.assertEquals(res.trueCount(), mByte128.trueCount()); } @@ -160,13 +168,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testByte128ToInt512(VectorMask v) { - return v.cast(IntVector.SPECIES_512); + return v.not().cast(IntVector.SPECIES_512); } @Run(test = "testByte128ToInt512") public static void testByte128ToInt512_runner() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); VectorMask res = testByte128ToInt512(mByte128); + mByte128 = mByte128.not(); Asserts.assertEquals(res.toString(), mByte128.toString()); Asserts.assertEquals(res.trueCount(), mByte128.trueCount()); } @@ -174,13 +183,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testByte128ToFloat512(VectorMask v) { - return v.cast(FloatVector.SPECIES_512); + return v.not().cast(FloatVector.SPECIES_512); } @Run(test = "testByte128ToFloat512") public static void testByte128ToFloat512_runner() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); VectorMask res = testByte128ToFloat512(mByte128); + mByte128 = mByte128.not(); Asserts.assertEquals(res.toString(), mByte128.toString()); Asserts.assertEquals(res.trueCount(), mByte128.trueCount()); } @@ -188,13 +198,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testByte256ToShort512(VectorMask v) { - return v.cast(ShortVector.SPECIES_512); + return v.not().cast(ShortVector.SPECIES_512); } @Run(test = "testByte256ToShort512") public static void testByte256ToShort512_runner() { VectorMask mByte256 = VectorMask.fromArray(ByteVector.SPECIES_256, mask_arr, 0); VectorMask res = testByte256ToShort512(mByte256); + mByte256 = mByte256.not(); Asserts.assertEquals(res.toString(), mByte256.toString()); Asserts.assertEquals(res.trueCount(), mByte256.trueCount()); } @@ -203,13 +214,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testShort64ToInt128(VectorMask v) { - return v.cast(IntVector.SPECIES_128); + return v.not().cast(IntVector.SPECIES_128); } @Run(test = "testShort64ToInt128") public static void testShort64ToInt128_runner() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); VectorMask res = testShort64ToInt128(mShort64); + mShort64 = mShort64.not(); Asserts.assertEquals(res.toString(), mShort64.toString()); Asserts.assertEquals(res.trueCount(), mShort64.trueCount()); } @@ -217,13 +229,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testShort64ToFloat128(VectorMask v) { - return v.cast(FloatVector.SPECIES_128); + return v.not().cast(FloatVector.SPECIES_128); } @Run(test = "testShort64ToFloat128") public static void testShort64ToFloat128_runner() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); VectorMask res = testShort64ToFloat128(mShort64); + mShort64 = mShort64.not(); Asserts.assertEquals(res.toString(), mShort64.toString()); Asserts.assertEquals(res.trueCount(), mShort64.trueCount()); } @@ -231,13 +244,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testShort64ToLong256(VectorMask v) { - return v.cast(LongVector.SPECIES_256); + return v.not().cast(LongVector.SPECIES_256); } @Run(test = "testShort64ToLong256") public static void testShort64ToLong256_runner() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); VectorMask res = testShort64ToLong256(mShort64); + mShort64 = mShort64.not(); Asserts.assertEquals(res.toString(), mShort64.toString()); Asserts.assertEquals(res.trueCount(), mShort64.trueCount()); } @@ -245,13 +259,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testShort64ToDouble256(VectorMask v) { - return v.cast(DoubleVector.SPECIES_256); + return v.not().cast(DoubleVector.SPECIES_256); } @Run(test = "testShort64ToDouble256") public static void testShort64ToDouble256_runner() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); VectorMask res = testShort64ToDouble256(mShort64); + mShort64 = mShort64.not(); Asserts.assertEquals(res.toString(), mShort64.toString()); Asserts.assertEquals(res.trueCount(), mShort64.trueCount()); } @@ -259,13 +274,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testShort128ToByte64(VectorMask v) { - return v.cast(ByteVector.SPECIES_64); + return v.not().cast(ByteVector.SPECIES_64); } @Run(test = "testShort128ToByte64") public static void testShort128ToByte64_runner() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); VectorMask res = testShort128ToByte64(mShort128); + mShort128 = mShort128.not(); Asserts.assertEquals(res.toString(), mShort128.toString()); Asserts.assertEquals(res.trueCount(), mShort128.trueCount()); } @@ -273,13 +289,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testShort128ToInt256(VectorMask v) { - return v.cast(IntVector.SPECIES_256); + return v.not().cast(IntVector.SPECIES_256); } @Run(test = "testShort128ToInt256") public static void testShort128ToInt256_runner() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); VectorMask res = testShort128ToInt256(mShort128); + mShort128 = mShort128.not(); Asserts.assertEquals(res.toString(), mShort128.toString()); Asserts.assertEquals(res.trueCount(), mShort128.trueCount()); } @@ -287,13 +304,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testShort128ToFloat256(VectorMask v) { - return v.cast(FloatVector.SPECIES_256); + return v.not().cast(FloatVector.SPECIES_256); } @Run(test = "testShort128ToFloat256") public static void testShort128ToFloat256_runner() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); VectorMask res = testShort128ToFloat256(mShort128); + mShort128 = mShort128.not(); Asserts.assertEquals(res.toString(), mShort128.toString()); Asserts.assertEquals(res.trueCount(), mShort128.trueCount()); } @@ -301,13 +319,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testShort128ToLong512(VectorMask v) { - return v.cast(LongVector.SPECIES_512); + return v.not().cast(LongVector.SPECIES_512); } @Run(test = "testShort128ToLong512") public static void testShort128ToLong512_runner() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); VectorMask res = testShort128ToLong512(mShort128); + mShort128 = mShort128.not(); Asserts.assertEquals(res.toString(), mShort128.toString()); Asserts.assertEquals(res.trueCount(), mShort128.trueCount()); } @@ -315,13 +334,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testShort128ToDouble512(VectorMask v) { - return v.cast(DoubleVector.SPECIES_512); + return v.not().cast(DoubleVector.SPECIES_512); } @Run(test = "testShort128ToDouble512") public static void testShort128ToDouble512_runner() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); VectorMask res = testShort128ToDouble512(mShort128); + mShort128 = mShort128.not(); Asserts.assertEquals(res.toString(), mShort128.toString()); Asserts.assertEquals(res.trueCount(), mShort128.trueCount()); } @@ -329,13 +349,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testShort256ToByte128(VectorMask v) { - return v.cast(ByteVector.SPECIES_128); + return v.not().cast(ByteVector.SPECIES_128); } @Run(test = "testShort256ToByte128") public static void testShort256ToByte128_runner() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); VectorMask res = testShort256ToByte128(mShort256); + mShort256 = mShort256.not(); Asserts.assertEquals(res.toString(), mShort256.toString()); Asserts.assertEquals(res.trueCount(), mShort256.trueCount()); } @@ -343,13 +364,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testShort256ToInt512(VectorMask v) { - return v.cast(IntVector.SPECIES_512); + return v.not().cast(IntVector.SPECIES_512); } @Run(test = "testShort256ToInt512") public static void testShort256ToInt512_runner() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); VectorMask res = testShort256ToInt512(mShort256); + mShort256 = mShort256.not(); Asserts.assertEquals(res.toString(), mShort256.toString()); Asserts.assertEquals(res.trueCount(), mShort256.trueCount()); } @@ -357,13 +379,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testShort256ToFloat512(VectorMask v) { - return v.cast(FloatVector.SPECIES_512); + return v.not().cast(FloatVector.SPECIES_512); } @Run(test = "testShort256ToFloat512") public static void testShort256ToFloat512_runner() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); VectorMask res = testShort256ToFloat512(mShort256); + mShort256 = mShort256.not(); Asserts.assertEquals(res.toString(), mShort256.toString()); Asserts.assertEquals(res.trueCount(), mShort256.trueCount()); } @@ -371,13 +394,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testShort512ToByte256(VectorMask v) { - return v.cast(ByteVector.SPECIES_256); + return v.not().cast(ByteVector.SPECIES_256); } @Run(test = "testShort512ToByte256") public static void testShort512ToByte256_runner() { VectorMask mShort512 = VectorMask.fromArray(ShortVector.SPECIES_512, mask_arr, 0); VectorMask res = testShort512ToByte256(mShort512); + mShort512 = mShort512.not(); Asserts.assertEquals(res.toString(), mShort512.toString()); Asserts.assertEquals(res.trueCount(), mShort512.trueCount()); } @@ -386,13 +410,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testInt64ToLong128(VectorMask v) { - return v.cast(LongVector.SPECIES_128); + return v.not().cast(LongVector.SPECIES_128); } @Run(test = "testInt64ToLong128") public static void testInt64ToLong128_runner() { VectorMask mInt64 = VectorMask.fromArray(IntVector.SPECIES_64, mask_arr, 0); VectorMask res = testInt64ToLong128(mInt64); + mInt64 = mInt64.not(); Asserts.assertEquals(res.toString(), mInt64.toString()); Asserts.assertEquals(res.trueCount(), mInt64.trueCount()); } @@ -400,13 +425,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testInt64ToDouble128(VectorMask v) { - return v.cast(DoubleVector.SPECIES_128); + return v.not().cast(DoubleVector.SPECIES_128); } @Run(test = "testInt64ToDouble128") public static void testInt64ToDouble128_runner() { VectorMask mInt64 = VectorMask.fromArray(IntVector.SPECIES_64, mask_arr, 0); VectorMask res = testInt64ToDouble128(mInt64); + mInt64 = mInt64.not(); Asserts.assertEquals(res.toString(), mInt64.toString()); Asserts.assertEquals(res.trueCount(), mInt64.trueCount()); } @@ -414,13 +440,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testInt128ToShort64(VectorMask v) { - return v.cast(ShortVector.SPECIES_64); + return v.not().cast(ShortVector.SPECIES_64); } @Run(test = "testInt128ToShort64") public static void testInt128ToShort64_runner() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); VectorMask res = testInt128ToShort64(mInt128); + mInt128 = mInt128.not(); Asserts.assertEquals(res.toString(), mInt128.toString()); Asserts.assertEquals(res.trueCount(), mInt128.trueCount()); } @@ -428,13 +455,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testInt128ToLong256(VectorMask v) { - return v.cast(LongVector.SPECIES_256); + return v.not().cast(LongVector.SPECIES_256); } @Run(test = "testInt128ToLong256") public static void testInt128ToLong256_runner() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); VectorMask res = testInt128ToLong256(mInt128); + mInt128 = mInt128.not(); Asserts.assertEquals(res.toString(), mInt128.toString()); Asserts.assertEquals(res.trueCount(), mInt128.trueCount()); } @@ -442,13 +470,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testInt128ToDouble256(VectorMask v) { - return v.cast(DoubleVector.SPECIES_256); + return v.not().cast(DoubleVector.SPECIES_256); } @Run(test = "testInt128ToDouble256") public static void testInt128ToDouble256_runner() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); VectorMask res = testInt128ToDouble256(mInt128); + mInt128 = mInt128.not(); Asserts.assertEquals(res.toString(), mInt128.toString()); Asserts.assertEquals(res.trueCount(), mInt128.trueCount()); } @@ -456,13 +485,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testInt256ToShort128(VectorMask v) { - return v.cast(ShortVector.SPECIES_128); + return v.not().cast(ShortVector.SPECIES_128); } @Run(test = "testInt256ToShort128") public static void testInt256ToShort128_runner() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); VectorMask res = testInt256ToShort128(mInt256); + mInt256 = mInt256.not(); Asserts.assertEquals(res.toString(), mInt256.toString()); Asserts.assertEquals(res.trueCount(), mInt256.trueCount()); } @@ -470,13 +500,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testInt256ToByte64(VectorMask v) { - return v.cast(ByteVector.SPECIES_64); + return v.not().cast(ByteVector.SPECIES_64); } @Run(test = "testInt256ToByte64") public static void testInt256ToByte64_runner() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); VectorMask res = testInt256ToByte64(mInt256); + mInt256 = mInt256.not(); Asserts.assertEquals(res.toString(), mInt256.toString()); Asserts.assertEquals(res.trueCount(), mInt256.trueCount()); } @@ -484,13 +515,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testInt256ToLong512(VectorMask v) { - return v.cast(LongVector.SPECIES_512); + return v.not().cast(LongVector.SPECIES_512); } @Run(test = "testInt256ToLong512") public static void testInt256ToLong512_runner() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); VectorMask res = testInt256ToLong512(mInt256); + mInt256 = mInt256.not(); Asserts.assertEquals(res.toString(), mInt256.toString()); Asserts.assertEquals(res.trueCount(), mInt256.trueCount()); } @@ -498,13 +530,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testInt256ToDouble512(VectorMask v) { - return v.cast(DoubleVector.SPECIES_512); + return v.not().cast(DoubleVector.SPECIES_512); } @Run(test = "testInt256ToDouble512") public static void testInt256ToDouble512_runner() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); VectorMask res = testInt256ToDouble512(mInt256); + mInt256 = mInt256.not(); Asserts.assertEquals(res.toString(), mInt256.toString()); Asserts.assertEquals(res.trueCount(), mInt256.trueCount()); } @@ -512,13 +545,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testInt512ToShort256(VectorMask v) { - return v.cast(ShortVector.SPECIES_256); + return v.not().cast(ShortVector.SPECIES_256); } @Run(test = "testInt512ToShort256") public static void testInt512ToShort256_runner() { VectorMask mInt512 = VectorMask.fromArray(IntVector.SPECIES_512, mask_arr, 0); VectorMask res = testInt512ToShort256(mInt512); + mInt512 = mInt512.not(); Asserts.assertEquals(res.toString(), mInt512.toString()); Asserts.assertEquals(res.trueCount(), mInt512.trueCount()); } @@ -526,13 +560,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testInt512ToByte128(VectorMask v) { - return v.cast(ByteVector.SPECIES_128); + return v.not().cast(ByteVector.SPECIES_128); } @Run(test = "testInt512ToByte128") public static void testInt512ToByte128_runner() { VectorMask mInt512 = VectorMask.fromArray(IntVector.SPECIES_512, mask_arr, 0); VectorMask res = testInt512ToByte128(mInt512); + mInt512 = mInt512.not(); Asserts.assertEquals(res.toString(), mInt512.toString()); Asserts.assertEquals(res.trueCount(), mInt512.trueCount()); } @@ -541,13 +576,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testFloat64ToLong128(VectorMask v) { - return v.cast(LongVector.SPECIES_128); + return v.not().cast(LongVector.SPECIES_128); } @Run(test = "testFloat64ToLong128") public static void testFloat64ToLong128_runner() { VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mask_arr, 0); VectorMask res = testFloat64ToLong128(mFloat64); + mFloat64 = mFloat64.not(); Asserts.assertEquals(res.toString(), mFloat64.toString()); Asserts.assertEquals(res.trueCount(), mFloat64.trueCount()); } @@ -555,13 +591,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testFloat64ToDouble128(VectorMask v) { - return v.cast(DoubleVector.SPECIES_128); + return v.not().cast(DoubleVector.SPECIES_128); } @Run(test = "testFloat64ToDouble128") public static void testFloat64ToDouble128_runner() { VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mask_arr, 0); VectorMask res = testFloat64ToDouble128(mFloat64); + mFloat64 = mFloat64.not(); Asserts.assertEquals(res.toString(), mFloat64.toString()); Asserts.assertEquals(res.trueCount(), mFloat64.trueCount()); } @@ -569,13 +606,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true", "rvv", "true"}) public static VectorMask testFloat128ToShort64(VectorMask v) { - return v.cast(ShortVector.SPECIES_64); + return v.not().cast(ShortVector.SPECIES_64); } @Run(test = "testFloat128ToShort64") public static void testFloat128ToShort64_runner() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); VectorMask res = testFloat128ToShort64(mFloat128); + mFloat128 = mFloat128.not(); Asserts.assertEquals(res.toString(), mFloat128.toString()); Asserts.assertEquals(res.trueCount(), mFloat128.trueCount()); } @@ -583,13 +621,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testFloat128ToLong256(VectorMask v) { - return v.cast(LongVector.SPECIES_256); + return v.not().cast(LongVector.SPECIES_256); } @Run(test = "testFloat128ToLong256") public static void testFloat128ToLong256_runner() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); VectorMask res = testFloat128ToLong256(mFloat128); + mFloat128 = mFloat128.not(); Asserts.assertEquals(res.toString(), mFloat128.toString()); Asserts.assertEquals(res.trueCount(), mFloat128.trueCount()); } @@ -597,13 +636,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testFloat128ToDouble256(VectorMask v) { - return v.cast(DoubleVector.SPECIES_256); + return v.not().cast(DoubleVector.SPECIES_256); } @Run(test = "testFloat128ToDouble256") public static void testFloat128ToDouble256_runner() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); VectorMask res = testFloat128ToDouble256(mFloat128); + mFloat128 = mFloat128.not(); Asserts.assertEquals(res.toString(), mFloat128.toString()); Asserts.assertEquals(res.trueCount(), mFloat128.trueCount()); } @@ -611,13 +651,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testFloat256ToShort128(VectorMask v) { - return v.cast(ShortVector.SPECIES_128); + return v.not().cast(ShortVector.SPECIES_128); } @Run(test = "testFloat256ToShort128") public static void testFloat256ToShort128_runner() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); VectorMask res = testFloat256ToShort128(mFloat256); + mFloat256 = mFloat256.not(); Asserts.assertEquals(res.toString(), mFloat256.toString()); Asserts.assertEquals(res.trueCount(), mFloat256.trueCount()); } @@ -625,13 +666,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testFloat256ToByte64(VectorMask v) { - return v.cast(ByteVector.SPECIES_64); + return v.not().cast(ByteVector.SPECIES_64); } @Run(test = "testFloat256ToByte64") public static void testFloat256ToByte64_runner() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); VectorMask res = testFloat256ToByte64(mFloat256); + mFloat256 = mFloat256.not(); Asserts.assertEquals(res.toString(), mFloat256.toString()); Asserts.assertEquals(res.trueCount(), mFloat256.trueCount()); } @@ -639,13 +681,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testFloat256ToLong512(VectorMask v) { - return v.cast(LongVector.SPECIES_512); + return v.not().cast(LongVector.SPECIES_512); } @Run(test = "testFloat256ToLong512") public static void testFloat256ToLong512_runner() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); VectorMask res = testFloat256ToLong512(mFloat256); + mFloat256 = mFloat256.not(); Asserts.assertEquals(res.toString(), mFloat256.toString()); Asserts.assertEquals(res.trueCount(), mFloat256.trueCount()); } @@ -653,13 +696,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testFloat256ToDouble512(VectorMask v) { - return v.cast(DoubleVector.SPECIES_512); + return v.not().cast(DoubleVector.SPECIES_512); } @Run(test = "testFloat256ToDouble512") public static void testFloat256ToDouble512_runner() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); VectorMask res = testFloat256ToDouble512(mFloat256); + mFloat256 = mFloat256.not(); Asserts.assertEquals(res.toString(), mFloat256.toString()); Asserts.assertEquals(res.trueCount(), mFloat256.trueCount()); } @@ -667,13 +711,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testFloat512ToShort256(VectorMask v) { - return v.cast(ShortVector.SPECIES_256); + return v.not().cast(ShortVector.SPECIES_256); } @Run(test = "testFloat512ToShort256") public static void testFloat512ToShort256_runner() { VectorMask mFloat512 = VectorMask.fromArray(FloatVector.SPECIES_512, mask_arr, 0); VectorMask res = testFloat512ToShort256(mFloat512); + mFloat512 = mFloat512.not(); Asserts.assertEquals(res.toString(), mFloat512.toString()); Asserts.assertEquals(res.trueCount(), mFloat512.trueCount()); } @@ -681,13 +726,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testFloat512ToByte128(VectorMask v) { - return v.cast(ByteVector.SPECIES_128); + return v.not().cast(ByteVector.SPECIES_128); } @Run(test = "testFloat512ToByte128") public static void testFloat512ToByte128_runner() { VectorMask mFloat512 = VectorMask.fromArray(FloatVector.SPECIES_512, mask_arr, 0); VectorMask res = testFloat512ToByte128(mFloat512); + mFloat512 = mFloat512.not(); Asserts.assertEquals(res.toString(), mFloat512.toString()); Asserts.assertEquals(res.trueCount(), mFloat512.trueCount()); } @@ -696,13 +742,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testLong128ToInt64(VectorMask v) { - return v.cast(IntVector.SPECIES_64); + return v.not().cast(IntVector.SPECIES_64); } @Run(test = "testLong128ToInt64") public static void testLong128ToInt64_runner() { VectorMask mLong128 = VectorMask.fromArray(LongVector.SPECIES_128, mask_arr, 0); VectorMask res = testLong128ToInt64(mLong128); + mLong128 = mLong128.not(); Asserts.assertEquals(res.toString(), mLong128.toString()); Asserts.assertEquals(res.trueCount(), mLong128.trueCount()); } @@ -710,13 +757,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testLong128ToFloat64(VectorMask v) { - return v.cast(FloatVector.SPECIES_64); + return v.not().cast(FloatVector.SPECIES_64); } @Run(test = "testLong128ToFloat64") public static void testLong128ToFloat64_runner() { VectorMask mLong128 = VectorMask.fromArray(LongVector.SPECIES_128, mask_arr, 0); VectorMask res = testLong128ToFloat64(mLong128); + mLong128 = mLong128.not(); Asserts.assertEquals(res.toString(), mLong128.toString()); Asserts.assertEquals(res.trueCount(), mLong128.trueCount()); } @@ -724,13 +772,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testLong256ToInt128(VectorMask v) { - return v.cast(IntVector.SPECIES_128); + return v.not().cast(IntVector.SPECIES_128); } @Run(test = "testLong256ToInt128") public static void testLong256ToInt128_runner() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); VectorMask res = testLong256ToInt128(mLong256); + mLong256 = mLong256.not(); Asserts.assertEquals(res.toString(), mLong256.toString()); Asserts.assertEquals(res.trueCount(), mLong256.trueCount()); } @@ -738,13 +787,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testLong256ToFloat128(VectorMask v) { - return v.cast(FloatVector.SPECIES_128); + return v.not().cast(FloatVector.SPECIES_128); } @Run(test = "testLong256ToFloat128") public static void testLong256ToFloat128_runner() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); VectorMask res = testLong256ToFloat128(mLong256); + mLong256 = mLong256.not(); Asserts.assertEquals(res.toString(), mLong256.toString()); Asserts.assertEquals(res.trueCount(), mLong256.trueCount()); } @@ -752,13 +802,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testLong256ToShort64(VectorMask v) { - return v.cast(ShortVector.SPECIES_64); + return v.not().cast(ShortVector.SPECIES_64); } @Run(test = "testLong256ToShort64") public static void testLong256ToShort64_runner() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); VectorMask res = testLong256ToShort64(mLong256); + mLong256 = mLong256.not(); Asserts.assertEquals(res.toString(), mLong256.toString()); Asserts.assertEquals(res.trueCount(), mLong256.trueCount()); } @@ -766,13 +817,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testLong512ToInt256(VectorMask v) { - return v.cast(IntVector.SPECIES_256); + return v.not().cast(IntVector.SPECIES_256); } @Run(test = "testLong512ToInt256") public static void testLong512ToInt256_runner() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); VectorMask res = testLong512ToInt256(mLong512); + mLong512 = mLong512.not(); Asserts.assertEquals(res.toString(), mLong512.toString()); Asserts.assertEquals(res.trueCount(), mLong512.trueCount()); } @@ -780,13 +832,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testLong512ToFloat256(VectorMask v) { - return v.cast(FloatVector.SPECIES_256); + return v.not().cast(FloatVector.SPECIES_256); } @Run(test = "testLong512ToFloat256") public static void testLong512ToFloat256_runner() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); VectorMask res = testLong512ToFloat256(mLong512); + mLong512 = mLong512.not(); Asserts.assertEquals(res.toString(), mLong512.toString()); Asserts.assertEquals(res.trueCount(), mLong512.trueCount()); } @@ -794,13 +847,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testLong512ToShort128(VectorMask v) { - return v.cast(ShortVector.SPECIES_128); + return v.not().cast(ShortVector.SPECIES_128); } @Run(test = "testLong512ToShort128") public static void testLong512ToShort128_runner() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); VectorMask res = testLong512ToShort128(mLong512); + mLong512 = mLong512.not(); Asserts.assertEquals(res.toString(), mLong512.toString()); Asserts.assertEquals(res.trueCount(), mLong512.trueCount()); } @@ -808,13 +862,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testLong512ToByte64(VectorMask v) { - return v.cast(ByteVector.SPECIES_64); + return v.not().cast(ByteVector.SPECIES_64); } @Run(test = "testLong512ToByte64") public static void testLong512ToByte64_runner() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); VectorMask res = testLong512ToByte64(mLong512); + mLong512 = mLong512.not(); Asserts.assertEquals(res.toString(), mLong512.toString()); Asserts.assertEquals(res.trueCount(), mLong512.trueCount()); } @@ -823,13 +878,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testDouble128ToInt64(VectorMask v) { - return v.cast(IntVector.SPECIES_64); + return v.not().cast(IntVector.SPECIES_64); } @Run(test = "testDouble128ToInt64") public static void testDouble128ToInt64_runner() { VectorMask mDouble128 = VectorMask.fromArray(DoubleVector.SPECIES_128, mask_arr, 0); VectorMask res = testDouble128ToInt64(mDouble128); + mDouble128 = mDouble128.not(); Asserts.assertEquals(res.toString(), mDouble128.toString()); Asserts.assertEquals(res.trueCount(), mDouble128.trueCount()); } @@ -837,13 +893,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"asimd", "true", "rvv", "true"}) public static VectorMask testDouble128ToFloat64(VectorMask v) { - return v.cast(FloatVector.SPECIES_64); + return v.not().cast(FloatVector.SPECIES_64); } @Run(test = "testDouble128ToFloat64") public static void testDouble128ToFloat64_runner() { VectorMask mDouble128 = VectorMask.fromArray(DoubleVector.SPECIES_128, mask_arr, 0); VectorMask res = testDouble128ToFloat64(mDouble128); + mDouble128 = mDouble128.not(); Asserts.assertEquals(res.toString(), mDouble128.toString()); Asserts.assertEquals(res.trueCount(), mDouble128.trueCount()); } @@ -851,13 +908,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testDouble256ToInt128(VectorMask v) { - return v.cast(IntVector.SPECIES_128); + return v.not().cast(IntVector.SPECIES_128); } @Run(test = "testDouble256ToInt128") public static void testDouble256ToInt128_runner() { VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); VectorMask res = testDouble256ToInt128(mDouble256); + mDouble256 = mDouble256.not(); Asserts.assertEquals(res.toString(), mDouble256.toString()); Asserts.assertEquals(res.trueCount(), mDouble256.trueCount()); } @@ -865,13 +923,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testDouble256ToFloat128(VectorMask v) { - return v.cast(FloatVector.SPECIES_128); + return v.not().cast(FloatVector.SPECIES_128); } @Run(test = "testDouble256ToFloat128") public static void testDouble256ToFloat128_runner() { VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); VectorMask res = testDouble256ToFloat128(mDouble256); + mDouble256 = mDouble256.not(); Asserts.assertEquals(res.toString(), mDouble256.toString()); Asserts.assertEquals(res.trueCount(), mDouble256.trueCount()); } @@ -879,13 +938,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) public static VectorMask testDouble256ToShort64(VectorMask v) { - return v.cast(ShortVector.SPECIES_64); + return v.not().cast(ShortVector.SPECIES_64); } @Run(test = "testDouble256ToShort64") public static void testDouble256ToShort64_runner() { VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); VectorMask res = testDouble256ToShort64(mDouble256); + mDouble256 = mDouble256.not(); Asserts.assertEquals(res.toString(), mDouble256.toString()); Asserts.assertEquals(res.trueCount(), mDouble256.trueCount()); } @@ -893,13 +953,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testDouble512ToInt256(VectorMask v) { - return v.cast(IntVector.SPECIES_256); + return v.not().cast(IntVector.SPECIES_256); } @Run(test = "testDouble512ToInt256") public static void testDouble512ToInt256_runner() { VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); VectorMask res = testDouble512ToInt256(mDouble512); + mDouble512 = mDouble512.not(); Asserts.assertEquals(res.toString(), mDouble512.toString()); Asserts.assertEquals(res.trueCount(), mDouble512.trueCount()); } @@ -907,13 +968,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testDouble512ToFloat256(VectorMask v) { - return v.cast(FloatVector.SPECIES_256); + return v.not().cast(FloatVector.SPECIES_256); } @Run(test = "testDouble512ToFloat256") public static void testDouble512ToFloat256_runner() { VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); VectorMask res = testDouble512ToFloat256(mDouble512); + mDouble512 = mDouble512.not(); Asserts.assertEquals(res.toString(), mDouble512.toString()); Asserts.assertEquals(res.trueCount(), mDouble512.trueCount()); } @@ -921,13 +983,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testDouble512ToShort128(VectorMask v) { - return v.cast(ShortVector.SPECIES_128); + return v.not().cast(ShortVector.SPECIES_128); } @Run(test = "testDouble512ToShort128") public static void testDouble512ToShort128_runner() { VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); VectorMask res = testDouble512ToShort128(mDouble512); + mDouble512 = mDouble512.not(); Asserts.assertEquals(res.toString(), mDouble512.toString()); Asserts.assertEquals(res.trueCount(), mDouble512.trueCount()); } @@ -935,13 +998,14 @@ public class VectorMaskCastTest { @Test @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) public static VectorMask testDouble512ToByte64(VectorMask v) { - return v.cast(ByteVector.SPECIES_64); + return v.not().cast(ByteVector.SPECIES_64); } @Run(test = "testDouble512ToByte64") public static void testDouble512ToByte64_runner() { VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); VectorMask res = testDouble512ToByte64(mDouble512); + mDouble512 = mDouble512.not(); Asserts.assertEquals(res.toString(), mDouble512.toString()); Asserts.assertEquals(res.trueCount(), mDouble512.trueCount()); } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskToLongTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskToLongTest.java index 35a5aca966a..43415496072 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskToLongTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskToLongTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * Copyright (c) 2025, 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -236,10 +236,29 @@ public class VectorMaskToLongTest { verifyMaskToLong(L_SPECIES, inputLong, got); } + // Test VectorMask.fromLong().toLong() with Float species. + // For floating-point types, VectorMaskCast is inserted between fromLong and toLong to convert + // between float and integer types. There are two relevant optimizations: + // 1. (VectorStoreMask (VectorMaskCast* (VectorLoadMask x))) => (x) + // 2. (VectorMaskToLong (VectorLongToMask x)) => (x) + // The optimization behavior varies by architecture: + // - SVE with bitperm: IR chain is (VectorMaskToLong (VectorStoreMask (VectorMaskCast + // (VectorLoadMask (VectorLongToMask x))))), so both optimizations are triggered. + // - AVX-512/RVV: IR pattern is (VectorMaskToLong (VectorMaskCast (VectorLongToMask x))), + // so neither optimization is triggered. + // - AVX2: Same as SVE with bitperm, both optimizations are triggered. + // - ASIMD (without SVE bitperm): VectorLongToMaskNode is not supported, + // so neither optimization is triggered. @Test + @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", + IRNode.VECTOR_MASK_TO_LONG, "= 0" }, + applyIfCPUFeature = { "svebitperm", "true" }) @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 1", IRNode.VECTOR_MASK_TO_LONG, "= 1" }, - applyIfCPUFeatureOr = { "svebitperm", "true", "avx2", "true", "rvv", "true" }) + applyIfCPUFeatureOr = { "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", + IRNode.VECTOR_MASK_TO_LONG, "= 0" }, + applyIfCPUFeatureAnd = { "avx2", "true", "avx512", "false" }) @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", IRNode.VECTOR_MASK_TO_LONG, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "svebitperm", "false" }) @@ -250,10 +269,18 @@ public class VectorMaskToLongTest { verifyMaskToLong(F_SPECIES, inputLong, got); } + // Test VectorMask.fromLong().toLong() with Double species. + // Same as testFromLongToLongFloat() - see comments there for detailed explanation. @Test + @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", + IRNode.VECTOR_MASK_TO_LONG, "= 0" }, + applyIfCPUFeature = { "svebitperm", "true" }) @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 1", IRNode.VECTOR_MASK_TO_LONG, "= 1" }, - applyIfCPUFeatureOr = { "svebitperm", "true", "avx2", "true", "rvv", "true" }) + applyIfCPUFeatureOr = { "avx512", "true", "rvv", "true" }) + @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", + IRNode.VECTOR_MASK_TO_LONG, "= 0" }, + applyIfCPUFeatureAnd = { "avx2", "true", "avx512", "false" }) @IR(counts = { IRNode.VECTOR_LONG_TO_MASK, "= 0", IRNode.VECTOR_MASK_TO_LONG, "= 1" }, applyIfCPUFeatureAnd = { "asimd", "true", "svebitperm", "false" }) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorStoreMaskIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorStoreMaskIdentityTest.java new file mode 100644 index 00000000000..c019f116373 --- /dev/null +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorStoreMaskIdentityTest.java @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* +* @test +* @bug 8370863 +* @library /test/lib / +* @summary VectorStoreMaskNode Identity optimization tests +* @modules jdk.incubator.vector +* +* @run driver ${test.main.class} +*/ + +package compiler.vectorapi; + +import compiler.lib.ir_framework.*; +import jdk.incubator.vector.*; +import jdk.test.lib.Asserts; + +public class VectorStoreMaskIdentityTest { + private static final VectorSpecies B64 = ByteVector.SPECIES_64; + private static final VectorSpecies S64 = ShortVector.SPECIES_64; + private static final VectorSpecies S128 = ShortVector.SPECIES_128; + private static final VectorSpecies I64 = IntVector.SPECIES_64; + private static final VectorSpecies I128 = IntVector.SPECIES_128; + private static final VectorSpecies I256 = IntVector.SPECIES_256; + private static final VectorSpecies L128 = LongVector.SPECIES_128; + private static final VectorSpecies L256 = LongVector.SPECIES_256; + private static final VectorSpecies F128 = FloatVector.SPECIES_128; + private static final VectorSpecies F256 = FloatVector.SPECIES_256; + private static final VectorSpecies D128 = DoubleVector.SPECIES_128; + private static final VectorSpecies D256 = DoubleVector.SPECIES_256; + private static final int LENGTH = 256; // large enough + private static boolean[] mask_in; + private static boolean[] mask_out; + static { + mask_in = new boolean[LENGTH]; + mask_out = new boolean[LENGTH]; + for (int i = 0; i < LENGTH; i++) { + mask_in[i] = (i & 3) == 0; + } + } + + @DontInline + private static void verifyResult(int vlen) { + for (int i = 0; i < vlen; i++) { + Asserts.assertEquals(mask_in[i], mask_out[i], "index " + i); + } + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_8, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityByte() { + VectorMask mask_byte_64 = VectorMask.fromArray(B64, mask_in, 0); + + mask_byte_64.cast(S128).intoArray(mask_out, 0); + verifyResult(B64.length()); + + mask_byte_64.cast(S128).cast(B64).intoArray(mask_out, 0); + verifyResult(B64.length()); + + mask_byte_64.cast(S128).cast(B64).cast(S128).intoArray(mask_out, 0); + verifyResult(B64.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_8, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityByte256() { + VectorMask mask_byte_64 = VectorMask.fromArray(B64, mask_in, 0); + + mask_byte_64.cast(I256).intoArray(mask_out, 0); + verifyResult(B64.length()); + + mask_byte_64.cast(S128).cast(F256).intoArray(mask_out, 0); + verifyResult(B64.length()); + + mask_byte_64.cast(F256).cast(S128).cast(I256).intoArray(mask_out, 0); + verifyResult(B64.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_8, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityShort() { + VectorMask mask_short_128 = VectorMask.fromArray(S128, mask_in, 0); + + mask_short_128.cast(B64).intoArray(mask_out, 0); + verifyResult(S128.length()); + + mask_short_128.cast(B64).cast(S128).intoArray(mask_out, 0); + verifyResult(S128.length()); + + mask_short_128.cast(B64).cast(S128).cast(B64).intoArray(mask_out, 0); + verifyResult(S128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_8, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityShort256() { + VectorMask mask_short_128 = VectorMask.fromArray(S128, mask_in, 0); + + mask_short_128.cast(I256).intoArray(mask_out, 0); + verifyResult(S128.length()); + + mask_short_128.cast(B64).cast(I256).intoArray(mask_out, 0); + verifyResult(S128.length()); + + mask_short_128.cast(F256).cast(B64).cast(I256).intoArray(mask_out, 0); + verifyResult(S128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityInt() { + VectorMask mask_int_128 = VectorMask.fromArray(I128, mask_in, 0); + + mask_int_128.cast(F128).intoArray(mask_out, 0); + verifyResult(I128.length()); + + mask_int_128.cast(S64).cast(F128).intoArray(mask_out, 0); + verifyResult(I128.length()); + + mask_int_128.cast(F128).cast(I128).cast(S64).intoArray(mask_out, 0); + verifyResult(I128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityInt256() { + VectorMask mask_int_128 = VectorMask.fromArray(I128, mask_in, 0); + + mask_int_128.cast(F128).intoArray(mask_out, 0); + verifyResult(I128.length()); + + mask_int_128.cast(S64).cast(L256).intoArray(mask_out, 0); + verifyResult(I128.length()); + + mask_int_128.cast(L256).cast(S64).cast(F128).intoArray(mask_out, 0); + verifyResult(I128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_2, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityLong() { + VectorMask mask_long_128 = VectorMask.fromArray(L128, mask_in, 0); + + mask_long_128.cast(D128).intoArray(mask_out, 0); + verifyResult(L128.length()); + + mask_long_128.cast(I64).cast(D128).intoArray(mask_out, 0); + verifyResult(L128.length()); + + mask_long_128.cast(I64).cast(D128).cast(I64).intoArray(mask_out, 0); + verifyResult(L128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityLong256() { + VectorMask mask_long_256 = VectorMask.fromArray(L256, mask_in, 0); + + mask_long_256.cast(I128).intoArray(mask_out, 0); + verifyResult(L256.length()); + + mask_long_256.cast(S64).cast(I128).intoArray(mask_out, 0); + verifyResult(L256.length()); + + mask_long_256.cast(F128).cast(I128).cast(S64).intoArray(mask_out, 0); + verifyResult(L256.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true", "avx", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityFloat() { + VectorMask mask_float_128 = VectorMask.fromArray(F128, mask_in, 0); + + mask_float_128.cast(I128).intoArray(mask_out, 0); + verifyResult(F128.length()); + + mask_float_128.cast(S64).cast(I128).intoArray(mask_out, 0); + verifyResult(F128.length()); + + mask_float_128.cast(S64).cast(I128).cast(S64).intoArray(mask_out, 0); + verifyResult(F128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityFloat256() { + VectorMask mask_float_128 = VectorMask.fromArray(F128, mask_in, 0); + + mask_float_128.cast(I128).intoArray(mask_out, 0); + verifyResult(F128.length()); + + mask_float_128.cast(S64).cast(L256).intoArray(mask_out, 0); + verifyResult(F128.length()); + + mask_float_128.cast(L256).cast(S64).cast(I128).intoArray(mask_out, 0); + verifyResult(F128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_2, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "asimd", "true" }, + applyIf = { "MaxVectorSize", ">= 16" }) + public static void testVectorMaskStoreIdentityDouble() { + VectorMask mask_double_128 = VectorMask.fromArray(D128, mask_in, 0); + + mask_double_128.cast(L128).intoArray(mask_out, 0); + verifyResult(D128.length()); + + mask_double_128.cast(I64).cast(L128).intoArray(mask_out, 0); + verifyResult(D128.length()); + + mask_double_128.cast(I64).cast(L128).cast(I64).intoArray(mask_out, 0); + verifyResult(D128.length()); + } + + @Test + @IR(counts = { IRNode.LOAD_VECTOR_Z, IRNode.VECTOR_SIZE_4, ">= 1", + IRNode.VECTOR_LOAD_MASK, "= 0", + IRNode.VECTOR_STORE_MASK, "= 0", + IRNode.VECTOR_MASK_CAST, "= 0" }, + applyIfCPUFeatureOr = { "sve", "true", "avx2", "true" }, + applyIf = { "MaxVectorSize", "> 16" }) + public static void testVectorMaskStoreIdentityDouble256() { + VectorMask mask_double_256 = VectorMask.fromArray(D256, mask_in, 0); + + mask_double_256.cast(F128).intoArray(mask_out, 0); + verifyResult(D256.length()); + + mask_double_256.cast(S64).cast(I128).intoArray(mask_out, 0); + verifyResult(D256.length()); + + mask_double_256.cast(I128).cast(S64).cast(L256).intoArray(mask_out, 0); + verifyResult(D256.length()); + } + + public static void main(String[] args) { + TestFramework testFramework = new TestFramework(); + testFramework.setDefaultWarmup(10000) + .addFlags("--add-modules=jdk.incubator.vector") + .start(); + } +} \ No newline at end of file diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/VectorStoreMaskBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/VectorStoreMaskBenchmark.java new file mode 100644 index 00000000000..d4dc321ccba --- /dev/null +++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/VectorStoreMaskBenchmark.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package org.openjdk.bench.jdk.incubator.vector; + +import jdk.incubator.vector.*; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; + +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Thread) +@Warmup(iterations = 10, time = 1) +@Measurement(iterations = 10, time = 1) +@Fork(value = 1, jvmArgs = {"--add-modules=jdk.incubator.vector"}) +public class VectorStoreMaskBenchmark { + static final int LENGTH = 256; + static final boolean[] mask_arr_input = new boolean[LENGTH]; + static final boolean[] mask_arr_output = new boolean[LENGTH]; + static { + for (int i = 0; i < LENGTH; i++) { + mask_arr_input[i] = (i & 1) == 0; + } + } + + @CompilerControl(CompilerControl.Mode.INLINE) + public void maskLoadCastStoreKernel(VectorSpecies species_from, VectorSpecies species_to) { + for (int i = 0; i < LENGTH; i += species_from.length()) { + VectorMask mask_from = VectorMask.fromArray(species_from, mask_arr_input, i); + VectorMask mask_to = mask_from.cast(species_to); + mask_to.intoArray(mask_arr_output, i); + } + } + + @Benchmark + public void microMaskLoadCastStoreByte64() { + maskLoadCastStoreKernel(ByteVector.SPECIES_64, ShortVector.SPECIES_128); + } + + @Benchmark + public void microMaskLoadCastStoreShort64() { + maskLoadCastStoreKernel(ShortVector.SPECIES_64, IntVector.SPECIES_128); + } + + @Benchmark + public void microMaskLoadCastStoreInt128() { + maskLoadCastStoreKernel(IntVector.SPECIES_128, ShortVector.SPECIES_64); + } + + @Benchmark + public void microMaskLoadCastStoreLong128() { + maskLoadCastStoreKernel(LongVector.SPECIES_128, IntVector.SPECIES_64); + } + + @Benchmark + public void microMaskLoadCastStoreFloat128() { + maskLoadCastStoreKernel(FloatVector.SPECIES_128, ShortVector.SPECIES_64); + } + + @Benchmark + public void microMaskLoadCastStoreDouble128() { + maskLoadCastStoreKernel(DoubleVector.SPECIES_128, IntVector.SPECIES_64); + } +} \ No newline at end of file