From 11e28bd61968700956d2155a77688459fd7c028f Mon Sep 17 00:00:00 2001 From: Emanuel Peter Date: Tue, 30 Jan 2024 20:14:20 +0000 Subject: [PATCH] 8324794: C2 SuperWord: do not ignore reductions in SuperWord::unrolling_analysis Reviewed-by: chagedorn, kvn --- src/hotspot/share/opto/superword.cpp | 1 - ...estUnorderedReductionPartialVectorization.java | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index 0b876504e4e..24e7dab2e28 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -177,7 +177,6 @@ void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) { for (uint i = 0; i < lpt()->_body.size(); i++) { Node* n = lpt()->_body.at(i); if (n == cl->incr() || - is_marked_reduction(n) || n->is_AddP() || n->is_Cmp() || n->is_Bool() || diff --git a/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java index 431bbe9ac4f..666ddaac441 100644 --- a/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java +++ b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -64,7 +64,7 @@ public class TestUnorderedReductionPartialVectorization { IRNode.OR_REDUCTION_V, "> 0",}, applyIfCPUFeatureOr = {"avx2", "true"}) static long test1(int[] data, long sum) { - for (int i = 0; i < data.length; i++) { + for (int i = 0; i < data.length; i+=2) { // Mixing int and long ops means we only end up allowing half of the int // loads in one pack, and we have two int packs. The first pack has one // of the pairs missing because of the store, which creates a dependency. @@ -77,6 +77,17 @@ public class TestUnorderedReductionPartialVectorization { int v = data[i]; // int read data[0] = 0; // ruin the first pack sum |= v; // long reduction (and implicit cast from int to long) + + // This example used to rely on that reductions were ignored in SuperWord::unrolling_analysis, + // and hence the largest data type in the loop was the ints. This would then unroll the doubles + // for twice the vector length, and this resulted in us having twice as many packs. Because of + // the store "data[0] = 0", the first packs were destroyed, since they do not have power of 2 + // size. + // Now, we no longer ignore reductions, and now we unroll half as much before SuperWord. This + // means we would only get one pack per operation, and that one would get ruined, and we have + // no vectorization. We now ensure there are again 2 packs per operation with a 2x hand unroll. + int v2 = data[i + 1]; + sum |= v2; } return sum; }