8373069: RISC-V: implement GHASH intrinsic

Reviewed-by: fjiang, fyang

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -2662,6 +2662,9 @@ enum Nf {
   INSN(vsha2ch_vv,  0b1110111, 0b010, 0b1, 0b101110);
   INSN(vsha2cl_vv,  0b1110111, 0b010, 0b1, 0b101111);
 
+  // Vector GHASH (Zvkg) Extension
+  INSN(vghsh_vv,    0b1110111, 0b010, 0b1, 0b101100);
+
 #undef INSN
 
 #define INSN(NAME, op, funct3, Vs1, funct6)                                    \

src/hotspot/cpu/riscv/globals_riscv.hpp

@@ -123,6 +123,7 @@ define_pd_global(intx, InlineSmallCode,          1000);
   product(bool, UseZvkn, false, EXPERIMENTAL,                                    \
           "Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt")                \
   product(bool, UseCtxFencei, false, EXPERIMENTAL,                               \
-          "Use PR_RISCV_CTX_SW_FENCEI_ON to avoid explicit icache flush")
+          "Use PR_RISCV_CTX_SW_FENCEI_ON to avoid explicit icache flush")        \
+  product(bool, UseZvkg, false, EXPERIMENTAL, "Use Zvkg instructions")
 
 #endif // CPU_RISCV_GLOBALS_RISCV_HPP

src/hotspot/cpu/riscv/stubGenerator_riscv.cpp

@@ -2655,8 +2655,7 @@ class StubGenerator: public StubCodeGenerator {
   //   x10       - input length
   //
   address generate_cipherBlockChaining_encryptAESCrypt() {
-    assert(UseAESIntrinsics, "Must be");
-    assert(UseZvkn, "need AES instructions (Zvkned extension) support");
+    assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
     __ align(CodeEntryAlignment);
     StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
     StubCodeMark mark(this, stub_id);
@@ -2745,8 +2744,7 @@ class StubGenerator: public StubCodeGenerator {
   //   x10       - input length
   //
   address generate_cipherBlockChaining_decryptAESCrypt() {
-    assert(UseAESIntrinsics, "Must be");
-    assert(UseZvkn, "need AES instructions (Zvkned extension) support");
+    assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
     __ align(CodeEntryAlignment);
     StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
     StubCodeMark mark(this, stub_id);
@@ -2950,9 +2948,7 @@ class StubGenerator: public StubCodeGenerator {
   //   x10       - input length
   //
   address generate_counterMode_AESCrypt() {
-    assert(UseAESCTRIntrinsics, "Must be");
-    assert(UseZvkn, "need AES instructions (Zvkned extension) support");
-    assert(UseZbb, "need basic bit manipulation (Zbb extension) support");
+    assert(UseAESCTRIntrinsics, "need AES instructions (Zvkned extension) and Zbb extension support");
 
     __ align(CodeEntryAlignment);
     StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
@@ -3001,6 +2997,63 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
+  /**
+   *  Arguments:
+   *
+   *  Input:
+   *  c_rarg0   - current state address
+   *  c_rarg1   - H key address
+   *  c_rarg2   - data address
+   *  c_rarg3   - number of blocks
+   *
+   *  Output:
+   *  Updated state at c_rarg0
+   */
+  address generate_ghash_processBlocks() {
+    assert(UseGHASHIntrinsics, "need GHASH instructions (Zvkg extension) and Zvbb support");
+
+    __ align(CodeEntryAlignment);
+    StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+    StubCodeMark mark(this, stub_id);
+
+    address start = __ pc();
+    __ enter();
+
+    Register state   = c_rarg0;
+    Register subkeyH = c_rarg1;
+    Register data    = c_rarg2;
+    Register blocks  = c_rarg3;
+
+    VectorRegister partial_hash = v1;
+    VectorRegister hash_subkey  = v2;
+    VectorRegister cipher_text  = v3;
+
+    const unsigned int BLOCK_SIZE = 16;
+
+    __ vsetivli(x0, 2, Assembler::e64, Assembler::m1);
+    __ vle64_v(hash_subkey, subkeyH);
+    __ vrev8_v(hash_subkey, hash_subkey);
+    __ vle64_v(partial_hash, state);
+    __ vrev8_v(partial_hash, partial_hash);
+
+    __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
+    Label L_ghash_loop;
+    __ bind(L_ghash_loop);
+      __ vle32_v(cipher_text, data);
+      __ addi(data, data, BLOCK_SIZE);
+      __ vghsh_vv(partial_hash, hash_subkey, cipher_text);
+      __ subi(blocks, blocks, 1);
+      __ bnez(blocks, L_ghash_loop);
+
+    __ vsetivli(x0, 2, Assembler::e64, Assembler::m1);
+    __ vrev8_v(partial_hash, partial_hash);
+    __ vse64_v(partial_hash, state);
+    __ leave();
+    __ ret();
+
+    return start;
+  }
+
   // code for comparing 8 characters of strings with Latin1 and Utf16 encoding
   void compare_string_8_x_LU(Register tmpL, Register tmpU,
                              Register strL, Register strU, Label& DIFF) {
@@ -7227,6 +7280,10 @@ static const int64_t right_3_bits = right_n_bits(3);
       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt();
     }
 
+    if (UseGHASHIntrinsics) {
+      StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
+    }
+
     if (UsePoly1305Intrinsics) {
       StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
     }

src/hotspot/cpu/riscv/vm_version_riscv.cpp

@@ -457,6 +457,22 @@ void VM_Version::c2_initialize() {
       FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
     }
   }
+
+  if (UseZvkg) {
+    if (FLAG_IS_DEFAULT(UseGHASHIntrinsics) && UseZvbb) {
+      FLAG_SET_DEFAULT(UseGHASHIntrinsics, true);
+    }
+
+    if (UseGHASHIntrinsics && !UseZvbb) {
+      warning("Cannot enable UseGHASHIntrinsics on cpu without UseZvbb support");
+      FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
+    }
+  } else {
+    if (UseGHASHIntrinsics) {
+      warning("Cannot enable UseGHASHIntrinsics on cpu without UseZvkg support");
+      FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
+    }
+  }
 }
 
 #endif // COMPILER2

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -289,6 +289,8 @@ class VM_Version : public Abstract_VM_Version {
   decl(Zvfh        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvfh, &ext_v, &ext_Zfh, nullptr)) \
   /* Shorthand for Zvkned + Zvknhb + Zvkb + Zvkt */                                                    \
   decl(Zvkn        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvkn, &ext_v, nullptr))           \
+  /* Zvkg crypto extension for ghash and gcm */                                                        \
+  decl(Zvkg        ,  RV_NO_FLAG_BIT,  true ,  UPDATE_DEFAULT_DEP(UseZvkg, &ext_v, nullptr))           \
 
   #define DECLARE_RV_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF)                             \
   struct ext_##PRETTY##RVExtFeatureValue : public RVExtFeatureValue {                           \

src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp

@@ -256,6 +256,9 @@ void RiscvHwprobe::add_features_from_query_result() {
       is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKT)) {
     VM_Version::ext_Zvkn.enable_feature();
   }
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKG)) {
+    VM_Version::ext_Zvkg.enable_feature();
+  }
 #endif
 
   // ====== non-extensions ======