diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index ceedb4f1063..ac2acfba923 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -3306,7 +3306,7 @@ void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement void MacroAssembler::reinit_heapbase() { if (UseCompressedOops) { - if (Universe::is_fully_initialized()) { + if (Universe::is_fully_initialized() && !AOTCodeCache::is_on_for_dump()) { mov(rheapbase, CompressedOops::base()); } else { lea(rheapbase, ExternalAddress(CompressedOops::base_addr())); diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp index d45f9865bd2..2e570bee709 100644 --- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp @@ -292,7 +292,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() { assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned"); const char* name = OptoRuntime::stub_name(StubId::c2_exception_id); - CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, (uint)BlobId::c2_exception_id, name); + CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, BlobId::c2_exception_id); if (blob != nullptr) { return blob->as_exception_blob(); } diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp index 695534604b8..95b59bd8590 100644 --- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp @@ -84,8 +84,7 @@ do_stub(compiler, count_positives) \ do_arch_entry(aarch64, compiler, count_positives, count_positives, \ count_positives) \ - do_stub(compiler, count_positives_long) \ - do_arch_entry(aarch64, compiler, count_positives_long, \ + do_arch_entry(aarch64, compiler, count_positives, \ count_positives_long, count_positives_long) \ do_stub(compiler, compare_long_string_LL) \ do_arch_entry(aarch64, compiler, compare_long_string_LL, \ @@ -108,8 +107,9 @@ do_stub(compiler, string_indexof_linear_ul) \ do_arch_entry(aarch64, compiler, string_indexof_linear_ul, \ string_indexof_linear_ul, string_indexof_linear_ul) \ - /* this uses the entry for ghash_processBlocks */ \ - do_stub(compiler, ghash_processBlocks_wide) \ + do_stub(compiler, ghash_processBlocks_small) \ + do_arch_entry(aarch64, compiler, ghash_processBlocks_small, \ + ghash_processBlocks_small, ghash_processBlocks_small) \ #define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \ @@ -142,6 +142,47 @@ /* stub only -- entries are not stored in StubRoutines::aarch64 */ \ /* n.b. these are not the same as the generic atomic stubs */ \ do_stub(final, atomic_entry_points) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_impl, atomic_fetch_add_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_impl, atomic_fetch_add_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_relaxed_impl, \ + atomic_fetch_add_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_relaxed_impl, \ + atomic_fetch_add_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_4_impl, atomic_xchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_8_impl, atomic_xchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_impl, atomic_cmpxchg_1_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_impl, atomic_cmpxchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_impl, atomic_cmpxchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_relaxed_impl, \ + atomic_cmpxchg_1_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_relaxed_impl, \ + atomic_cmpxchg_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_relaxed_impl, \ + atomic_cmpxchg_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_release_impl, \ + atomic_cmpxchg_4_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_release_impl, \ + atomic_cmpxchg_8_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_seq_cst_impl, \ + atomic_cmpxchg_4_seq_cst_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_seq_cst_impl, \ + atomic_cmpxchg_8_seq_cst_impl) \ #endif // CPU_AARCH64_STUBDECLARATIONS_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8dbc5dbac03..78ba9881184 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -79,6 +79,166 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +// Constant data definitions + +static const uint32_t _sha256_round_consts[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +static const uint64_t _sha512_round_consts[80] = { + 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, + 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, + 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, + 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, + 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, + 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, + 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, + 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, + 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, + 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, + 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, + 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, + 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, + 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, + 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, + 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, + 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, + 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, + 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, + 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, + 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, + 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, + 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, + 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, + 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, + 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, + 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L +}; + +static const uint64_t _sha3_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const uint64_t _double_keccak_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const char _encodeBlock_toBase64[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' +}; + +static const char _encodeBlock_toBase64URL[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' +}; + +// Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, +// except the trailing character '=' is also treated illegal value in this intrinsic. That +// is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. +static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +// A legal value of base64 code is in range [0, 127]. We need two lookups +// with tbl/tbx and combine them to get the decode data. The 1st table vector +// lookup use tbl, out of range indices are set to 0 in destination. The 2nd +// table vector lookup use tbx, out of range indices are unchanged in +// destination. Input [64..126] is mapped to index [65, 127] in second lookup. +// The value of index 64 is set to 0, so that we know that we already get the +// decoded data with the 1st lookup. +static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + + // Stub Code definitions class StubGenerator: public StubCodeGenerator { @@ -197,12 +357,52 @@ class StubGenerator: public StubCodeGenerator { retaddr_off = 1, }; + // Helper used to restore saved ranges and handler addresses. + // Expects entries to contain 3 * count addresses beginning at + // offset begin which identify start of range, end of range and + // address of handler pc. end of range may be nullptr in which case + // it defaults to stub_end. hanlder pc may be nullptr in which case + // it defaults to default_handler. + + void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { + for (int i = 0; i < count; i++) { + int offset = begin + 3 * i; + address start = entries.at(offset); + address end = entries.at(offset + 1); + if (end == nullptr) { + end = stub_end; + } + address handler = entries.at(offset + 2); + if (handler == nullptr) { + handler = UnsafeMemoryAccess::common_exit_stub_pc(); + } + UnsafeMemoryAccess::add_to_table(start, end, handler); + } + } + + void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries) { + UnsafeMemoryAccess::collect_entries(start, end, entries); + } + address generate_call_stub(address& return_address) { assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, "adjust this code"); StubId stub_id = StubId::stubgen_call_stub_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // this stub has an extra return entry which we need to retrieve + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, "expected 1 extra entry"); + return_address = entries.at(0); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -323,6 +523,7 @@ class StubGenerator: public StubCodeGenerator { // save current address for use by exception handling code return_address = __ pc(); + entries.append(return_address); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) @@ -406,6 +607,9 @@ class StubGenerator: public StubCodeGenerator { __ strd(j_farg0, Address(j_rarg2, 0)); __ br(Assembler::AL, exit); + // record the stub entry and end plus the auxiliary entry + store_archive_data(stub_id, start, __ pc(), &entries); + return start; } @@ -423,6 +627,15 @@ class StubGenerator: public StubCodeGenerator { address generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -460,6 +673,9 @@ class StubGenerator: public StubCodeGenerator { "_call_stub_return_address must have been generated before"); __ b(StubRoutines::_call_stub_return_address); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -479,6 +695,14 @@ class StubGenerator: public StubCodeGenerator { address generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -551,6 +775,9 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ br(r19); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -569,6 +796,15 @@ class StubGenerator: public StubCodeGenerator { // [tos + 5]: saved rscratch1 address generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -613,12 +849,24 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ hlt(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Generate indices for iota vector. address generate_iota_indices(StubId stub_id) { __ align(CodeEntryAlignment); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); // B @@ -639,6 +887,10 @@ class StubGenerator: public StubCodeGenerator { // D - FP __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -656,14 +908,23 @@ class StubGenerator: public StubCodeGenerator { // r11 < MacroAssembler::zero_words_block_size. address generate_zero_blocks() { + StubId stub_id = StubId::stubgen_zero_blocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + StubCodeMark mark(this, stub_id); Label done; Label base_aligned; Register base = r10, cnt = r11; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_zero_blocks_id; - StubCodeMark mark(this, stub_id); address start = __ pc(); if (UseBlockZeroing) { @@ -707,6 +968,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -803,6 +1067,15 @@ class StubGenerator: public StubCodeGenerator { // s and d are adjusted to point to the remaining words to copy // address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + BasicType type; copy_direction direction; @@ -1154,6 +1427,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1508,8 +1784,8 @@ class StubGenerator: public StubCodeGenerator { // stub_id - is used to name the stub and identify all details of // how to perform the copy. // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // Inputs: // c_rarg0 - source array address @@ -1525,8 +1801,6 @@ class StubGenerator: public StubCodeGenerator { // copy method // address generate_disjoint_copy(StubId stub_id, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_reg = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1607,17 +1881,48 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); break; } + // all stubs provide a 2nd entry which omits the frame push for + // use when bailing out from a conjoint copy. However we may also + // need some extra addressses for memory access protection. + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry"); + + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries, extras_ptr); + assert(entries.length() == entry_count - 1, + "unexpected entries count %d", entries.length()); + *nopush_entry = entries.at(0); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_reg = RegSet::of(s, d, count); __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); - if (nopush_entry != nullptr) { - *nopush_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); - } + *nopush_entry = __ pc(); + entries.append(*nopush_entry); + + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Post-Push Entry:"); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; if (dest_uninitialized) { @@ -1636,8 +1941,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size); } @@ -1652,6 +1956,20 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count + , "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, &entries, extras_ptr); + return start; } @@ -1663,8 +1981,8 @@ class StubGenerator: public StubCodeGenerator { // corresponding disjoint copy routine which can be // jumped to if the ranges do not actually overlap // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // // Inputs: @@ -1681,8 +1999,6 @@ class StubGenerator: public StubCodeGenerator { // used by some other conjoint copy method // address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_regs = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1762,15 +2078,50 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + // We need to protect memory accesses in certain cases + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*entries_ptr = (nopush_entry != nullptr ? &entries : nullptr); + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_regs = RegSet::of(s, d, count); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); + BLOCK_COMMENT("Post-Push Entry:"); } // use fwd copy when (d-s) above_equal (count*size) @@ -1798,8 +2149,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size); } if (is_oop) { @@ -1811,6 +2161,23 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus any no_push entry and/or + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -1864,6 +2231,30 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); } + // the normal stub provides a 2nd entry which omits the frame push + // for use when bailing out from a disjoint copy + // We need to protect memory accesses in certain cases + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + GrowableArray
entries; + GrowableArray
*entries_ptr = (expected_entry_count == 1 ? nullptr : &entries); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr); + assert(entries.length() + 1 == expected_entry_count, + "expected entry count %d does not match return entry count %d for stub %s", + expected_entry_count, entries.length() + 1, StubInfo::name(stub_id)); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + return start; + } + Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; // Input registers (after setup_arg_regs) @@ -1913,6 +2304,7 @@ class StubGenerator: public StubCodeGenerator { // Caller of this entry point must set up the argument registers. if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); BLOCK_COMMENT("Entry:"); } @@ -2010,6 +2402,8 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc() , entries_ptr); return start; } @@ -2072,6 +2466,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address long_copy_entry) { StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_long_aligned, L_int_aligned, L_short_aligned; Register s = c_rarg0, d = c_rarg1, count = c_rarg2; @@ -2104,6 +2509,9 @@ class StubGenerator: public StubCodeGenerator { __ lsr(count, count, LogBytesPerLong); // size => long_count __ b(RuntimeAddress(long_copy_entry)); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2125,6 +2533,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { StubId stub_id = StubId::stubgen_generic_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_failed, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; @@ -2383,6 +2802,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2427,6 +2849,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2563,14 +2993,31 @@ class StubGenerator: public StubCodeGenerator { __ bind(L_exit2); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_unsafecopy_common_error_exit() { + StubId stub_id = StubId::stubgen_unsafecopy_common_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } address start_pc = __ pc(); __ leave(); __ mov(r0, 0); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start_pc, __ pc()); + return start_pc; } @@ -2589,13 +3036,30 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - byte value // address generate_unsafe_setmemory() { + StubId stub_id = StubId::stubgen_unsafe_setmemory_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + // we expect one set of extra unsafememory access handler entries + GrowableArray
extras; + int extra_count = 3; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, nullptr, &extras); + assert(extras.length() == extra_count, + "unexpected extra entry count %d", extras.length()); + register_unsafe_access_handlers(extras, 0, 1, end); + return start; + } + __ align(CodeEntryAlignment); - StubCodeMark mark(this, StubId::stubgen_unsafe_setmemory_id); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register dest = c_rarg0, count = c_rarg1, value = c_rarg2; Label tail; + { UnsafeMemoryAccessMark umam(this, true, false); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2679,6 +3143,17 @@ class StubGenerator: public StubCodeGenerator { __ bind(finished); __ leave(); __ ret(lr); + // have to exit the block and destroy the UnsafeMemoryAccessMark + // in order to retrieve the handler end address + } + + // install saved handler addresses in extras + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + // record the stub entry and end plus the extras + store_archive_data(stub_id, start, end, nullptr, &extras); return start; } @@ -2686,9 +3161,18 @@ class StubGenerator: public StubCodeGenerator { address generate_data_cache_writeback() { const Register line = c_rarg0; // address of line to write back + StubId stub_id = StubId::stubgen_data_cache_writeback_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2697,15 +3181,27 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_data_cache_writeback_sync() { + StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register is_pre = c_rarg0; // pre or post sync __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; StubCodeMark mark(this, stub_id); // pre wbsync is a no-op @@ -2720,6 +3216,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2882,8 +3381,18 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - K (key) in little endian int array // address generate_aescrypt_encryptBlock() { - __ align(CodeEntryAlignment); + assert(UseAES, "need AES cryptographic extension support"); StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); const Register from = c_rarg0; // source array address @@ -2904,6 +3413,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2916,8 +3428,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_aescrypt_decryptBlock() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; @@ -2938,6 +3459,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2955,8 +3479,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_encryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3043,6 +3576,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3060,8 +3596,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_decryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3152,6 +3697,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3188,6 +3736,16 @@ class StubGenerator: public StubCodeGenerator { // r0 - input length // address generate_counterMode_AESCrypt() { + StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register in = c_rarg0; const Register out = c_rarg1; const Register key = c_rarg2; @@ -3248,7 +3806,6 @@ class StubGenerator: public StubCodeGenerator { // Wide bulk encryption of whole blocks. __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; StubCodeMark mark(this, stub_id); const address start = __ pc(); __ enter(); @@ -3435,6 +3992,9 @@ class StubGenerator: public StubCodeGenerator { __ strw(used, Address(used_ptr)); __ b(large_block_return); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3451,9 +4011,17 @@ class StubGenerator: public StubCodeGenerator { // return - number of processed bytes address generate_galoisCounterMode_AESCrypt() { Label ghash_polynomial; // local data generated after code - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -3567,6 +4135,9 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3685,6 +4256,15 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3815,6 +4395,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3838,6 +4421,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3919,6 +4510,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int32(0x8f1bbcdc); __ emit_int32(0xca62c1d6); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3943,25 +4537,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint32_t round_consts[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3987,7 +4570,7 @@ class StubGenerator: public StubCodeGenerator { // t1 == v7 // load 16 keys to v16..v31 - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts)); __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64)); __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64)); __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64)); @@ -4048,6 +4631,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4099,36 +4685,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[80] = { - 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, - 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, - 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, - 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, - 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, - 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, - 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, - 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, - 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, - 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, - 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, - 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, - 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, - 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, - 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, - 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, - 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, - 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, - 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, - 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, - 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, - 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, - 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, - 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, - 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, - 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, - 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4151,7 +4715,7 @@ class StubGenerator: public StubCodeGenerator { __ ld1(v8, v9, v10, v11, __ T2D, state); // load first 4 round constants - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts)); __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64)); __ BIND(sha512_loop); @@ -4236,6 +4800,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4349,17 +4916,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4396,7 +4960,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts)); // load input __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); @@ -4488,6 +5052,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4495,21 +5062,21 @@ class StubGenerator: public StubCodeGenerator { // c_rarg0 - long[] state0 // c_rarg1 - long[] state1 address generate_double_keccak() { - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + StubId stub_id = StubId::stubgen_double_keccak_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } // Implements the double_keccak() method of the // sun.secyrity.provider.SHA3Parallel class __ align(CodeEntryAlignment); - StubCodeMark mark(this, "StubRoutines", "double_keccak"); + + StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4546,7 +5113,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts)); __ BIND(rounds24_loop); __ subw(rscratch2, rscratch2, 1); @@ -4578,6 +5145,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4611,9 +5181,18 @@ class StubGenerator: public StubCodeGenerator { // vectors write their first lane back to the keystream buffer, followed // by the second lane from all vectors and so on. address generate_chacha20Block_blockpar() { + StubId stub_id = StubId::stubgen_chacha20Block_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_twoRounds, L_cc20_const; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_chacha20Block_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4770,6 +5349,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0605040702010003UL); __ emit_int64(0x0E0D0C0F0A09080BUL); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5258,9 +5840,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberNtt() { + StubId stub_id = StubId::stubgen_kyberNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5486,6 +6076,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5496,9 +6089,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberInverseNtt() { + StubId stub_id = StubId::stubgen_kyberInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5770,6 +6371,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5783,9 +6387,17 @@ class StubGenerator: public StubCodeGenerator { // nttb (short[256]) = c_rarg2 // zetas (short[128]) = c_rarg3 address generate_kyberNttMult() { + StubId stub_id = StubId::stubgen_kyberNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNttMult_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5889,6 +6501,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5900,9 +6515,17 @@ class StubGenerator: public StubCodeGenerator { // a (short[256]) = c_rarg1 // b (short[256]) = c_rarg2 address generate_kyberAddPoly_2() { + StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5973,6 +6596,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5985,9 +6611,17 @@ class StubGenerator: public StubCodeGenerator { // b (short[256]) = c_rarg2 // c (short[256]) = c_rarg3 address generate_kyberAddPoly_3() { + StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6072,6 +6706,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6088,10 +6725,19 @@ class StubGenerator: public StubCodeGenerator { // parsed (short[112 or 256]) = c_rarg2 // parsedLength (112 or 256) = c_rarg3 address generate_kyber12To16() { + StubId stub_id = StubId::stubgen_kyber12To16_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_F00, L_loop, L_end; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyber12To16_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6288,6 +6934,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0f000f000f000f00); __ emit_int64(0x0f000f000f000f00); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6297,9 +6946,17 @@ class StubGenerator: public StubCodeGenerator { // // coeffs (short[256]) = c_rarg0 address generate_kyberBarrettReduce() { + StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6381,6 +7038,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6544,9 +7204,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6659,6 +7327,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6751,9 +7422,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostInverseNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6851,6 +7530,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6864,9 +7546,17 @@ class StubGenerator: public StubCodeGenerator { // poly1 (int[256]) = c_rarg1 // poly2 (int[256]) = c_rarg2 address generate_dilithiumNttMult() { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6917,6 +7607,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6928,9 +7621,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // constant (int) = c_rarg1 address generate_dilithiumMontMulByConstant() { + StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6978,6 +7679,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6992,9 +7696,17 @@ class StubGenerator: public StubCodeGenerator { // twoGamma2 (int) = c_rarg3 // multiplier (int) = c_rarg4 address generate_dilithiumDecomposePoly() { + StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; StubCodeMark mark(this, stub_id); address start = __ pc(); Label L_loop; @@ -7136,6 +7848,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7275,17 +7990,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -7441,7 +8153,7 @@ class StubGenerator: public StubCodeGenerator { __ fmovs(v1, 1.0); // exact representation __ str(buf, Address(sp, 16)); - __ lea(tmp3, ExternalAddress((address) round_consts)); + __ lea(tmp3, ExternalAddress((address) _sha3_round_consts)); __ BIND(loop_body); keccak_round_gpr(can_use_fp, can_use_r18, tmp3, @@ -7496,6 +8208,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7512,9 +8227,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32() { assert(UseCRC32Intrinsics, "what are we doing here?"); + StubId stub_id = StubId::stubgen_updateBytesCRC32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_updateBytesCRC32_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7537,6 +8260,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7554,9 +8280,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32C() { assert(UseCRC32CIntrinsics, "what are we doing here?"); + StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7579,6 +8313,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7594,8 +8331,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg0 - int adler result */ address generate_updateBytesAdler32() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesAdler32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7765,6 +8511,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7816,8 +8565,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg4 - z address */ address generate_multiplyToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_multiplyToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7842,6 +8600,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7849,8 +8610,17 @@ class StubGenerator: public StubCodeGenerator { // squareToLen algorithm for sizes 1..127 described in java code works // faster than multiply_to_len on some CPUs and slower on others, but // multiply_to_len shows a bit better overall results - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_squareToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7879,12 +8649,25 @@ class StubGenerator: public StubCodeGenerator { __ pop(spilled_regs, sp); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_mulAdd() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_mulAdd_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7901,6 +8684,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7914,8 +8700,17 @@ class StubGenerator: public StubCodeGenerator { // c_rarg4 - numIter // address generate_bigIntegerRightShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8024,6 +8819,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8039,6 +8837,15 @@ class StubGenerator: public StubCodeGenerator { address generate_bigIntegerLeftShift() { __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8135,10 +8942,28 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_count_positives(address &count_positives_long) { + StubId stub_id = StubId::stubgen_count_positives_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + // We have an extra entry for count_positives_long. + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, + "unexpected extra entry count %d", entries.length()); + count_positives_long = entries.at(0); + return start; + } + const u1 large_loop_size = 64; const uint64_t UPPER_BIT_MASK=0x8080808080808080; int dcache_line = VM_Version::dcache_line_size(); @@ -8147,7 +8972,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_count_positives_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8190,6 +9014,7 @@ class StubGenerator: public StubCodeGenerator { const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6; count_positives_long = __ pc(); // 2nd entry point + entries.append(count_positives_long); __ enter(); @@ -8304,6 +9129,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(result, result, len); __ ret(lr); + // record the stub entry and end plus the extra entry + store_archive_data(stub_id, entry, __ pc(), &entries); + return entry; } @@ -8394,6 +9222,16 @@ class StubGenerator: public StubCodeGenerator { // r3-r5 are reserved temporary registers // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2 address generate_large_array_equals() { + StubId stub_id = StubId::stubgen_large_array_equals_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, tmp7 = r12, tmp8 = r13; @@ -8409,7 +9247,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_large_array_equals_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8484,6 +9321,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(NOT_EQUAL_NO_POP); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8492,6 +9333,36 @@ class StubGenerator: public StubCodeGenerator { // cnt = r2 - elements count // Clobbers: v0-v13, rscratch1, rscratch2 address generate_large_arrays_hashcode(BasicType eltype) { + StubId stub_id; + switch (eltype) { + case T_BOOLEAN: + stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; + break; + case T_BYTE: + stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; + break; + case T_CHAR: + stub_id = StubId::stubgen_large_arrays_hashcode_char_id; + break; + case T_SHORT: + stub_id = StubId::stubgen_large_arrays_hashcode_short_id; + break; + case T_INT: + stub_id = StubId::stubgen_large_arrays_hashcode_int_id; + break; + default: + stub_id = StubId::NO_STUBID; + ShouldNotReachHere(); + }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register result = r0, ary = r1, cnt = r2; const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0; const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7; @@ -8535,28 +9406,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id; - switch (eltype) { - case T_BOOLEAN: - stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; - break; - case T_BYTE: - stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; - break; - case T_CHAR: - stub_id = StubId::stubgen_large_arrays_hashcode_char_id; - break; - case T_SHORT: - stub_id = StubId::stubgen_large_arrays_hashcode_short_id; - break; - case T_INT: - stub_id = StubId::stubgen_large_arrays_hashcode_int_id; - break; - default: - stub_id = StubId::NO_STUBID; - ShouldNotReachHere(); - }; - StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8791,12 +9640,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_dsin_dcos(bool isCos) { - __ align(CodeEntryAlignment); StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw, @@ -8804,6 +9665,10 @@ class StubGenerator: public StubCodeGenerator { (address)StubRoutines::aarch64::_pio2, (address)StubRoutines::aarch64::_dsin_coef, (address)StubRoutines::aarch64::_dcos_coef); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8847,8 +9712,17 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_different_encoding(bool isLU) { - __ align(CodeEntryAlignment); StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2, @@ -8950,20 +9824,37 @@ class StubGenerator: public StubCodeGenerator { __ subw(result, tmp1, rscratch1); __ bind(DONE); __ ret(lr); - return entry; + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + + return entry; } // r0 = input (float16) // v0 = result (float) // v1 = temporary float register address generate_float16ToFloat() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_hf2f_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt16_to_flt(v0, r0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8971,19 +9862,39 @@ class StubGenerator: public StubCodeGenerator { // r0 = result (float16) // v1 = temporary float register address generate_floatToFloat16() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_f2hf_id; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt_to_flt16(r0, v0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_method_entry_barrier() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_method_entry_barrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label deoptimize_label; @@ -9037,6 +9948,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(sp, rscratch1); __ br(rscratch2); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -9048,8 +9962,18 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_same_encoding(bool isLL) { - __ align(CodeEntryAlignment); StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, @@ -9157,6 +10081,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(LENGTH_DIFF); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9188,8 +10116,17 @@ class StubGenerator: public StubCodeGenerator { case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break; default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, tmp1 = r10, tmp2 = r11; @@ -9224,8 +10161,6 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); \ } - StubCodeMark mark(this, stub_id); - __ mov(idx, 0); __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt); @@ -9269,6 +10204,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(DONE); __ ret(lr); #undef LOAD_PAIR + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9330,6 +10269,15 @@ class StubGenerator: public StubCodeGenerator { stub_id = StubId::stubgen_string_indexof_linear_uu_id; } } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9598,6 +10546,10 @@ class StubGenerator: public StubCodeGenerator { __ BIND(DONE); __ pop(spilled_regs, sp); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9628,8 +10580,17 @@ class StubGenerator: public StubCodeGenerator { // v1 = loaded 8 bytes // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6 address generate_large_byte_array_inflate() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_large_byte_array_inflate_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE; @@ -9668,6 +10629,10 @@ class StubGenerator: public StubCodeGenerator { __ br(__ GE, LOOP); __ bind(DONE); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9683,7 +10648,7 @@ class StubGenerator: public StubCodeGenerator { * Output: * Updated state at c_rarg0 */ - address generate_ghash_processBlocks() { + address generate_ghash_processBlocks_small() { // Bafflingly, GCM uses little-endian for the byte order, but // big-endian for the bit order. For example, the polynomial 1 is // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00. @@ -9695,7 +10660,16 @@ class StubGenerator: public StubCodeGenerator { // that) and keep the data in little-endian bit order through the // calculation, bit-reversing the inputs and outputs. - StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); Label polynomial; // local data generated at end of stub __ align(CodeEntryAlignment); @@ -9759,16 +10733,26 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } - address generate_ghash_processBlocks_wide() { - address small = generate_ghash_processBlocks(); + address generate_ghash_processBlocks(address small) { + StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } - StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id; - StubCodeMark mark(this, stub_id); Label polynomial; // local data generated after stub __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register state = c_rarg0; @@ -9811,8 +10795,10 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); - return start; + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_base64_encode_simdround(Register src, Register dst, @@ -9863,24 +10849,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_base64_encodeBlock() { - static const char toBase64[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' - }; - - static const char toBase64URL[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' - }; + StubId stub_id = StubId::stubgen_base64_encodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_encodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -9902,9 +10881,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); // load the codec base address - __ lea(codec, ExternalAddress((address) toBase64)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64)); __ cbz(isURL, ProcessData); - __ lea(codec, ExternalAddress((address) toBase64URL)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL)); __ BIND(ProcessData); @@ -9957,6 +10936,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10078,78 +11060,17 @@ class StubGenerator: public StubCodeGenerator { // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section // titled "Base64 decoding". - // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, - // except the trailing character '=' is also treated illegal value in this intrinsic. That - // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. - static const uint8_t fromBase64ForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - // A legal value of base64 code is in range [0, 127]. We need two lookups - // with tbl/tbx and combine them to get the decode data. The 1st table vector - // lookup use tbl, out of range indices are set to 0 in destination. The 2nd - // table vector lookup use tbx, out of range indices are unchanged in - // destination. Input [64..126] is mapped to index [65, 127] in second lookup. - // The value of index 64 is set to 0, so that we know that we already get the - // decoded data with the 1st lookup. - static const uint8_t fromBase64ForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; + StubId stub_id = StubId::stubgen_base64_decodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_decodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10178,9 +11099,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); __ bfm(length, zr, 0, 1); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD)); __ cbz(isURL, ProcessData); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD)); __ BIND(ProcessData); __ mov(rscratch1, length); @@ -10225,9 +11146,9 @@ class StubGenerator: public StubCodeGenerator { __ cbzw(rscratch1, Exit); __ sub(length, length, 80); - __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD)); __ cbz(isURL, SIMDEnter); - __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD)); __ BIND(SIMDEnter); __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64)); @@ -10260,24 +11181,55 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Support for spin waits. address generate_spin_wait() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_spin_wait_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ spin_wait(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_lookup_secondary_supers_table_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, + "unexpected extra entry count %d", entries.length()); + StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; + for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { + StubRoutines::_lookup_secondary_supers_table_stubs[0] = entries.at(slot - 1); + } + return; + } + StubCodeMark mark(this, stub_id); const Register @@ -10291,8 +11243,15 @@ class StubGenerator: public StubCodeGenerator { const FloatRegister vtemp = v0; + address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { - StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); + address next_entry = __ pc(); + StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; + if (slot == 0) { + start = next_entry; + } else { + entries.append(next_entry); + } Label L_success; __ enter(); __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, @@ -10302,11 +11261,22 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); } + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); } // Slow path implementation for UseSecondarySupersTable. address generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10321,6 +11291,9 @@ class StubGenerator: public StubCodeGenerator { __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10460,14 +11433,45 @@ class StubGenerator: public StubCodeGenerator { if (! UseLSE) { return; } - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_atomic_entry_points_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start; + int idx = 0; + aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + assert(idx == entries.length(), "sanity!"); + return; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address first_entry = __ pc(); - + address end; + { // ADD, memory_order_conservative AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl); gen_ldadd_entry(Assembler::word, memory_order_conservative); + AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl); gen_ldadd_entry(Assembler::xword, memory_order_conservative); @@ -10475,6 +11479,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_fetch_add_4_relaxed (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl); gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_fetch_add_8_relaxed (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl); gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10482,14 +11487,17 @@ class StubGenerator: public StubCodeGenerator { // XCHG, memory_order_conservative AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl); gen_swpal_entry(Assembler::word); - AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl); + + AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl); gen_swpal_entry(Assembler::xword); // CAS, memory_order_conservative AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl); gen_cas_entry(MacroAssembler::byte, memory_order_conservative); + AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl); gen_cas_entry(MacroAssembler::word, memory_order_conservative); + AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl); gen_cas_entry(MacroAssembler::xword, memory_order_conservative); @@ -10497,9 +11505,11 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_1_relaxed (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl); gen_cas_entry(MacroAssembler::byte, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_4_relaxed (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl); gen_cas_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_8_relaxed (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl); gen_cas_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10507,6 +11517,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_release (_masm, &aarch64_atomic_cmpxchg_4_release_impl); gen_cas_entry(MacroAssembler::word, memory_order_release); + AtomicStubMark mark_cmpxchg_8_release (_masm, &aarch64_atomic_cmpxchg_8_release_impl); gen_cas_entry(MacroAssembler::xword, memory_order_release); @@ -10514,11 +11525,41 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_seq_cst (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl); gen_cas_entry(MacroAssembler::word, memory_order_seq_cst); + AtomicStubMark mark_cmpxchg_8_seq_cst (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl); gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst); - ICache::invalidate_range(first_entry, __ pc() - first_entry); + end = __ pc(); + + ICache::invalidate_range(first_entry, end - first_entry); + // exit block to force update of AtomicStubMark targets + } + + assert(first_entry == (address)aarch64_atomic_fetch_add_4_impl, + "atomic stub should be at start of buffer"); + // record the stub start and end plus all the entries saved by the + // AtomicStubMark destructor + entries.append((address)aarch64_atomic_fetch_add_8_impl); + entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl); + entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl); + entries.append((address)aarch64_atomic_xchg_4_impl); + entries.append((address)aarch64_atomic_xchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl); + + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + + store_archive_data(stub_id, first_entry, end, &entries); } #endif // LINUX @@ -10622,9 +11663,22 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_thaw_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_top); + + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10633,11 +11687,23 @@ class StubGenerator: public StubCodeGenerator { // TODO: will probably need multiple return barriers depending on return type StubId stub_id = StubId::stubgen_cont_returnBarrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10645,17 +11711,38 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier_exception); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_preempt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10682,6 +11769,9 @@ class StubGenerator: public StubCodeGenerator { __ ldr(rscratch1, Address(rscratch1)); __ br(rscratch1); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10737,8 +11827,17 @@ class StubGenerator: public StubCodeGenerator { // computation. address generate_poly1305_processBlocks() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_poly1305_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); Label here; @@ -10848,12 +11947,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } // exception handler for upcall stubs address generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10864,6 +11975,9 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ should_not_reach_here(); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10872,6 +11986,15 @@ class StubGenerator: public StubCodeGenerator { // rmethod = result address generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10887,6 +12010,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -11915,7 +13041,8 @@ class StubGenerator: public StubCodeGenerator { } if (UseGHASHIntrinsics) { // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(); - StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide(); + StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small(); + StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small); } if (UseAESIntrinsics && UseGHASHIntrinsics) { StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt(); @@ -11962,7 +13089,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -11984,10 +13111,36 @@ class StubGenerator: public StubCodeGenerator { break; }; } + + static void init_AOTAddressTable(GrowableArray
& external_addresses) { + // external data defined in this file +#define ADD(addr) external_addresses.append((address)addr); + ADD(_sha256_round_consts); + ADD(_sha512_round_consts); + ADD(_sha3_round_consts); + ADD(_double_keccak_round_consts); + ADD(_encodeBlock_toBase64); + ADD(_encodeBlock_toBase64URL); + ADD(_decodeBlock_fromBase64ForNoSIMD); + ADD(_decodeBlock_fromBase64URLForNoSIMD); + ADD(_decodeBlock_fromBase64ForSIMD); + ADD(_decodeBlock_fromBase64URLForSIMD); +#undef ADD + } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +void StubGenerator_AOTAddressTable_init() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by aarch64 generator + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested aarch64 class + StubRoutines::aarch64::init_AOTAddressTable(external_addresses); + AOTCodeCache::publish_external_addresses(external_addresses); +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index 88993818b47..73f0797cb7a 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -413,3 +413,19 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = { 2.73370053816464559624e-44, // 0x36E3822280000000 2.16741683877804819444e-51, // 0x3569F31D00000000 }; + +#define ADD(addr) external_addresses.append((address)addr); + +void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& external_addresses) { + ADD(_kyberConsts); + ADD(_dilithiumConsts); + ADD(_crc_table); + ADD(_adler_table); + ADD(_npio2_hw); + ADD(_dsin_coef); + ADD(_dcos_coef); + ADD(_two_over_pi); + ADD(_pio2); +} + +#undef ADD diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp index c35371e1083..89c72a6a5ea 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp @@ -110,6 +110,8 @@ private: _completed = true; } + static void init_AOTAddressTable(GrowableArray
& external_addresses); + private: static uint16_t _kyberConsts[]; static uint32_t _dilithiumConsts[]; diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index a4c956ff5be..e1867c7c4ab 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -278,10 +278,6 @@ bool Runtime1::initialize(BufferBlob* blob) { if (!generate_blob_for(blob, id)) { return false; } - if (id == StubId::c1_forward_exception_id) { - // publish early c1 stubs at this point so later stubs can refer to them - AOTCodeCache::init_early_c1_table(); - } } // printing #ifndef PRODUCT diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 04776f4c16c..a72a7ba9d49 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -32,9 +32,12 @@ #include "classfile/javaAssertions.hpp" #include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/gcConfig.hpp" #include "logging/logStream.hpp" #include "memory/memoryReserver.hpp" +#include "prims/jvmtiThreadState.hpp" +#include "prims/upcallLinker.hpp" #include "runtime/deoptimization.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/globals_extension.hpp" @@ -153,10 +156,13 @@ static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) { } else if (kind == AOTCodeEntry::C1Blob) { assert(StubInfo::is_c1(static_cast(id)), "not a c1 blob id %d", id); return id; - } else { - // kind must be AOTCodeEntry::C2Blob + } else if (kind == AOTCodeEntry::C2Blob) { assert(StubInfo::is_c2(static_cast(id)), "not a c2 blob id %d", id); return id; + } else { + // kind must be AOTCodeEntry::StubGenBlob + assert(StubInfo::is_stubgen(static_cast(id)), "not a stubgen blob id %d", id); + return id; } } @@ -182,7 +188,7 @@ void AOTCodeCache::initialize() { } // Disable stubs caching until JDK-8357398 is fixed. - FLAG_SET_ERGO(AOTStubCaching, false); + // FLAG_SET_ERGO(AOTStubCaching, false); if (VerifyOops) { // Disable AOT stubs caching when VerifyOops flag is on. @@ -278,6 +284,19 @@ bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) { return true; } +// Called after continuations_init() when continuation stub callouts +// have been initialized +void AOTCodeCache::init3() { + if (opened_cache == nullptr) { + return; + } + // initialize external routines for continuations so we can save + // generated continuation blob that references them + AOTCodeAddressTable* table = opened_cache->_table; + assert(table != nullptr, "should be initialized already"); + table->init_extrs2(); +} + void AOTCodeCache::close() { if (is_on()) { delete _cache; // Free memory @@ -354,24 +373,59 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) : _table = new AOTCodeAddressTable(); } -void AOTCodeCache::init_early_stubs_table() { - AOTCodeAddressTable* table = addr_table(); - if (table != nullptr) { - table->init_early_stubs(); +void AOTCodeCache::add_stub_entries(StubId stub_id, address start, GrowableArray
*entries, int begin_idx) { + EntryId entry_id = StubInfo::entry_base(stub_id); + add_stub_entry(entry_id, start); + // skip past first entry + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + // now check for any more entries + int count = StubInfo::entry_count(stub_id) - 1; + assert(start != nullptr, "invalid start address for stub %s", StubInfo::name(stub_id)); + assert(entries == nullptr || begin_idx + count <= entries->length(), "sanity"); + // write any extra entries + for (int i = 0; i < count; i++) { + assert(entry_id != EntryId::NO_ENTRYID, "not enough entries for stub %s", StubInfo::name(stub_id)); + address a = entries->at(begin_idx + i); + add_stub_entry(entry_id, a); + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + } + assert(entry_id == EntryId::NO_ENTRYID, "too many entries for stub %s", StubInfo::name(stub_id)); +} + +void AOTCodeCache::add_stub_entry(EntryId entry_id, address a) { + if (a != nullptr) { + if (_table != nullptr) { + log_trace(aot, codecache, stubs)("Publishing stub entry %s at address " INTPTR_FORMAT, StubInfo::name(entry_id), p2i(a)); + return _table->add_stub_entry(entry_id, a); + } } } -void AOTCodeCache::init_shared_blobs_table() { +void AOTCodeCache::set_shared_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_shared_blobs(); + table->set_shared_stubs_complete(); } } -void AOTCodeCache::init_early_c1_table() { +void AOTCodeCache::set_c1_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_early_c1(); + table->set_c1_stubs_complete(); + } +} + +void AOTCodeCache::set_c2_stubs_complete() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->set_c2_stubs_complete(); + } +} + +void AOTCodeCache::set_stubgen_stubs_complete() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->set_stubgen_stubs_complete(); } } @@ -799,13 +853,42 @@ bool AOTCodeCache::finish_write() { //------------------Store/Load AOT code ---------------------- -bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + + // we only expect stub data and a code buffer for a multi stub blob + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + + assert((stub_data == nullptr) == (code_buffer == nullptr), + "stub data and code buffer must both be null or both non null"); + + // If this is a stub and the cache is on for either load or dump we + // need to insert the stub entries into the AOTCacheAddressTable so + // that relocs which refer to entries defined by this blob get + // translated correctly. + // + // Entry insertion needs to be be done up front before writing the + // blob because some blobs rely on internal daisy-chain references + // from one entry to another. + // + // Entry insertion also needs to be done even if the cache is open + // for use but not for dump. This may be needed when an archived + // blob omits some entries -- either because of a config change or a + // load failure -- with the result that the entries end up being + // generated. These generated entry addresses may be needed to + // resolve references from subsequently loaded blobs (for either + // stubs or nmethods). + + if (is_on() && AOTCodeEntry::is_blob(entry_kind)) { + publish_stub_addresses(blob, (BlobId)id, stub_data); + } + AOTCodeCache* cache = open_for_dump(); if (cache == nullptr) { return false; } - assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); - if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) { return false; } @@ -851,8 +934,44 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } CodeBlob::archive_blob(&blob, archive_buffer); - uint reloc_data_size = blob.relocation_size(); - n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size); + // For a relocatable code blob its relocations are linked from the + // blob. However, for a non-relocatable (stubgen) blob we only have + // transient relocations attached to the code buffer that are added + // in order to support AOT-load time patching. in either case, we + // need to explicitly save these relocs when storing the blob to the + // archive so we can then reload them and reattach them to either + // the blob or to a code buffer when we reload the blob into a + // production JVM. + // + // Either way we are then in a position to iterate over the relocs + // and AOT patch the ones that refer to code that may move between + // assembly and production time. We also need to save and restore + // AOT address table indexes for the target addresses of affected + // relocs. That happens below. + + int reloc_count; + address reloc_data; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + reloc_count = (cs->has_locs() ? cs->locs_count() : 0); + reloc_data = (reloc_count > 0 ? (address)cs->locs_start() : nullptr); + } else { + reloc_count = blob.relocation_size() / sizeof(relocInfo); + reloc_data = (address)blob.relocation_begin(); + } + n = cache->write_bytes(&reloc_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // align to heap word size before writing the relocs so we can + // install them into a code buffer when they get restored + if (!cache->align_write()) { + return false; + } + } + uint reloc_data_size = (uint)(reloc_count * sizeof(relocInfo)); + n = cache->write_bytes(reloc_data, reloc_data_size); if (n != reloc_data_size) { return false; } @@ -875,7 +994,29 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } #endif /* PRODUCT */ - if (!cache->write_relocations(blob)) { + // In the case of a multi-stub blob we need to write start, end, + // secondary entries and extras. For any other blob entry addresses + // beyond the blob start will be stored in the blob as offsets. + if (stub_data != nullptr) { + if (!cache->write_stub_data(blob, stub_data)) { + return false; + } + } + + // now we have added all the other data we can write the AOT + // relocations + + bool write_ok; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + RelocIterator iter(cs); + write_ok = cache->write_relocations(blob, iter); + } else { + RelocIterator iter(&blob); + write_ok = cache->write_relocations(blob, iter); + } + + if (!write_ok) { if (!cache->failed()) { // We may miss an address in AOT table - skip this code blob. cache->set_write_position(entry_position); @@ -884,6 +1025,7 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } uint entry_size = cache->_write_position - entry_position; + AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id), entry_position, entry_size, name_offset, name_size, blob_offset, has_oop_maps, blob.content_begin()); @@ -891,19 +1033,130 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind return true; } -bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), - "wrong entry kind for blob id %s", StubInfo::name(id)); - return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id)); +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return store_code_blob(blob, entry_kind, (uint)id, name, nullptr, nullptr); } -CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) { + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), + "wrong entry kind for blob id %s", StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), nullptr, nullptr); +} + +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), + "wrong entry kind for multi stub blob id %s", StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), stub_data, code_buffer); +} + +bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) { + BlobId blob_id = stub_data->blob_id(); + StubId stub_id = StubInfo::stub_base(blob_id); + address blob_base = blob.code_begin(); + int stub_cnt = StubInfo::stub_count(blob_id); + int n; + + LogStreamHandle(Trace, aot, codecache, stubs) log; + + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", _write_position); + } + + for (int i = 0; i < stub_cnt; i++, stub_id = StubInfo::next_in_blob(blob_id, stub_id)) { + // for each stub we find in the ranges list we write an int + // sequence where + // + // - start_pos is the stub start address encoded as a code section offset + // + // - end is the stub end address encoded as an offset from start + // + // - N counts the number of stub-local entries/extras + // + // - offseti is a stub-local entry/extra address encoded as len for + // a null address otherwise as an offset in range [1,len-1] + + StubAddrRange& range = stub_data->get_range(i); + GrowableArray
& addresses = stub_data->address_array(); + int base = range.start_index(); + if (base >= 0) { + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + address start = addresses.at(base); + assert (blob_base <= start, "sanity"); + uint offset = (uint)(start - blob_base); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + address end = addresses.at(base + 1); + assert (start < end, "sanity"); + offset = (uint)(end - start); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + // write number of secondary and extra entries + int count = range.count() - 2; + n = write_bytes(&count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int j = 0; j < count; j++) { + address next = addresses.at(base + 2 + j); + if (next != nullptr) { + // n.b. This maps next == end to the stub length which + // means we will reconstitute the address as nullptr. That + // happens when we have a handler range covers the end of + // a stub and needs to be handled specially by the client + // that restores the extras. + assert(start <= next && next <= end, "sanity"); + offset = (uint)(next - start); + } else { + // this can happen when a stub is not generated or an + // extra is the common handler target + offset = (uint)(end - start); + } + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + } + if (log.is_enabled()) { + log.print_cr("======== wrote stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), range.count(), _write_position); + } + } + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + // write NO_STUBID as an end marker + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", _write_position); + } + + return true; +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data) { AOTCodeCache* cache = open_for_use(); if (cache == nullptr) { return nullptr; } assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) { return nullptr; } @@ -917,20 +1170,32 @@ CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, c return nullptr; } AOTCodeReader reader(cache, entry); - CodeBlob* blob = reader.compile_code_blob(name); + CodeBlob* blob = reader.compile_code_blob(name, entry_kind, id, stub_data); log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache", (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]); return blob; } -CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), - "wrong entry kind for blob id %s", StubInfo::name(id)); - return load_code_blob(entry_kind, (uint)id, StubInfo::name(id)); +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return load_code_blob(entry_kind, (uint)id, name, nullptr); } -CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) { + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), + "wrong entry kind for blob id %s", StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), nullptr); +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), + "wrong entry kind for blob id %s", StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), stub_data); +} + +CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data) { uint entry_position = _entry->offset(); // Read name @@ -945,13 +1210,19 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return nullptr; } - // Read archived code blob + // Read archived code blob and related info uint offset = entry_position + _entry->blob_offset(); CodeBlob* archived_blob = (CodeBlob*)addr(offset); offset += archived_blob->size(); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // position of relocs will have been aligned to heap word size so + // we can install them into a code buffer + offset = align_up(offset, DATA_ALIGNMENT); + } address reloc_data = (address)addr(offset); - offset += archived_blob->relocation_size(); + offset += reloc_count * sizeof(relocInfo); set_read_position(offset); ImmutableOopMapSet* oop_maps = nullptr; @@ -959,11 +1230,13 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { oop_maps = read_oop_map_set(); } + // Note that for a non-relocatable blob reloc_data will not be + // restored into the blob. We fix that later. + CodeBlob* code_blob = CodeBlob::create(archived_blob, stored_name, reloc_data, - oop_maps - ); + oop_maps); if (code_blob == nullptr) { // no space left in CodeCache return nullptr; } @@ -975,7 +1248,49 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { read_dbg_strings(code_blob->dbg_strings()); #endif // PRODUCT - fix_relocations(code_blob); + if (AOTCodeEntry::is_blob(entry_kind)) { + BlobId blob_id = static_cast(id); + if (StubInfo::is_stubgen(blob_id)) { + assert(stub_data != nullptr, "sanity"); + read_stub_data(code_blob, stub_data); + } + // publish entries found either in stub_data or as offsets in blob + AOTCodeCache::publish_stub_addresses(*code_blob, blob_id, stub_data); + } + + // Now that all the entry points are in the address table we can + // read all the extra reloc info and fix up any addresses that need + // patching to adjust for a new location in a new JVM. We can be + // sure to correctly update all runtime references, including + // cross-linked stubs that are internally daisy-chained. If + // relocation fails and we have to re-generate any of the stubs then + // the entry points for newly generated stubs will get updated, + // ensuring that any other stubs or nmethods we need to relocate + // will use the correct address. + + // if we have a relocatable code blob then the relocs are already + // attached to the blob and we can iterate over it to find the ones + // we need to patch. With a non-relocatable code blob we need to + // wrap it with a CodeBuffer and then reattach the relocs to the + // code buffer. + + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // the blob doesn't have any proper runtime relocs but we can + // reinstate the AOT-load time relocs we saved from the code + // buffer that generated this blob in a new code buffer and use + // the latter to iterate over them + CodeBuffer code_buffer(code_blob); + relocInfo* locs = (relocInfo*)reloc_data; + code_buffer.insts()->initialize_shared_locs(locs, reloc_count); + code_buffer.insts()->set_locs_end(locs + reloc_count); + CodeSection *cs = code_buffer.code_section(CodeBuffer::SECT_INSTS); + RelocIterator reloc_iter(cs); + fix_relocations(code_blob, reloc_iter); + } else { + // the AOT-load time relocs will be in the blob's restored relocs + RelocIterator reloc_iter(code_blob); + fix_relocations(code_blob, reloc_iter); + } #ifdef ASSERT LogStreamHandle(Trace, aot, codecache, stubs) log; @@ -987,15 +1302,146 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return code_blob; } -// ------------ process code and data -------------- +void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) { + GrowableArray
& addresses = stub_data->address_array(); + // Read the list of stub ids and associated start, end, secondary + // and extra addresses and install them in the stub data. + // + // Also insert all start and secondary addresses into the AOTCache + // address table so we correctly relocate this blob and any followng + // blobs/nmethods. + // + // n.b. if an error occurs and we need to regenerate any of these + // stubs the address table will be updated as a side-effect of + // regeneration. + + address blob_base = code_blob->code_begin(); + uint blob_size = (uint)(code_blob->code_end() - blob_base); + int offset = read_position(); + LogStreamHandle(Trace, aot, codecache, stubs) log; + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", offset); + } + // read stub and entries until we see NO_STUBID + StubId stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + // we ought to have at least one saved stub in the blob + assert(stub_id != StubId::NO_STUBID, "blob %s contains no stubs!", StubInfo::name(stub_data->blob_id())); + while (stub_id != StubId::NO_STUBID) { + assert(StubInfo::blob(stub_id) == stub_data->blob_id(), "sanity"); + int idx = StubInfo::stubgen_offset_in_blob(stub_data->blob_id(), stub_id); + StubAddrRange& range = stub_data->get_range(idx); + // we should only see a stub once + assert(range.start_index() < 0, "repeated entry for stub %s", StubInfo::name(stub_id)); + int address_base = addresses.length(); + // start is an offset from the blob base + uint start = *(uint*)addr(offset); offset += sizeof(uint); + assert(start < blob_size, "stub %s start offset %d exceeds buffer length %d", StubInfo::name(stub_id), start, blob_size); + address stub_start = blob_base + start; + addresses.append(stub_start); + // end is an offset from the stub start + uint end = *(uint*)addr(offset); offset += sizeof(uint); + assert(start + end <= blob_size, "stub %s end offset %d exceeds remaining buffer length %d", StubInfo::name(stub_id), end, blob_size - start); + addresses.append(stub_start + end); + // read count of secondary entries plus extras + int entries_count = *(int*)addr(offset); offset += sizeof(int); + assert(entries_count >= (StubInfo::entry_count(stub_id) - 1), "not enough entries for %s", StubInfo::name(stub_id)); + for (int i = 0; i < entries_count; i++) { + // entry offset is an offset from the stub start less than or + // equal to end + uint entry = *(uint*)addr(offset); offset += sizeof(uint); + assert(entry <= end, "stub %s entry offset %d lies beyond stub end %d", StubInfo::name(stub_id), entry, end); + if (entry < end) { + addresses.append(stub_start + entry); + } else { + // entry offset == end encodes a nullptr + addresses.append(nullptr); + } + } + if (log.is_enabled()) { + log.print_cr("======== read stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), 2 + entries_count, offset); + } + range.init_entry(address_base, 2 + entries_count); + // move on to next stub or NO_STUBID + stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + } + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", offset); + } + + set_read_position(offset); +} + +void AOTCodeCache::publish_external_addresses(GrowableArray
& addresses) { + DEBUG_ONLY( _passed_init2 = true; ) + if (opened_cache == nullptr) { + return; + } + + cache()->_table->add_external_addresses(addresses); +} + +void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, AOTStubData *stub_data) { + if (stub_data != nullptr) { + // register all entries in stub + assert(StubInfo::stub_count(blob_id) > 1, + "multiple stub data provided for single stub blob %s", + StubInfo::name(blob_id)); + assert(blob_id == stub_data->blob_id(), + "blob id %s does not match id in stub data %s", + StubInfo::name(blob_id), + StubInfo::name(stub_data->blob_id())); + // iterate over all stubs in the blob + StubId stub_id = StubInfo::stub_base(blob_id); + int stub_cnt = StubInfo::stub_count(blob_id); + GrowableArray
& addresses = stub_data->address_array(); + for (int i = 0; i < stub_cnt; i++) { + assert(stub_id != StubId::NO_STUBID, "sanity"); + StubAddrRange& range = stub_data->get_range(i); + int base = range.start_index(); + if (base >= 0) { + cache()->add_stub_entries(stub_id, addresses.at(base), &addresses, base + 2); + } + stub_id = StubInfo::next_in_blob(blob_id, stub_id); + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + } else { + // register entry or entries for a single stub blob + StubId stub_id = StubInfo::stub_base(blob_id); + assert(StubInfo::stub_count(blob_id) == 1, + "multiple stub blob %s provided without stub data", + StubInfo::name(blob_id)); + address start = code_blob.code_begin(); + if (StubInfo::entry_count(stub_id) == 1) { + assert(!code_blob.is_deoptimization_stub(), "expecting multiple entries for stub %s", StubInfo::name(stub_id)); + // register the blob base address as the only entry + cache()->add_stub_entries(stub_id, start); + } else { + assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id)); + DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob(); + assert(deopt_blob->unpack() == start, "unexpected offset 0x%lx for deopt stub entry", deopt_blob->unpack() - start); + GrowableArray
addresses; + addresses.append(deopt_blob->unpack_with_exception()); + addresses.append(deopt_blob->unpack_with_reexecution()); + addresses.append(deopt_blob->unpack_with_exception_in_tls()); +#if INCLUDE_JVMCI + addresses.append(deopt_blob->uncommon_trap()); + addresses.append(deopt_blob->implicit_exception_uncommon_trap()); +#endif // INCLUDE_JVMCI + cache()->add_stub_entries(stub_id, start, &addresses, 0); + } + } +} + + // ------------ process code and data -------------- // Can't use -1. It is valid value for jump to iteself destination // used by static call stub: see NativeJump::jump_destination(). #define BAD_ADDRESS_ID -2 -bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { +bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) { GrowableArray reloc_data; - RelocIterator iter(&code_blob); LogStreamHandle(Trace, aot, codecache, reloc) log; while (iter.next()) { int idx = reloc_data.append(0); // default value @@ -1049,6 +1495,11 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { // Write the count first int count = reloc_data.length(); write_bytes(&count, sizeof(int)); + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", count); + log.print( " {"); + } + bool first = true; for (GrowableArrayIterator iter = reloc_data.begin(); iter != reloc_data.end(); ++iter) { uint value = *iter; @@ -1056,23 +1507,43 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { if (n != sizeof(uint)) { return false; } + if (log.is_enabled()) { + if (first) { + first = false; + log.print("%d", value); + } else { + log.print(", %d", value); + } + } } + log.print_cr("}"); return true; } -void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { - LogStreamHandle(Trace, aot, reloc) log; +void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) { uint offset = read_position(); - int count = *(int*)addr(offset); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); - if (log.is_enabled()) { - log.print_cr("======== extra relocations count=%d", count); - } uint* reloc_data = (uint*)addr(offset); - offset += (count * sizeof(uint)); + offset += (reloc_count * sizeof(uint)); set_read_position(offset); - RelocIterator iter(code_blob); + LogStreamHandle(Trace, aot, codecache, reloc) log; + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", reloc_count); + } + if (log.is_enabled()) { + log.print(" {"); + for(int i = 0; i < reloc_count; i++) { + if (i == 0) { + log.print("%d", reloc_data[i]); + } else { + log.print(", %d", reloc_data[i]); + } + } + log.print_cr("}"); + } + int j = 0; while (iter.next()) { switch (iter.type()) { @@ -1121,7 +1592,7 @@ void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { } j++; } - assert(j == count, "sanity"); + assert(j == reloc_count, "sanity"); } bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) { @@ -1231,27 +1702,23 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { //======================= AOTCodeAddressTable =============== -// address table ids for generated routines, external addresses and C -// string addresses are partitioned into positive integer ranges -// defined by the following positive base and max values -// i.e. [_extrs_base, _extrs_base + _extrs_max -1], -// [_blobs_base, _blobs_base + _blobs_max -1], -// ... -// [_c_str_base, _c_str_base + _c_str_max -1], +// address table ids for generated routine entry adresses, external +// addresses and C string addresses are partitioned into positive +// integer ranges defined by the following positive base and max +// values i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_stubs_base, _stubs_base + _stubs_max -1], [_c_str_base, +// _c_str_base + _c_str_max -1], -#define _extrs_max 100 -#define _stubs_max 3 - -#define _shared_blobs_max 20 -#define _C1_blobs_max 10 -#define _blobs_max (_shared_blobs_max+_C1_blobs_max) -#define _all_max (_extrs_max+_stubs_max+_blobs_max) +#define _extrs_max 200 +#define _stubs_max static_cast(EntryId::NUM_ENTRYIDS) #define _extrs_base 0 #define _stubs_base (_extrs_base + _extrs_max) -#define _shared_blobs_base (_stubs_base + _stubs_max) -#define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max) -#define _blobs_end (_shared_blobs_base + _blobs_max) +#define _all_max (_stubs_base + _stubs_max) + +// setter for external addresses and string addresses inserts new +// addresses in the order they are encountered them which must remain +// th esame across an assembly run and subsequent production run #define SET_ADDRESS(type, addr) \ { \ @@ -1259,18 +1726,41 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { assert(type##_length <= type##_max, "increase size"); \ } +// setter for stub entry addresses inserts them using the stub entry +// id as an index + +#define SET_ENTRY_ADDRESS(type, addr, entry_id) \ + { \ + int idx = static_cast(entry); \ + _stub_addr[idx] = (address) (addr); \ + } + static bool initializing_extrs = false; void AOTCodeAddressTable::init_extrs() { if (_extrs_complete || initializing_extrs) return; // Done already - assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting"); - initializing_extrs = true; _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); _extrs_length = 0; + { + // Required by initial stubs + SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); // used by forward_exception +#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) + SET_ADDRESS(_extrs, MacroAssembler::debug64); // used by many eg forward_exception, call_stub +#endif // defined(AMD64) || defined(AARCH64) || defined(RISCV64) +#if defined(AMD64) + SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_std()); // used by call_stub + SET_ADDRESS(_extrs, StubRoutines::x86::addr_mxcsr_rz()); // used by libmFmod +#endif + SET_ADDRESS(_extrs, CompressedOops::base_addr()); // used by call_stub + SET_ADDRESS(_extrs, Thread::current); // used by call_stub + SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); + SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError); + } + // Record addresses of VM runtime methods SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); @@ -1279,6 +1769,93 @@ void AOTCodeAddressTable::init_extrs() { #if defined(AARCH64) && !defined(ZERO) SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); #endif + +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_jbyte_array_copy_ctr); // used by arraycopy stub on arm32 and x86_64 + SET_ADDRESS(_extrs, &SharedRuntime::_jshort_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jint_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jlong_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_oop_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_checkcast_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_generic_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_set_memory_ctr); // used by arraycopy stub +#endif /* PRODUCT */ + + SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); + +#ifdef AMD64 + SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply); + SET_ADDRESS(_extrs, SharedRuntime::montgomery_square); +#endif // AMD64 + + SET_ADDRESS(_extrs, SharedRuntime::d2f); + SET_ADDRESS(_extrs, SharedRuntime::d2i); + SET_ADDRESS(_extrs, SharedRuntime::d2l); + SET_ADDRESS(_extrs, SharedRuntime::dcos); + SET_ADDRESS(_extrs, SharedRuntime::dexp); + SET_ADDRESS(_extrs, SharedRuntime::dlog); + SET_ADDRESS(_extrs, SharedRuntime::dlog10); + SET_ADDRESS(_extrs, SharedRuntime::dpow); + SET_ADDRESS(_extrs, SharedRuntime::drem); + SET_ADDRESS(_extrs, SharedRuntime::dsin); + SET_ADDRESS(_extrs, SharedRuntime::dtan); + SET_ADDRESS(_extrs, SharedRuntime::f2i); + SET_ADDRESS(_extrs, SharedRuntime::f2l); + SET_ADDRESS(_extrs, SharedRuntime::frem); + SET_ADDRESS(_extrs, SharedRuntime::l2d); + SET_ADDRESS(_extrs, SharedRuntime::l2f); + SET_ADDRESS(_extrs, SharedRuntime::ldiv); + SET_ADDRESS(_extrs, SharedRuntime::lmul); + SET_ADDRESS(_extrs, SharedRuntime::lrem); + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc); +#endif /* INCLUDE_JVMTI */ + + SET_ADDRESS(_extrs, SafepointSynchronize::handle_polling_page_exception); + + SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset()); + SET_ADDRESS(_extrs, Thread::current); + + SET_ADDRESS(_extrs, os::javaTimeMillis); + SET_ADDRESS(_extrs, os::javaTimeNanos); +#ifndef PRODUCT + SET_ADDRESS(_extrs, os::breakpoint); +#endif + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events); +#endif /* INCLUDE_JVMTI */ + SET_ADDRESS(_extrs, StubRoutines::crc_table_addr()); +#if defined(AARCH64) + SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); +#endif +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr); + SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); +#endif + +#if defined(AMD64) || defined(AARCH64) || defined(RISCV64) + SET_ADDRESS(_extrs, MacroAssembler::debug64); +#endif +#if defined(AMD64) + SET_ADDRESS(_extrs, StubRoutines::x86::arrays_hashcode_powers_of_31()); +#endif + +#ifdef X86 + SET_ADDRESS(_extrs, LIR_Assembler::float_signmask_pool); + SET_ADDRESS(_extrs, LIR_Assembler::double_signmask_pool); + SET_ADDRESS(_extrs, LIR_Assembler::float_signflip_pool); + SET_ADDRESS(_extrs, LIR_Assembler::double_signflip_pool); +#endif + + SET_ADDRESS(_extrs, JfrIntrinsicSupport::write_checkpoint); + SET_ADDRESS(_extrs, JfrIntrinsicSupport::return_lease); + + + SET_ADDRESS(_extrs, UpcallLinker::handle_uncaught_exception); // used by upcall_stub_exception_handler + { // Required by Shared blobs SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info); @@ -1366,6 +1943,11 @@ void AOTCodeAddressTable::init_extrs() { #if INCLUDE_G1GC SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry); + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_post_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, BarrierSetNMethod::nmethod_stub_entry_barrier); // used by method_entry_barrier + #endif #if INCLUDE_SHENANDOAHGC SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre); @@ -1384,91 +1966,70 @@ void AOTCodeAddressTable::init_extrs() { #endif #endif // ZERO + log_debug(aot, codecache, init)("External addresses opened and recorded"); + // allocate storage for stub entries + _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); + log_debug(aot, codecache, init)("Stub addresses opened"); +} + +void AOTCodeAddressTable::init_extrs2() { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for init_extrs2"); + + { + SET_ADDRESS(_extrs, Continuation::prepare_thaw); // used by cont_thaw + SET_ADDRESS(_extrs, Continuation::thaw_entry()); // used by cont_thaw + SET_ADDRESS(_extrs, ContinuationEntry::thaw_call_pc_address()); // used by cont_preempt_stub + } _extrs_complete = true; + initializing_extrs = false; + log_debug(aot, codecache, init)("External addresses recorded and closed"); +} + +void AOTCodeAddressTable::add_external_addresses(GrowableArray
& addresses) { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for add_external_addresses"); + for (int i = 0; i < addresses.length(); i++) { + SET_ADDRESS(_extrs, addresses.at(i)); + } log_debug(aot, codecache, init)("External addresses recorded"); } -static bool initializing_early_stubs = false; - -void AOTCodeAddressTable::init_early_stubs() { - if (_complete || initializing_early_stubs) return; // Done already - initializing_early_stubs = true; - _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); - _stubs_length = 0; - SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); - - { - // Required by C1 blobs -#if defined(AMD64) && !defined(ZERO) - SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip()); - SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup()); -#endif // AMD64 - } - - _early_stubs_complete = true; - log_info(aot, codecache, init)("Early stubs recorded"); +void AOTCodeAddressTable::add_stub_entry(EntryId entry_id, address a) { + assert(_extrs_complete || initializing_extrs, + "recording stub entry address before external addresses complete"); + assert(!(StubInfo::is_shared(StubInfo::stub(entry_id)) && _shared_stubs_complete), "too late to add shared entry"); + assert(!(StubInfo::is_stubgen(StubInfo::stub(entry_id)) && _stubgen_stubs_complete), "too late to add stubgen entry"); + assert(!(StubInfo::is_c1(StubInfo::stub(entry_id)) && _c1_stubs_complete), "too late to add c1 entry"); + assert(!(StubInfo::is_c2(StubInfo::stub(entry_id)) && _c2_stubs_complete), "too late to add c2 entry"); + log_debug(aot, stubs)("Recording address 0x%p for %s entry %s", a, StubInfo::name(StubInfo::stubgroup(entry_id)), StubInfo::name(entry_id)); + int idx = static_cast(entry_id); + _stubs_addr[idx] = a; } -static bool initializing_shared_blobs = false; - -void AOTCodeAddressTable::init_shared_blobs() { - if (_complete || initializing_shared_blobs) return; // Done already - initializing_shared_blobs = true; - address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); - - // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel - _shared_blobs_addr = blobs_addr; - _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max; - - _shared_blobs_length = 0; - _C1_blobs_length = 0; - - // clear the address table - memset(blobs_addr, 0, sizeof(address)* _blobs_max); - - // Record addresses of generated code blobs - SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls()); -#if INCLUDE_JVMCI - if (EnableJVMCI) { - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap()); - } -#endif - - _shared_blobs_complete = true; - log_debug(aot, codecache, init)("Early shared blobs recorded"); - _complete = true; +void AOTCodeAddressTable::set_shared_stubs_complete() { + assert(!_shared_stubs_complete, "repeated close for shared stubs!"); + _shared_stubs_complete = true; + log_debug(aot, codecache, init)("Shared stubs closed"); } -void AOTCodeAddressTable::init_early_c1() { -#ifdef COMPILER1 - // Runtime1 Blobs - StubId id = StubInfo::stub_base(StubGroup::C1); - // include forward_exception in range we publish - StubId limit = StubInfo::next(StubId::c1_forward_exception_id); - for (; id != limit; id = StubInfo::next(id)) { - if (Runtime1::blob_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id)); - continue; - } - if (Runtime1::entry_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id)); - continue; - } - address entry = Runtime1::entry_for(id); - SET_ADDRESS(_C1_blobs, entry); - } -#endif // COMPILER1 - assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length); - _early_c1_complete = true; +void AOTCodeAddressTable::set_c1_stubs_complete() { + assert(!_c1_stubs_complete, "repeated close for c1 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C1 stubs closed"); } -#undef SET_ADDRESS +void AOTCodeAddressTable::set_c2_stubs_complete() { + assert(!_c2_stubs_complete, "repeated close for c2 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C2 stubs closed"); +} + +void AOTCodeAddressTable::set_stubgen_stubs_complete() { + assert(!_stubgen_stubs_complete, "repeated close for stubgen stubs!"); + _stubgen_stubs_complete = true; + log_debug(aot, codecache, init)("Stubgen stubs closed"); +} AOTCodeAddressTable::~AOTCodeAddressTable() { if (_extrs_addr != nullptr) { @@ -1477,9 +2038,6 @@ AOTCodeAddressTable::~AOTCodeAddressTable() { if (_stubs_addr != nullptr) { FREE_C_HEAP_ARRAY(address, _stubs_addr); } - if (_shared_blobs_addr != nullptr) { - FREE_C_HEAP_ARRAY(address, _shared_blobs_addr); - } } #ifdef PRODUCT @@ -1564,7 +2122,7 @@ const char* AOTCodeCache::add_C_string(const char* str) { } const char* AOTCodeAddressTable::add_C_string(const char* str) { - if (_extrs_complete) { + if (_extrs_complete || initializing_extrs) { // Check previous strings address for (int i = 0; i < _C_strings_count; i++) { if (_C_strings_in[i] == str) { @@ -1627,7 +2185,7 @@ static int search_address(address addr, address* table, uint length) { } address AOTCodeAddressTable::address_for_id(int idx) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); if (idx == -1) { return (address)-1; } @@ -1644,15 +2202,9 @@ address AOTCodeAddressTable::address_for_id(int idx) { if (/* id >= _extrs_base && */ id < _extrs_length) { return _extrs_addr[id - _extrs_base]; } - if (id >= _stubs_base && id < _stubs_base + _stubs_length) { + if (id >= _stubs_base && id < _c_str_base) { return _stubs_addr[id - _stubs_base]; } - if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) { - return _shared_blobs_addr[id - _shared_blobs_base]; - } - if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) { - return _C1_blobs_addr[id - _C1_blobs_base]; - } if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { return address_for_C_string(id - _c_str_base); } @@ -1661,7 +2213,7 @@ address AOTCodeAddressTable::address_for_id(int idx) { } int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); int id = -1; if (addr == (address)-1) { // Static call stub has jump to itself return id; @@ -1671,9 +2223,9 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB if (id >= 0) { return id + _c_str_base; } - if (StubRoutines::contains(addr)) { - // Search in stubs - id = search_address(addr, _stubs_addr, _stubs_length); + if (StubRoutines::contains(addr) || CodeCache::find_blob(addr) != nullptr) { + // Search for a matching stub entry + id = search_address(addr, _stubs_addr, _stubs_max); if (id < 0) { StubCodeDesc* desc = StubCodeDesc::desc_for(addr); if (desc == nullptr) { @@ -1685,51 +2237,39 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB return id + _stubs_base; } } else { - CodeBlob* cb = CodeCache::find_blob(addr); - if (cb != nullptr) { - // Search in code blobs - int id_base = _shared_blobs_base; - id = search_address(addr, _shared_blobs_addr, _blobs_max); - if (id < 0) { - assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name()); + // Search in runtime functions + id = search_address(addr, _extrs_addr, _extrs_length); + if (id < 0) { + ResourceMark rm; + const int buflen = 1024; + char* func_name = NEW_RESOURCE_ARRAY(char, buflen); + int offset = 0; + if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { + if (offset > 0) { + // Could be address of C string + uint dist = (uint)pointer_delta(addr, (address)os::init, 1); + log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", + p2i(addr), dist, (const char*)addr); + assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); + return dist; + } +#ifdef ASSERT + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); +#endif } else { - return id_base + id; +#ifdef ASSERT + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + os::find(addr, tty); + assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); +#endif } } else { - // Search in runtime functions - id = search_address(addr, _extrs_addr, _extrs_length); - if (id < 0) { - ResourceMark rm; - const int buflen = 1024; - char* func_name = NEW_RESOURCE_ARRAY(char, buflen); - int offset = 0; - if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { - if (offset > 0) { - // Could be address of C string - uint dist = (uint)pointer_delta(addr, (address)os::init, 1); - log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", - p2i(addr), dist, (const char*)addr); - assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); - return dist; - } -#ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); -#endif - } else { -#ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - os::find(addr, tty); - assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); -#endif - } - } else { - return _extrs_base + id; - } + return _extrs_base + id; } } return id; @@ -1758,3 +2298,176 @@ void AOTCodeCache::print_on(outputStream* st) { } } } + +// methods for managing entries in multi-stub blobs + + +AOTStubData::AOTStubData(BlobId blob_id) : + _blob_id(blob_id), + _cached_blob(nullptr), + _stub_cnt(0), + _ranges(nullptr), + _current(StubId::NO_STUBID), + _current_idx(-1), + _flags(0) { + assert(StubInfo::is_stubgen(blob_id), + "AOTStubData expects a multi-stub blob not %s", + StubInfo::name(blob_id)); + + // we cannot save or restore preuniversestubs because the cache + // cannot be accessed before initialising the universe + if (blob_id == BlobId::stubgen_preuniverse_id) { + // invalidate any attempt to use this + _flags |= INVALID; + return; + } + if (AOTCodeCache::is_on()) { + // allow update of stub entry addresses + _flags |= OPEN; + if (AOTCodeCache::is_using_stub()) { + // allow stub loading + _flags |= USING; + } + if (AOTCodeCache::is_dumping_stub()) { + // allow stub saving + _flags |= DUMPING; + } + // we need to track all the blob's entries + _stub_cnt = StubInfo::stub_count(_blob_id); + _ranges = NEW_C_HEAP_ARRAY(StubAddrRange, _stub_cnt, mtCode); + for (int i = 0; i < _stub_cnt; i++) { + _ranges[i].default_init(); + } + } +} + +bool AOTStubData::load_code_blob() { + assert(is_using(), "should not call"); + assert(!is_invalid() && _cached_blob == nullptr, "repeated init"); + _cached_blob = AOTCodeCache::load_code_blob(AOTCodeEntry::StubGenBlob, + _blob_id, + this); + if (_cached_blob == nullptr) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) { + assert(is_dumping(), "should not call"); + assert(_cached_blob == nullptr, "should not be loading and storing!"); + if (!AOTCodeCache::store_code_blob(new_blob, + AOTCodeEntry::StubGenBlob, + _blob_id, this, code_buffer)) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::find_archive_data(StubId stub_id) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + if (is_invalid()) { + return false; + } + int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt); + // ensure we have a valid associated range + StubAddrRange range = _ranges[idx]; + int start_index = range.start_index(); + if (start_index < 0) { + _current = StubId::NO_STUBID; +#ifdef DEBUG + // reset index so we can idenitfy which ones we failed to find + range.init_entry(-2, 0); +#endif + return false; + } + _current = stub_id; + _current_idx = idx; + return true; +} + +void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(_current == stub_id && stub_id != StubId::NO_STUBID, "sanity check"); + assert(!is_invalid(), "should not load stubs when archive data is invalid"); + assert(_current_idx >= 0, "sanity"); + StubAddrRange& range = _ranges[_current_idx]; + int base = range.start_index(); + int count = range.count(); + assert(base >= 0, "sanity"); + assert(count >= 2, "sanity"); + // first two saved addresses are start and end + start = _address_array.at(base); + end = _address_array.at(base + 1); + assert(start != nullptr, "failed to load start address of stub %s", StubInfo::name(stub_id)); + assert(end != nullptr, "failed to load end address of stub %s", StubInfo::name(stub_id)); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + + int entry_count = StubInfo::entry_count(stub_id); + // the address count must at least include the stub start, end + // and secondary addresses + assert(count >= entry_count + 1, "stub %s requires %d saved addresses but only has %d", StubInfo::name(stub_id), entry_count + 1, count); + + // caller must retrieve secondary entries if and only if they exist + assert((entry_count == 1) == (entries == nullptr), "trying to retrieve wrong number of entries for stub %s", StubInfo::name(stub_id)); + int index = 2; + if (entries != nullptr) { + assert(entries->length() == 0, "non-empty array when retrieving entries for stub %s!", StubInfo::name(stub_id)); + while (index < entry_count + 1) { + address entry = _address_array.at(base + index++); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + entries->append(entry); + } + } + // caller must retrieve extras if and only if they exist + assert((index < count) == (extras != nullptr), "trying to retrieve wrong number of extras for stub %s", StubInfo::name(stub_id)); + if (extras != nullptr) { + assert(extras->length() == 0, "non-empty array when retrieving extras for stub %s!", StubInfo::name(stub_id)); + while (index < count) { + address extra = _address_array.at(base + index++); + assert(extra == nullptr || (start <= extra && extra < end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + extras->append(extra); + } + } +} + +void AOTStubData::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(start != nullptr, "start address cannot be null"); + assert(end != nullptr, "end address cannot be null"); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + _current = stub_id; + _current_idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + StubAddrRange& range = _ranges[_current_idx]; + assert(range.start_index() == -1, "sanity"); + int base = _address_array.length(); + assert(base >= 0, "sanity"); + // first two saved addresses are start and end + _address_array.append(start); + _address_array.append(end); + // caller must save secondary entries if and only if they exist + assert((StubInfo::entry_count(stub_id) == 1) == (entries == nullptr), "trying to save wrong number of entries for stub %s", StubInfo::name(stub_id)); + if (entries != nullptr) { + assert(entries->length() == StubInfo::entry_count(stub_id) - 1, "incorrect entry count %d when saving entries for stub %s!", entries->length(), StubInfo::name(stub_id)); + for (int i = 0; i < entries->length(); i++) { + address entry = entries->at(i); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + _address_array.append(entry); + } + } + // caller may wish to save extra addresses + if (extras != nullptr) { + for (int i = 0; i < extras->length(); i++) { + address extra = extras->at(i); + // handler range end may be end -- it gets restored as nullptr + assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + _address_array.append(extra); + } + } + range.init_entry(base, _address_array.length() - base); +} diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 778ad34e448..2487dc7ff61 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -37,6 +37,7 @@ class CodeBuffer; class RelocIterator; class AOTCodeCache; +class AOTCodeReader; class AdapterBlob; class ExceptionBlob; class ImmutableOopMapSet; @@ -52,6 +53,7 @@ enum CompLevel : signed char; Fn(SharedBlob) \ Fn(C1Blob) \ Fn(C2Blob) \ + Fn(StubGenBlob) \ // Descriptor of AOT Code Cache's entry class AOTCodeEntry { @@ -113,7 +115,9 @@ public: address dumptime_content_start_addr() const { return _dumptime_content_start_addr; } static bool is_valid_entry_kind(Kind kind) { return kind > None && kind < Kind_count; } - static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob; } + static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_single_stub_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_multi_stub_blob(Kind kind) { return kind == StubGenBlob; } static bool is_adapter(Kind kind) { return kind == Adapter; } }; @@ -122,40 +126,36 @@ class AOTCodeAddressTable : public CHeapObj { private: address* _extrs_addr; address* _stubs_addr; - address* _shared_blobs_addr; - address* _C1_blobs_addr; uint _extrs_length; - uint _stubs_length; - uint _shared_blobs_length; - uint _C1_blobs_length; bool _extrs_complete; - bool _early_stubs_complete; - bool _shared_blobs_complete; - bool _early_c1_complete; + bool _shared_stubs_complete; + bool _c1_stubs_complete; + bool _c2_stubs_complete; + bool _stubgen_stubs_complete; bool _complete; public: AOTCodeAddressTable() : _extrs_addr(nullptr), _stubs_addr(nullptr), - _shared_blobs_addr(nullptr), - _C1_blobs_addr(nullptr), _extrs_length(0), - _stubs_length(0), - _shared_blobs_length(0), - _C1_blobs_length(0), _extrs_complete(false), - _early_stubs_complete(false), - _shared_blobs_complete(false), - _early_c1_complete(false), + _shared_stubs_complete(false), + _c1_stubs_complete(false), + _c2_stubs_complete(false), + _stubgen_stubs_complete(false), _complete(false) { } ~AOTCodeAddressTable(); void init_extrs(); - void init_early_stubs(); - void init_shared_blobs(); - void init_early_c1(); + void init_extrs2(); + void add_stub_entry(EntryId entry_id, address entry); + void add_external_addresses(GrowableArray
& addresses); + void set_shared_stubs_complete(); + void set_c1_stubs_complete(); + void set_c2_stubs_complete(); + void set_stubgen_stubs_complete(); const char* add_C_string(const char* str); int id_for_C_string(address str); address address_for_C_string(int idx); @@ -163,6 +163,138 @@ public: address address_for_id(int id); }; +// Auxiliary class used by AOTStubData to locate addresses owned by a +// stub in the _address_array. + +class StubAddrRange { +private: + // Index of the first address owned by a stub or -1 if none present + int _start_index; + // Total number of addresses owned by a stub, including in order: + // start address for stub code and first entry, (exclusive) end + // address for stub code, all secondary entry addresses, any + // auxiliary addresses + uint _naddr; + public: + StubAddrRange() : _start_index(-1), _naddr(0) {} + int start_index() { return _start_index; } + int count() { return _naddr; } + + void default_init() { + _start_index = -1; + _naddr = 0; + } + + void init_entry(int start_index, int naddr) { + _start_index = start_index; + _naddr = naddr; + } +}; + +// class used to save and restore details of stubs embedded in a +// multi-stub (StubGen) blob + +class AOTStubData : public StackObj { + friend class AOTCodeCache; + friend class AOTCodeReader; +private: + BlobId _blob_id; // must be a stubgen blob id + // whatever buffer blob was successfully loaded from the AOT cache + // following a call to load_code_blob or nullptr + CodeBlob *_cached_blob; + // Array of addresses owned by stubs. Each stub appends addresses to + // this array as a block, whether at the end of generation or at the + // end of restoration from the cache. The first two addresses in + // each block are the "start" and "end2 address of the stub. Any + // other visible addresses located within the range [start,end) + // follow, either extra entries, data addresses or SEGV-protected + // subrange start, end and handler addresses. In the special case + // that the SEGV handler address is the (external) common address + // handler the array will hold value nullptr. + GrowableArray
_address_array; + // count of how many stubs exist in the current blob (not all of + // which may actually be generated) + int _stub_cnt; + // array identifying range of entries in _address_array for each stub + // indexed by offset of stub in blob + StubAddrRange* _ranges; + // id of last looked up stub or NO_STUBID if lookup not attempted or failed + StubId _current; + // offset of _current in blob or -1 if lookup not attempted or failed + int _current_idx; + + // flags indicating whether the AOT code cache is open and, if so, + // whether we are loading or storing stubs.the first of those + // cases whether we have encountered any invalid stubs or failed to + // find a stub that was being generated + enum Flags { + OPEN = 1 << 0, // cache is open for use + USING = 1 << 1, // open and loading stubs + DUMPING = 1 << 2, // open and storing stubs + INVALID = 1 << 3, // found invalid stub when loading + }; + + uint32_t _flags; + + void set_invalid() { _flags |= INVALID; } + + StubAddrRange& get_range(int idx) const { return _ranges[idx]; } + GrowableArray
& address_array() { return _address_array; } + // accessor for entry/auxiliary addresses defaults to start entry +public: + AOTStubData(BlobId blob_id); + + ~AOTStubData() { + FREE_C_HEAP_ARRAY(StubAddrRange, _ranges); + } + + bool is_open() { return (_flags & OPEN) != 0; } + bool is_using() { return (_flags & USING) != 0; } + bool is_dumping() { return (_flags & DUMPING) != 0; } + bool is_aot() { return is_using() || is_dumping(); } + bool is_invalid() { return (_flags & INVALID) != 0; } + + BlobId blob_id() { return _blob_id; } + StubId current_stub_id() { return _current; } + // + bool load_code_blob(); + bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffe); + // determine whether a stub is available in the AOT cache + bool find_archive_data(StubId stub_id); + // retrieve stub entry data if it we are using archived stubs and + // the stub has been found in an AOT-restored blob or store stub + // entry data if we are saving archived stubs and the stub has just + // been successfully generated into the current blob. + // + // start and end identify the inclusive start and exclusive end + // address for stub code and must lie in the current blob's code + // range. Stubs presented via this interface must declare at least + // one entry and start is always taken to be the first entry. + // + // Optional arrays entries and extras present other addresses of + // interest all of which must either lie in the interval (start, + // end) or be nullptr (verified by load and store methods). + // + // entries lists secondary entries for the stub each of which must + // match a corresponding entry declaration for the stub (entry count + // verified by load and store methods). Null entry addresses are + // allowed when an architecture does not require a specific entry + // but may not vary from one run to the next. If the cache is in use + // at a store (for loading or saving code) then non-null entry + // addresses are entered into the AOT cache stub address table + // allowing references to them from other stubs or nmethods to be + // relocated. + // + // extras lists other non-entry stub addresses of interest such as + // memory protection ranges and associated handler addresses. These + // do do not need to be declared as entries and their number and + // meaning may vary according to the architecture. + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr); + + const AOTStubData* as_const() { return (const AOTStubData*)this; } +}; + class AOTCodeCache : public CHeapObj { // Classes used to describe AOT code cache. @@ -284,6 +416,7 @@ private: void clear_lookup_failed() { _lookup_failed = false; } bool lookup_failed() const { return _lookup_failed; } + void add_stub_entry(EntryId entry_id, address entry) NOT_CDS_RETURN; public: AOTCodeCache(bool is_dumping, bool is_using); ~AOTCodeCache(); @@ -300,9 +433,12 @@ public: void load_strings(); int store_strings(); - static void init_early_stubs_table() NOT_CDS_RETURN; - static void init_shared_blobs_table() NOT_CDS_RETURN; - static void init_early_c1_table() NOT_CDS_RETURN; + static void set_shared_stubs_complete(); + static void set_c1_stubs_complete(); + static void set_c2_stubs_complete(); + static void set_stubgen_stubs_complete(); + + void add_stub_entries(StubId stub_id, address start, GrowableArray
*entries = nullptr, int offset = -1) NOT_CDS_RETURN; address address_for_C_string(int idx) const { return _table->address_for_C_string(idx); } address address_for_id(int id) const { return _table->address_for_id(id); } @@ -322,22 +458,41 @@ public: bool finish_write(); - bool write_relocations(CodeBlob& code_blob); + bool write_relocations(CodeBlob& code_blob, RelocIterator& iter); bool write_oop_map_set(CodeBlob& cb); + bool write_stub_data(CodeBlob& blob, AOTStubData *stub_data); #ifndef PRODUCT bool write_asm_remarks(CodeBlob& cb); bool write_dbg_strings(CodeBlob& cb); #endif // PRODUCT +private: + // internal private API to save and restore blobs + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind entry_kind, + uint id, + const char* name, + AOTStubData* stub_data, + CodeBuffer* code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + uint id, + const char* name, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + +public: // save and restore API for non-enumerable code blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, - uint id, const char* name) NOT_CDS_RETURN_(false); + uint id, + const char* name) NOT_CDS_RETURN_(false); static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, uint id, const char* name) NOT_CDS_RETURN_(nullptr); // save and restore API for enumerable code blobs + + // API for single-stub blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) NOT_CDS_RETURN_(false); @@ -345,6 +500,22 @@ public: static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, BlobId id) NOT_CDS_RETURN_(nullptr); + // API for multi-stub blobs -- for use by class StubGenerator. + + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data, + CodeBuffer *code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + + static void publish_external_addresses(GrowableArray
& addresses); + // publish all entries for a code blob in code cache address table + static void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); + static uint store_entries_cnt() { if (is_on_for_dump()) { return cache()->_store_entries_cnt; @@ -369,6 +540,7 @@ public: static AOTCodeCache* cache() { assert(_passed_init2, "Too early to ask"); return _cache; } static void initialize() NOT_CDS_RETURN; static void init2() NOT_CDS_RETURN; + static void init3() NOT_CDS_RETURN; static void close() NOT_CDS_RETURN; static bool is_on() CDS_ONLY({ return cache() != nullptr && !_cache->closing(); }) NOT_CDS_RETURN_(false); static bool is_on_for_use() CDS_ONLY({ return is_on() && _cache->for_use(); }) NOT_CDS_RETURN_(false); @@ -389,7 +561,7 @@ public: // Concurent AOT code reader class AOTCodeReader { private: - const AOTCodeCache* _cache; + AOTCodeCache* _cache; const AOTCodeEntry* _entry; const char* _load_buffer; // Loaded cached code buffer uint _read_position; // Position in _load_buffer @@ -406,11 +578,14 @@ private: public: AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry); - CodeBlob* compile_code_blob(const char* name); + CodeBlob* compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data = nullptr); ImmutableOopMapSet* read_oop_map_set(); + void read_stub_data(CodeBlob* code_blob, AOTStubData *stub_data); + void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); - void fix_relocations(CodeBlob* code_blob); + + void fix_relocations(CodeBlob* code_blob, RelocIterator& iter); #ifndef PRODUCT void read_asm_remarks(AsmRemarks& asm_remarks); void read_dbg_strings(DbgStrings& dbg_strings); diff --git a/src/hotspot/share/prims/downcallLinker.hpp b/src/hotspot/share/prims/downcallLinker.hpp index 01ee5c56776..b27b18aff25 100644 --- a/src/hotspot/share/prims/downcallLinker.hpp +++ b/src/hotspot/share/prims/downcallLinker.hpp @@ -71,7 +71,7 @@ public: bool needs_return_buffer, int captured_state_mask, bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), + : StubCodeGenerator(buffer, PrintMethodHandleStubs), _signature(signature), _num_args(num_args), _ret_bt(ret_bt), diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index adc49f84358..463460fe3d5 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -70,6 +70,8 @@ void VM_Version_init(); void icache_init2(); void initialize_stub_info(); // must precede all blob/stub generation void preuniverse_stubs_init(); + +void stubs_AOTAddressTable_init(); void initial_stubs_init(); jint universe_init(); // depends on codeCache_init and preuniverse_stubs_init @@ -149,13 +151,17 @@ jint init_globals() { AOTCodeCache::init2(); // depends on universe_init, must be before initial_stubs_init AsyncLogWriter::initialize(); + stubs_AOTAddressTable_init(); // publish external addresses used by stubs + // depends on AOTCodeCache::init2 initial_stubs_init(); // stubgen initial stub routines // stack overflow exception blob is referenced by the interpreter - AOTCodeCache::init_early_stubs_table(); // need this after stubgen initial stubs and before shared runtime initial stubs SharedRuntime::generate_initial_stubs(); gc_barrier_stubs_init(); // depends on universe_init, must be before interpreter_init continuations_init(); // must precede continuation stub generation - continuation_stubs_init(); // depends on continuations_init + AOTCodeCache::init3(); // depends on stubs_AOTAddressTable_init + // and continuations_init and must + // precede continuation stub generation + continuation_stubs_init(); // depends on continuations_init and AOTCodeCache::init3 #if INCLUDE_JFR SharedRuntime::generate_jfr_stubs(); #endif @@ -164,7 +170,6 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); - AOTCodeCache::init_shared_blobs_table(); // need this after generate_stubs SharedRuntime::init_adapter_library(); // do this after AOTCodeCache::init_shared_blobs_table return JNI_OK; } diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 43250c004ca..6a9d82d6fca 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "compiler/disassembler.hpp" #include "oops/oop.inline.hpp" @@ -69,14 +70,16 @@ void StubCodeDesc::print() const { print_on(tty); } StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, bool print_code) { _masm = new MacroAssembler(code); _blob_id = BlobId::NO_BLOBID; + _stub_data = nullptr; _print_code = PrintStubCode || print_code; } -StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code) { +StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data, bool print_code) { assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); _masm = new MacroAssembler(code); _blob_id = blob_id; + _stub_data = stub_data; _print_code = PrintStubCode || print_code; } @@ -91,11 +94,29 @@ StubCodeGenerator::~StubCodeGenerator() { #endif } +void StubCodeGenerator::setup_code_desc(const char* name, address start, address end, bool loaded_from_cache) { + StubCodeDesc* cdesc = new StubCodeDesc("StubRoutines", name, start, end); + cdesc->set_disp(uint(start - _masm->code_section()->outer()->insts_begin())); + if (loaded_from_cache) { + cdesc->set_loaded_from_cache(); + } + print_stub_code_desc(cdesc); + // copied from ~StubCodeMark() + Forte::register_stub(cdesc->name(), cdesc->begin(), cdesc->end()); + if (JvmtiExport::should_post_dynamic_code_generated()) { + JvmtiExport::post_dynamic_code_generated(cdesc->name(), cdesc->begin(), cdesc->end()); + } +} + void StubCodeGenerator::stub_prolog(StubCodeDesc* cdesc) { // default implementation - do nothing } void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { + print_stub_code_desc(cdesc); +} + +void StubCodeGenerator::print_stub_code_desc(StubCodeDesc* cdesc) { LogTarget(Debug, stubs) lt; if (lt.is_enabled()) { LogStream ls(lt); @@ -119,6 +140,53 @@ void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { } } +bool StubCodeGenerator::find_archive_data(StubId stub_id) { + // punt to stub data if it exists and is not for dumping + if (_stub_data == nullptr || _stub_data->is_dumping()) { + return false; + } + return _stub_data->find_archive_data(stub_id); +} + +void StubCodeGenerator::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries, GrowableArray
* extras) { + assert(_stub_data != nullptr && _stub_data->current_stub_id() == stub_id, "no current archive data for %s", StubInfo::name(stub_id)); + + // punt to stub data + _stub_data->load_archive_data(stub_id, start, end, entries, extras); + + setup_code_desc(StubInfo::name(stub_id), start, end, true); +} + +void StubCodeGenerator::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + // punt to stub data if we have any + if (_stub_data != nullptr) { + _stub_data->store_archive_data(stub_id, start, end, entries, extras); + } +} + +void StubCodeGenerator::print_statistics_on(outputStream* st) { + st->print_cr("StubRoutines Stubs:"); + st->print_cr(" Initial stubs: %d", StubInfo::stub_count(BlobId::stubgen_initial_id)); + st->print_cr(" Continuation stubs: %d", StubInfo::stub_count(BlobId::stubgen_continuation_id)); + st->print_cr(" Compiler stubs: %d", StubInfo::stub_count(BlobId::stubgen_compiler_id)); + st->print_cr(" Final stubs: %d", StubInfo::stub_count(BlobId::stubgen_final_id)); + + int emitted = 0; + int loaded_from_cache = 0; + + StubCodeDesc* scd = StubCodeDesc::first(); + while (scd != nullptr) { + if (!strcmp(scd->group(), "StubRoutines")) { + emitted += 1; + if (scd->loaded_from_cache()) { + loaded_from_cache += 1; + } + } + scd = StubCodeDesc::next(scd); + } + st->print_cr("Total stubroutines stubs emitted: %d (generated=%d, loaded from cache=%d)", emitted, emitted - loaded_from_cache, loaded_from_cache); +} + #ifdef ASSERT void StubCodeGenerator::verify_stub(StubId stub_id) { assert(StubRoutines::stub_to_blob(stub_id) == blob_id(), "wrong blob %s for generation of stub %s", StubRoutines::get_blob_name(blob_id()), StubRoutines::get_stub_name(stub_id)); diff --git a/src/hotspot/share/runtime/stubCodeGenerator.hpp b/src/hotspot/share/runtime/stubCodeGenerator.hpp index 7d8944c85ea..fb3c6a292cc 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.hpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.hpp @@ -26,6 +26,7 @@ #define SHARE_RUNTIME_STUBCODEGENERATOR_HPP #include "asm/assembler.hpp" +#include "code/aotCodeCache.hpp" #include "memory/allocation.hpp" #include "runtime/stubInfo.hpp" @@ -48,6 +49,7 @@ class StubCodeDesc: public CHeapObj { address _begin; // points to the first byte of the stub code (included) address _end; // points to the first byte after the stub code (excluded) uint _disp; // Displacement relative base address in buffer. + bool _loaded_from_cache; friend class StubCodeMark; friend class StubCodeGenerator; @@ -65,6 +67,8 @@ class StubCodeDesc: public CHeapObj { void set_disp(uint disp) { _disp = disp; } + void set_loaded_from_cache() { _loaded_from_cache = true; } + public: static StubCodeDesc* first() { return _list; } static StubCodeDesc* next(StubCodeDesc* desc) { return desc->_next; } @@ -81,6 +85,7 @@ class StubCodeDesc: public CHeapObj { _end = end; _disp = 0; _list = this; + _loaded_from_cache = false; }; static void freeze(); @@ -93,12 +98,11 @@ class StubCodeDesc: public CHeapObj { uint disp() const { return _disp; } int size_in_bytes() const { return pointer_delta_as_int(_end, _begin); } bool contains(address pc) const { return _begin <= pc && pc < _end; } + bool loaded_from_cache() const { return _loaded_from_cache; } void print_on(outputStream* st) const; void print() const; }; -// forward declare blob and stub id enums - // The base class for all stub-generating code generators. // Provides utility functions. @@ -108,10 +112,13 @@ class StubCodeGenerator: public StackObj { BlobId _blob_id; protected: MacroAssembler* _masm; + AOTStubData* _stub_data; + + void setup_code_desc(const char* name, address start, address end, bool loaded_from_cache); public: StubCodeGenerator(CodeBuffer* code, bool print_code = false); - StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code = false); + StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data = nullptr, bool print_code = false); ~StubCodeGenerator(); MacroAssembler* assembler() const { return _masm; } @@ -120,9 +127,16 @@ class StubCodeGenerator: public StackObj { virtual void stub_prolog(StubCodeDesc* cdesc); // called by StubCodeMark constructor virtual void stub_epilog(StubCodeDesc* cdesc); // called by StubCodeMark destructor + void print_stub_code_desc(StubCodeDesc* cdesc); + + static void print_statistics_on(outputStream* st); + bool find_archive_data(StubId stub_id); + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); #ifdef ASSERT void verify_stub(StubId stub_id); #endif + }; // Stack-allocated helper class used to associate a stub code with a name. diff --git a/src/hotspot/share/runtime/stubDeclarations.hpp b/src/hotspot/share/runtime/stubDeclarations.hpp index b051c6d0e18..15735797d7c 100644 --- a/src/hotspot/share/runtime/stubDeclarations.hpp +++ b/src/hotspot/share/runtime/stubDeclarations.hpp @@ -975,9 +975,15 @@ do_entry_init(final, arrayof_jlong_arraycopy, \ arrayof_jlong_arraycopy, arrayof_jlong_arraycopy, \ StubRoutines::arrayof_jlong_copy) \ + do_entry(final, arrayof_jlong_arraycopy, \ + arrayof_jlong_arraycopy_nopush, \ + arrayof_jlong_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy) \ do_entry_init(final, arrayof_oop_arraycopy, arrayof_oop_arraycopy, \ arrayof_oop_arraycopy, StubRoutines::arrayof_oop_copy) \ + do_entry(final, arrayof_oop_arraycopy, \ + arrayof_oop_arraycopy_nopush, \ + arrayof_oop_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy_uninit) \ do_entry_init(final, arrayof_oop_arraycopy_uninit, \ arrayof_oop_arraycopy_uninit, \ diff --git a/src/hotspot/share/runtime/stubInfo.cpp b/src/hotspot/share/runtime/stubInfo.cpp index bca6ff344ea..6c7769dd66d 100644 --- a/src/hotspot/share/runtime/stubInfo.cpp +++ b/src/hotspot/share/runtime/stubInfo.cpp @@ -1096,6 +1096,15 @@ int StubInfo::stubgen_offset(StubId id) { return local_offset(StubGroup::STUBGEN, id); } +int StubInfo::stubgen_offset_in_blob(BlobId blob_id, StubId id) { + assert(blob(id) == blob_id, "sanity!"); + StubGroup group = StubGroup::STUBGEN; + assert(stubgroup(blob_id) == group, "sanity"); + StubId base_id = stub_base(blob_id); + assert(base_id != StubId::NO_STUBID, "sanity"); + return local_offset(group, id) - local_offset(group, base_id); +} + // initialization function called to populate blob. stub and entry // tables. this must be called before any stubs are generated void initialize_stub_info() { diff --git a/src/hotspot/share/runtime/stubInfo.hpp b/src/hotspot/share/runtime/stubInfo.hpp index eaea07f7e6c..323d006e850 100644 --- a/src/hotspot/share/runtime/stubInfo.hpp +++ b/src/hotspot/share/runtime/stubInfo.hpp @@ -678,6 +678,11 @@ public: static int c1_offset(StubId id); static int c2_offset(StubId id); static int stubgen_offset(StubId id); + + // Convert a stub id to a unique, zero-based offset in the range of + // stub ids for a given blob in the stubgen stub group. + + static int stubgen_offset_in_blob(BlobId blob_id, StubId id); }; diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 5246613738e..3b8c004b368 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -102,7 +102,10 @@ BlobId StubRoutines::stub_to_blob(StubId id) { // Initialization -extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id); // only interface to generators +extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); // only interface to generators +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) +extern void StubGenerator_AOTAddressTable_init(); +#endif void UnsafeMemoryAccess::create_table(int max_size) { UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size]; @@ -169,6 +172,33 @@ static BufferBlob* initialize_stubs(BlobId blob_id, assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); ResourceMark rm; TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime)); + // If we are loading stubs we need to check if we can retrieve a + // blob and/or an associated archived stub descriptor from the + // AOTCodeCache. If we are storing stubs we need to create a blob + // but we still need a stub data descriptor to fill in during + // generation. + AOTStubData stub_data(blob_id); + AOTStubData* stub_data_p = nullptr; + LogTarget(Info, stubs) lt; + + if (code_size > 0 && stub_data.is_using()) { + // AOTCodeEntry tracks and logs status of any cached blob + bool loaded = stub_data.load_code_blob(); + if (loaded) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Found blob %s in AOT cache", StubInfo::name(blob_id)); + } + stub_data_p = &stub_data; + } + } else if (stub_data.is_dumping()) { + stub_data_p = &stub_data; + } + + // Even if we managed to load a blob from the AOT cache we still + // need to allocate a code blob and associated buffer. The AOT blob + // may not include all the stubs we need for this runtime. + // Add extra space for large CodeEntryAlignment int size = code_size + CodeEntryAlignment * max_aligned_stubs; BufferBlob* stubs_code = BufferBlob::create(buffer_name, size); @@ -178,6 +208,10 @@ static BufferBlob* initialize_stubs(BlobId blob_id, // In that case we can tolerate an allocation failure because the // compiler will have been shut down and we have no need of the // blob. + // TODO: Ideally we would still like to try to use any AOT cached + // blob here but we don't have a fallback if we find that it is + // missing stubs we need so for now we exit. This should only + // happen in cases where we have a very small code cache. if (Thread::current()->is_Compiler_thread()) { assert(blob_id == BlobId::stubgen_compiler_id, "sanity"); assert(DelayCompilerStubsGeneration, "sanity"); @@ -187,7 +221,10 @@ static BufferBlob* initialize_stubs(BlobId blob_id, vm_exit_out_of_memory(code_size, OOM_MALLOC_ERROR, "CodeCache: no room for %s", buffer_name); } CodeBuffer buffer(stubs_code); - StubGenerator_generate(&buffer, blob_id); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + StubGenerator_generate(&buffer, blob_id, stub_data_p); if (code_size == 0) { assert(buffer.insts_size() == 0, "should not write into buffer when bob size declared as 0"); LogTarget(Info, stubs) lt; @@ -203,7 +240,25 @@ static BufferBlob* initialize_stubs(BlobId blob_id, "increase %s, code_size: %d, used: %d, free: %d", assert_msg, code_size, buffer.total_content_size(), buffer.insts_remaining()); - LogTarget(Info, stubs) lt; + if (AOTCodeCache::is_dumping_stub()) { + if (stub_data.store_code_blob(*stubs_code, &buffer)) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Stored blob '%s' to Startup Code Cache", buffer_name); + } + } else { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Failed to store blob '%s' to Startup Code Cache", buffer_name); + } + } + } + + // close off recording of any further stubgen generation + if (blob_id == BlobId::stubgen_final_id) { + AOTCodeCache::set_stubgen_stubs_complete(); + } + if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("%s\t [" INTPTR_FORMAT ", " INTPTR_FORMAT "] used: %d, free: %d", @@ -214,6 +269,8 @@ static BufferBlob* initialize_stubs(BlobId blob_id, return stubs_code; } +// per blob initializer methods StubRoutines::initialize_xxx_stubs() + #define DEFINE_BLOB_INIT_METHOD(blob_name) \ void StubRoutines::initialize_ ## blob_name ## _stubs() { \ if (STUBGEN_BLOB_FIELD_NAME(blob_name) == nullptr) { \ @@ -234,6 +291,7 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_METHOD) #undef DEFINE_BLOB_INIT_METHOD +// external driver API functions for per blob init: xxx_stubs_init() #define DEFINE_BLOB_INIT_FUNCTION(blob_name) \ void blob_name ## _stubs_init() { \ @@ -244,11 +302,24 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_FUNCTION) #undef DEFINE_BLOB_INIT_FUNCTION + +// Non-generated init method + +void StubRoutines::init_AOTAddressTable() { +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) + StubGenerator_AOTAddressTable_init(); +#endif +} + +// non-generated external API init driver function + +void stubs_AOTAddressTable_init() { StubRoutines::init_AOTAddressTable(); } + /* - * we generate the underlying driver method but this wrapper is needed - * to perform special handling depending on where the compiler init - * gets called from. it ought to be possible to remove this at some - * point and have a determinate ordered init. + * we generate the underlying driver function compiler_stubs_init() + * but this wrapper is needed to perform special handling depending on + * where the compiler init gets called from. it ought to be possible + * to remove this at some point and have a determinate ordered init. */ void compiler_stubs_init(bool in_compiler_thread) { diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 97e3e46b870..b2384c33a99 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -130,6 +130,7 @@ class UnsafeMemoryAccess : public CHeapObj { static UnsafeMemoryAccess* add_to_table(address start_pc, address end_pc, address error_exit_pc) { guarantee(_table_length < _table_max_length, "Incorrect UnsafeMemoryAccess::_table_max_length"); UnsafeMemoryAccess* entry = &_table[_table_length]; + assert(start_pc != nullptr, "invalid start address"); entry->set_start_pc(start_pc); entry->set_end_pc(end_pc); entry->set_error_exit_pc(error_exit_pc); @@ -283,6 +284,9 @@ public: static BlobId stub_to_blob(StubId id); #endif + // Initalization + static void init_AOTAddressTable(); + // Debugging static jint verify_oop_count() { return _verify_oop_count; } static jint* verify_oop_count_addr() { return &_verify_oop_count; } diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java index 761c2c5382d..4219cbdf6be 100644 --- a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java +++ b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java @@ -51,7 +51,7 @@ public class AOTCodeFlags { public static void main(String... args) throws Exception { Tester t = new Tester(); // Run only 2 modes (0 - no AOT code, 1 - AOT adapters) until JDK-8357398 is fixed - for (int mode = 0; mode < 2; mode++) { + for (int mode = 0; mode < 4; mode++) { t.setTestMode(mode); t.run(new String[] {"AOT", "--two-step-training"}); }