diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index ece7f8a347af9..4dc674d28c510 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -8379,7 +8379,7 @@ class StubGenerator: public StubCodeGenerator { // Initialize table for copy memory (arraycopy) check. if (UnsafeCopyMemory::_table == nullptr) { - UnsafeCopyMemory::create_table(8); + UnsafeCopyMemory::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory } if (UseCRC32Intrinsics) { diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index 7f9645f749ada..9d526c7ebdca9 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -3135,7 +3135,7 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError)); if (UnsafeCopyMemory::_table == nullptr) { - UnsafeCopyMemory::create_table(32); + UnsafeCopyMemory::create_table(32 + 4); // 32 for copyMemory; 4 for setMemory } // integer division used both by interpreter and compiler diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 094757ad3e16c..4a9e5476dce5b 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2023 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -4746,7 +4746,7 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_catch_exception_entry = generate_catch_exception(); if (UnsafeCopyMemory::_table == nullptr) { - UnsafeCopyMemory::create_table(8); + UnsafeCopyMemory::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory } // Build this early so it's available for the interpreter. diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 7ee17438b284e..8ac8263b30892 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -5501,7 +5501,7 @@ static const int64_t right_3_bits = right_n_bits(3); StubRoutines::_forward_exception_entry = generate_forward_exception(); if (UnsafeCopyMemory::_table == nullptr) { - UnsafeCopyMemory::create_table(8); + UnsafeCopyMemory::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory } StubRoutines::_call_stub_entry = diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp index dd3f9c64e20ef..b9215af2ff6e3 100644 --- a/src/hotspot/cpu/x86/assembler_x86.cpp +++ b/src/hotspot/cpu/x86/assembler_x86.cpp @@ -934,6 +934,7 @@ address Assembler::locate_operand(address inst, WhichOperand which) { case 0x6F: // movdq case 0x7F: // movdq case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush + case 0xD6: // movq case 0xFE: // paddd debug_only(has_disp32 = true); break; diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 3d427fd0cdeb6..6c8ca583b10cb 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -6299,7 +6299,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned, orl(value, rtmp); } - cmpl(count, 2<> 3); + __ decrementq(size); + __ jccb(Assembler::notZero, L_TailLoop); +} + +// Generate 'unsafe' set memory stub +// Though just as safe as the other stubs, it takes an unscaled +// size_t (# bytes) argument instead of an element count. +// +// Input: +// c_rarg0 - destination array address +// c_rarg1 - byte count (size_t) +// c_rarg2 - byte value +// +// Examines the alignment of the operands and dispatches +// to an int, short, or byte fill loop. +// +address StubGenerator::generate_unsafe_setmemory(const char *name, + address unsafe_byte_fill) { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + __ enter(); // required for proper stackwalking of RuntimeStub frame + + assert(unsafe_byte_fill != nullptr, "Invalid call"); + + // bump this on entry, not on exit: + INC_COUNTER_NP(SharedRuntime::_unsafe_set_memory_ctr, rscratch1); + + { + Label L_exit, L_fillQuadwords, L_fillDwords, L_fillBytes; + + const Register dest = c_rarg0; + const Register size = c_rarg1; + const Register byteVal = c_rarg2; + const Register wide_value = rax; + const Register rScratch1 = r10; + + assert_different_registers(dest, size, byteVal, wide_value, rScratch1); + + // fill_to_memory_atomic(unsigned char*, unsigned long, unsigned char) + + __ testq(size, size); + __ jcc(Assembler::zero, L_exit); + + // Propagate byte to full Register + __ movzbl(rScratch1, byteVal); + __ mov64(wide_value, 0x0101010101010101ULL); + __ imulq(wide_value, rScratch1); + + // Check for pointer & size alignment + __ movq(rScratch1, dest); + __ orq(rScratch1, size); + + __ testb(rScratch1, 7); + __ jcc(Assembler::equal, L_fillQuadwords); + + __ testb(rScratch1, 3); + __ jcc(Assembler::equal, L_fillDwords); + + __ testb(rScratch1, 1); + __ jcc(Assembler::notEqual, L_fillBytes); + + // Fill words + { + Label L_wordsTail, L_wordsLoop, L_wordsTailLoop; + UnsafeCopyMemoryMark usmm(this, true, true); + + // At this point, we know the lower bit of size is zero and a + // multiple of 2 + do_setmemory_atomic_loop(USM_SHORT, dest, size, wide_value, rScratch1, + L_exit, _masm); + } + __ jmpb(L_exit); + + __ BIND(L_fillQuadwords); + + // Fill QUADWORDs + { + Label L_qwordLoop, L_qwordsTail, L_qwordsTailLoop; + UnsafeCopyMemoryMark usmm(this, true, true); + + // At this point, we know the lower 3 bits of size are zero and a + // multiple of 8 + do_setmemory_atomic_loop(USM_QUADWORD, dest, size, wide_value, rScratch1, + L_exit, _masm); + } + __ BIND(L_exit); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ ret(0); + + __ BIND(L_fillDwords); + + // Fill DWORDs + { + Label L_dwordLoop, L_dwordsTail, L_dwordsTailLoop; + UnsafeCopyMemoryMark usmm(this, true, true); + + // At this point, we know the lower 2 bits of size are zero and a + // multiple of 4 + do_setmemory_atomic_loop(USM_DWORD, dest, size, wide_value, rScratch1, + L_exit, _masm); + } + __ jmpb(L_exit); + + __ BIND(L_fillBytes); + // Set up for tail call to previously generated byte fill routine + // Parameter order is (ptr, byteVal, size) + __ xchgq(c_rarg1, c_rarg2); + __ leave(); // Clear effect of enter() + __ jump(RuntimeAddress(unsafe_byte_fill)); + } + + return start; +} + // Perform range checks on the proposed arraycopy. // Kills temp, but nothing else. // Also, clean the sign bits of src_pos and dst_pos. diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index c2d6e82622ce8..5c7772021ea35 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright 2007, 2008, 2010, 2015 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -148,6 +148,9 @@ class StubGenerator: public StubCodeGenerator { // Shared code tests for "null" to discover the stub is not generated. StubRoutines::_unsafe_arraycopy = nullptr; + // Shared code tests for "null" to discover the stub is not generated. + StubRoutines::_unsafe_setmemory = nullptr; + // We don't generate specialized code for HeapWord-aligned source // arrays, so just use the code we've already generated StubRoutines::_arrayof_jbyte_disjoint_arraycopy = diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index ddc1b1c335d0f..37f40ebe59688 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -2795,11 +2795,11 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; } - bool is_unsafe_arraycopy = (in_native || in_java) && UnsafeCopyMemory::contains_pc(pc); - if (((in_vm || in_native || is_unsafe_arraycopy) && thread->doing_unsafe_access()) || + bool is_unsafe_memory_access = (in_native || in_java) && UnsafeCopyMemory::contains_pc(pc); + if (((in_vm || in_native || is_unsafe_memory_access) && thread->doing_unsafe_access()) || (nm != nullptr && nm->has_unsafe_access())) { address next_pc = Assembler::locate_next_instruction(pc); - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } return Handle_Exception(exceptionInfo, SharedRuntime::handle_unsafe_access(thread, next_pc)); diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp index 8711c9a89b352..a3101c0228513 100644 --- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp +++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp @@ -340,10 +340,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // underlying file has been truncated. Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = cb ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = pc + 4; - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } next_pc = SharedRuntime::handle_unsafe_access(thread, next_pc); diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp index 3dfe9e30f7904..436c68f5a30cf 100644 --- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp +++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp @@ -257,10 +257,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = pc + NativeCall::instruction_size; - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } stub = SharedRuntime::handle_unsafe_access(thread, next_pc); diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp index 593f6494540ee..35ee2de2aa2da 100644 --- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp +++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp @@ -441,19 +441,17 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = Assembler::locate_next_instruction(pc); - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } stub = SharedRuntime::handle_unsafe_access(thread, next_pc); } - } - else - + } else #ifdef AMD64 - if (sig == SIGFPE && + if (sig == SIGFPE && (info->si_code == FPE_INTDIV || info->si_code == FPE_FLTDIV // Workaround for macOS ARM incorrectly reporting FPE_FLTINV for "div by 0" // instead of the expected FPE_FLTDIV when running x86_64 binary under Rosetta emulation diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp index e1c9dc8a13ab2..fc6cbe5faf0e6 100644 --- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp +++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp @@ -240,10 +240,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = pc + NativeCall::instruction_size; - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } stub = SharedRuntime::handle_unsafe_access(thread, next_pc); diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp index 6f9ac548ce1cd..5b5364e1117f2 100644 --- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp +++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp @@ -324,17 +324,22 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - if ((nm != nullptr && nm->has_unsafe_access()) || (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc))) { + if ((nm != nullptr && nm->has_unsafe_access()) || + (thread->doing_unsafe_access() && + UnsafeCopyMemory::contains_pc(pc))) { unsafe_access = true; } } else if (sig == SIGSEGV && MacroAssembler::uses_implicit_null_check(info->si_addr)) { - // Determination of interpreter/vtable stub/compiled code null exception - CodeBlob* cb = CodeCache::find_blob(pc); - if (cb != nullptr) { - stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL); - } - } else if (sig == SIGILL && *(int *)pc == NativeInstruction::not_entrant_illegal_instruction) { + // Determination of interpreter/vtable stub/compiled code null exception + CodeBlob* cb = CodeCache::find_blob(pc); + if (cb != nullptr) { + stub = SharedRuntime::continuation_for_implicit_exception( + thread, pc, SharedRuntime::IMPLICIT_NULL); + } + } else if (sig == SIGILL && + *(int*)pc == + NativeInstruction::not_entrant_illegal_instruction) { // Not entrant stub = SharedRuntime::get_handle_wrong_method_stub(); } diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp index 90640a6f06a81..89197df449311 100644 --- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp +++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp @@ -355,10 +355,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // underlying file has been truncated. Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = pc + 4; - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } next_pc = SharedRuntime::handle_unsafe_access(thread, next_pc); diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp index 079c3b42a9c3c..ec880236793b6 100644 --- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp +++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp @@ -230,10 +230,10 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = Assembler::locate_next_instruction(pc); - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } stub = SharedRuntime::handle_unsafe_access(thread, next_pc); diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp index b37a8d1f3a624..9815ff2c6cefa 100644 --- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp +++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp @@ -260,19 +260,17 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, // Do not crash the VM in such a case. CodeBlob* cb = CodeCache::find_blob(pc); nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr; - bool is_unsafe_arraycopy = thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc); - if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + bool is_unsafe_memory_access = thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc); + if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_memory_access) { address next_pc = Assembler::locate_next_instruction(pc); - if (is_unsafe_arraycopy) { + if (is_unsafe_memory_access) { next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); } stub = SharedRuntime::handle_unsafe_access(thread, next_pc); } - } - else - + } else #ifdef AMD64 - if (sig == SIGFPE && + if (sig == SIGFPE && (info->si_code == FPE_INTDIV || info->si_code == FPE_FLTDIV)) { stub = SharedRuntime:: diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index 6e72b4cbac31a..8d4f57165e151 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -506,6 +506,9 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_copyMemory: if (!InlineArrayCopy || !InlineUnsafeOps) return true; break; + case vmIntrinsics::_setMemory: + if (!InlineUnsafeOps) return true; + break; #ifdef COMPILER2 case vmIntrinsics::_clone: case vmIntrinsics::_copyOf: diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 6104fb5683b3b..a0db1a65d3a46 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -620,6 +620,9 @@ class methodHandle; do_intrinsic(_copyMemory, jdk_internal_misc_Unsafe, copyMemory_name, copyMemory_signature, F_RN) \ do_name( copyMemory_name, "copyMemory0") \ do_signature(copyMemory_signature, "(Ljava/lang/Object;JLjava/lang/Object;JJ)V") \ + do_intrinsic(_setMemory, jdk_internal_misc_Unsafe, setMemory_name, setMemory_signature, F_RN) \ + do_name( setMemory_name, "setMemory0") \ + do_signature(setMemory_signature, "(Ljava/lang/Object;JJB)V") \ do_intrinsic(_loadFence, jdk_internal_misc_Unsafe, loadFence_name, loadFence_signature, F_R) \ do_name( loadFence_name, "loadFence") \ do_alias( loadFence_signature, void_method_signature) \ diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index a57d1f7e6736f..cadbba27f6e25 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -342,6 +342,7 @@ static_field(StubRoutines, _generic_arraycopy, address) \ static_field(StubRoutines, _array_sort, address) \ static_field(StubRoutines, _array_partition, address) \ + static_field(StubRoutines, _unsafe_setmemory, address) \ \ static_field(StubRoutines, _aescrypt_encryptBlock, address) \ static_field(StubRoutines, _aescrypt_decryptBlock, address) \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 25dfee91a9105..dc15e82dff8f1 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -257,6 +257,9 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_copyMemory: if (StubRoutines::unsafe_arraycopy() == nullptr) return false; break; + case vmIntrinsics::_setMemory: + if (StubRoutines::unsafe_setmemory() == nullptr) return false; + break; case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: if (StubRoutines::electronicCodeBook_encryptAESCrypt() == nullptr) return false; break; diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 88a395feb6440..b018fcf509713 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -507,6 +507,7 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false); case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate(); case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory(); + case vmIntrinsics::_setMemory: return inline_unsafe_setMemory(); case vmIntrinsics::_getLength: return inline_native_getLength(); case vmIntrinsics::_copyOf: return inline_array_copyOf(false); case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true); @@ -4948,6 +4949,57 @@ bool LibraryCallKit::inline_unsafe_copyMemory() { return true; } +// unsafe_setmemory(void *base, ulong offset, size_t length, char fill_value); +// Fill 'length' bytes starting from 'base[offset]' with 'fill_value' +bool LibraryCallKit::inline_unsafe_setMemory() { + if (callee()->is_static()) return false; // caller must have the capability! + null_check_receiver(); // null-check receiver + if (stopped()) return true; + + C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". + + Node* dst_base = argument(1); // type: oop + Node* dst_off = ConvL2X(argument(2)); // type: long + Node* size = ConvL2X(argument(4)); // type: long + Node* byte = argument(6); // type: byte + + assert(Unsafe_field_offset_to_byte_offset(11) == 11, + "fieldOffset must be byte-scaled"); + + Node* dst_addr = make_unsafe_address(dst_base, dst_off); + + Node* thread = _gvn.transform(new ThreadLocalNode()); + Node* doing_unsafe_access_addr = basic_plus_adr(top(), thread, in_bytes(JavaThread::doing_unsafe_access_offset())); + BasicType doing_unsafe_access_bt = T_BYTE; + assert((sizeof(bool) * CHAR_BIT) == 8, "not implemented"); + + // update volatile field + store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + + int flags = RC_LEAF | RC_NO_FP; + + const TypePtr* dst_type = TypePtr::BOTTOM; + + // Adjust memory effects of the runtime call based on input values. + if (!has_wide_mem(_gvn, dst_addr, dst_base)) { + dst_type = _gvn.type(dst_addr)->is_ptr(); // narrow out memory + + flags |= RC_NARROW_MEM; // narrow in memory + } + + // Call it. Note that the length argument is not scaled. + make_runtime_call(flags, + OptoRuntime::make_setmemory_Type(), + StubRoutines::unsafe_setmemory(), + "unsafe_setmemory", + dst_type, + dst_addr, size XTOP, byte); + + store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); + + return true; +} + #undef XTOP //------------------------clone_coping----------------------------------- diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 47f9d7d47133a..cb4f34a0db6b9 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -231,6 +231,7 @@ class LibraryCallKit : public GraphKit { bool inline_unsafe_writeback0(); bool inline_unsafe_writebackSync0(bool is_pre); bool inline_unsafe_copyMemory(); + bool inline_unsafe_setMemory(); bool inline_native_currentCarrierThread(); bool inline_native_currentThread(); diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index 605c95316e24b..2c0215047857c 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -771,6 +771,29 @@ const TypeFunc* OptoRuntime::void_void_Type() { } +// Takes as parameters: +// void *dest +// long size +// uchar byte +const TypeFunc* OptoRuntime::make_setmemory_Type() { + // create input type (domain) + int argcnt = NOT_LP64(3) LP64_ONLY(4); + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // dest + fields[argp++] = TypeX_X; // size + LP64_ONLY(fields[argp++] = Type::HALF); // size + fields[argp++] = TypeInt::UBYTE; // bytevalue + assert(argp == TypeFunc::Parms+argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields); + + // no result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = nullptr; // void + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); + return TypeFunc::make(domain, range); +} + // arraycopy stub variations: enum ArrayCopyType { ac_fast, // void(ptr, ptr, size_t) diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index b85542423e848..30656044cbb20 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -266,6 +266,8 @@ class OptoRuntime : public AllStatic { static const TypeFunc* generic_arraycopy_Type(); static const TypeFunc* slow_arraycopy_Type(); // the full routine + static const TypeFunc* make_setmemory_Type(); + static const TypeFunc* array_fill_Type(); static const TypeFunc* array_sort_Type(); diff --git a/src/hotspot/share/prims/unsafe.cpp b/src/hotspot/share/prims/unsafe.cpp index dcc47f0f64e95..88672fe16b90e 100644 --- a/src/hotspot/share/prims/unsafe.cpp +++ b/src/hotspot/share/prims/unsafe.cpp @@ -393,7 +393,12 @@ UNSAFE_ENTRY_SCOPED(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject { GuardUnsafeAccess guard(thread); - Copy::fill_to_memory_atomic(p, sz, value); + if (StubRoutines::unsafe_setmemory() != nullptr) { + MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXExec, thread)); + StubRoutines::UnsafeSetMemory_stub()(p, sz, value); + } else { + Copy::fill_to_memory_atomic(p, sz, value); + } } } UNSAFE_END diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 299f1900906d9..802ef22ed6263 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -176,6 +176,7 @@ uint SharedRuntime::_generic_array_copy_ctr=0; uint SharedRuntime::_slow_array_copy_ctr=0; uint SharedRuntime::_find_handler_ctr=0; uint SharedRuntime::_rethrow_ctr=0; +uint SharedRuntime::_unsafe_set_memory_ctr=0; int SharedRuntime::_ICmiss_index = 0; int SharedRuntime::_ICmiss_count[SharedRuntime::maxICmiss_count]; @@ -541,7 +542,6 @@ address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* curr tty->print_cr("b) other problem"); } #endif // PRODUCT - ShouldNotReachHere(); return nullptr; } @@ -1983,6 +1983,7 @@ void SharedRuntime::print_statistics() { if (_slow_array_copy_ctr) tty->print_cr("%5u slow array copies", _slow_array_copy_ctr); if (_find_handler_ctr) tty->print_cr("%5u find exception handler", _find_handler_ctr); if (_rethrow_ctr) tty->print_cr("%5u rethrow handler", _rethrow_ctr); + if (_unsafe_set_memory_ctr) tty->print_cr("%5u unsafe set memorys", _unsafe_set_memory_ctr); AdapterHandlerLibrary::print_statistics(); diff --git a/src/hotspot/share/runtime/sharedRuntime.hpp b/src/hotspot/share/runtime/sharedRuntime.hpp index 93182a10126e9..46a2046565666 100644 --- a/src/hotspot/share/runtime/sharedRuntime.hpp +++ b/src/hotspot/share/runtime/sharedRuntime.hpp @@ -546,6 +546,8 @@ class SharedRuntime: AllStatic { static uint _generic_array_copy_ctr; // Slow-path includes type decoding static uint _slow_array_copy_ctr; // Slow-path failed out to a method call + static uint _unsafe_set_memory_ctr; // Slow-path includes alignment checks + static uint _new_instance_ctr; // 'new' object requires GC static uint _new_array_ctr; // 'new' array requires GC static uint _multi2_ctr, _multi3_ctr, _multi4_ctr, _multi5_ctr; diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 4e5cd7f03895d..550f289bf3cbb 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -110,6 +110,8 @@ address StubRoutines::_checkcast_arraycopy_uninit = nullptr; address StubRoutines::_unsafe_arraycopy = nullptr; address StubRoutines::_generic_arraycopy = nullptr; +address StubRoutines::_unsafe_setmemory = nullptr; + address StubRoutines::_jbyte_fill; address StubRoutines::_jshort_fill; address StubRoutines::_jint_fill; diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index d7adf320131ae..4cb031799e4b9 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -193,6 +193,8 @@ class StubRoutines: AllStatic { static address _unsafe_arraycopy; static address _generic_arraycopy; + static address _unsafe_setmemory; + static address _jbyte_fill; static address _jshort_fill; static address _jint_fill; @@ -384,6 +386,11 @@ class StubRoutines: AllStatic { typedef void (*UnsafeArrayCopyStub)(const void* src, void* dst, size_t count); static UnsafeArrayCopyStub UnsafeArrayCopy_stub() { return CAST_TO_FN_PTR(UnsafeArrayCopyStub, _unsafe_arraycopy); } + static address unsafe_setmemory() { return _unsafe_setmemory; } + + typedef void (*UnsafeSetMemoryStub)(const void* src, size_t count, char byte); + static UnsafeSetMemoryStub UnsafeSetMemory_stub() { return CAST_TO_FN_PTR(UnsafeSetMemoryStub, _unsafe_setmemory); } + static address generic_arraycopy() { return _generic_arraycopy; } static address select_arraysort_function() { return _array_sort; } static address select_array_partition_function() { return _array_partition; } diff --git a/src/hotspot/share/utilities/copy.cpp b/src/hotspot/share/utilities/copy.cpp index ed7797969438b..2eda39fae54d3 100644 --- a/src/hotspot/share/utilities/copy.cpp +++ b/src/hotspot/share/utilities/copy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2006, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -211,42 +211,43 @@ void Copy::conjoint_swap(const void* src, void* dst, size_t byte_count, size_t e // Fill bytes; larger units are filled atomically if everything is aligned. void Copy::fill_to_memory_atomic(void* to, size_t size, jubyte value) { - address dst = (address) to; - uintptr_t bits = (uintptr_t) to | (uintptr_t) size; + address dst = (address)to; + uintptr_t bits = (uintptr_t)to | (uintptr_t)size; if (bits % sizeof(jlong) == 0) { - jlong fill = (julong)( (jubyte)value ); // zero-extend + jlong fill = (julong)((jubyte)value); // zero-extend if (fill != 0) { fill += fill << 8; fill += fill << 16; fill += fill << 32; } - //Copy::fill_to_jlongs_atomic((jlong*) dst, size / sizeof(jlong)); + // Copy::fill_to_jlongs_atomic((jlong*) dst, size / sizeof(jlong)); for (uintptr_t off = 0; off < size; off += sizeof(jlong)) { *(jlong*)(dst + off) = fill; } } else if (bits % sizeof(jint) == 0) { - jint fill = (juint)( (jubyte)value ); // zero-extend + jint fill = (juint)((jubyte)value); // zero-extend if (fill != 0) { fill += fill << 8; fill += fill << 16; } - //Copy::fill_to_jints_atomic((jint*) dst, size / sizeof(jint)); + // Copy::fill_to_jints_atomic((jint*) dst, size / sizeof(jint)); for (uintptr_t off = 0; off < size; off += sizeof(jint)) { *(jint*)(dst + off) = fill; } } else if (bits % sizeof(jshort) == 0) { - jshort fill = (jushort)( (jubyte)value ); // zero-extend + jshort fill = (jushort)((jubyte)value); // zero-extend fill += (jshort)(fill << 8); - //Copy::fill_to_jshorts_atomic((jshort*) dst, size / sizeof(jshort)); + // Copy::fill_to_jshorts_atomic((jshort*) dst, size / sizeof(jshort)); for (uintptr_t off = 0; off < size; off += sizeof(jshort)) { *(jshort*)(dst + off) = fill; } } else { // Not aligned, so no need to be atomic. #ifdef MUSL_LIBC - // This code is used by Unsafe and may hit the next page after truncation of mapped memory. - // Therefore, we use volatile to prevent compilers from replacing the loop by memset which - // may not trigger SIGBUS as needed (observed on Alpine Linux x86_64) + // This code is used by Unsafe and may hit the next page after truncation + // of mapped memory. Therefore, we use volatile to prevent compilers from + // replacing the loop by memset which may not trigger SIGBUS as needed + // (observed on Alpine Linux x86_64) jbyte fill = value; for (uintptr_t off = 0; off < size; off += sizeof(jbyte)) { *(volatile jbyte*)(dst + off) = fill; diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java index eab07313da273..9c81c053e57ca 100644 --- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java @@ -3824,6 +3824,7 @@ private void putShortParts(Object o, long offset, byte i0, byte i1) { private native long allocateMemory0(long bytes); private native long reallocateMemory0(long address, long bytes); private native void freeMemory0(long address); + @IntrinsicCandidate private native void setMemory0(Object o, long offset, long bytes, byte value); @IntrinsicCandidate private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes); diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/MemorySegmentZeroUnsafe.java b/test/micro/org/openjdk/bench/java/lang/foreign/MemorySegmentZeroUnsafe.java new file mode 100644 index 0000000000000..ba705e2ecd4db --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/MemorySegmentZeroUnsafe.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + package org.openjdk.bench.java.lang.foreign; + +import sun.misc.Unsafe; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@State(org.openjdk.jmh.annotations.Scope.Thread) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(value = 3, jvmArgsAppend = {"--enable-native-access=ALL-UNNAMED"}) +public class MemorySegmentZeroUnsafe { + + static final Unsafe UNSAFE = Utils.unsafe; + long src; + + @Param({"1", "2", "3", "4", "5", "6", "7", "8", "15", "16", "63", "64", "255", "256"}) + public int size; + + @Param({"true", "false"}) + public boolean aligned; + + private MemorySegment segment; + private long address; + + @Setup + public void setup() throws Throwable { + Arena arena = Arena.global(); + long alignment = 1; + // this complex logic is to ensure that if in the future we decide to batch writes with different + // batches based on alignment, we would spot it here + if (size == 2 || size == 3) { + alignment = 2; + } else if (size >= 4 && size <= 7) { + alignment = 4; + } else { + alignment = 8; + } + if (aligned) { + segment = arena.allocate(size, alignment); + } else { + // forcibly misaligned in both address AND size, given that would be the worst case + segment = arena.allocate(size + 1, alignment).asSlice(1); + } + address = segment.address(); + } + + @Benchmark + public void panama() { + segment.fill((byte) 0); + } + + @Benchmark + public void unsafe() { + UNSAFE.setMemory(address, size, (byte) 0); + } +}