diff --git a/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp index 1fd59b81d9ee0..a1a5209de7ab2 100644 --- a/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp @@ -200,6 +200,41 @@ inline void FreezeBase::patch_pd_unused(intptr_t* sp) { *fp_addr = badAddressVal; } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + intptr_t* sp = _top_frame.sp(); + if (_top_frame.is_interpreted_frame()) { + // In case the top frame is interpreted we need to set up the anchor using + // the last_sp saved in the frame (remove possible alignment added while + // thawing, see ThawBase::finish_thaw()). We also clear last_sp to match + // the behavior when calling the VM from the interpreter (we check for this + // in FreezeBase::prepare_freeze_interpreted_top_frame, which can be reached + // if preempting again at redo_vmcall()). + _last_sp_from_frame = _top_frame.interpreter_frame_last_sp(); + assert(_last_sp_from_frame != nullptr, ""); + _top_frame.interpreter_frame_set_last_sp(nullptr); + if (sp != _last_sp_from_frame) { + // We need to move up return pc and fp. They will be read next in + // set_anchor() and set as _last_Java_pc and _last_Java_fp respectively. + _last_sp_from_frame[-1] = (intptr_t)_top_frame.pc(); + _last_sp_from_frame[-2] = (intptr_t)_top_frame.fp(); + } + _is_interpreted = true; + sp = _last_sp_from_frame; + } + return sp; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + if (_is_interpreted) { + // Restore last_sp_from_frame and possibly overwritten pc. + _top_frame.interpreter_frame_set_last_sp(_last_sp_from_frame); + intptr_t* sp = _top_frame.sp(); + if (sp != _last_sp_from_frame) { + sp[-1] = (intptr_t)_top_frame.pc(); + } + } +} + //////// Thaw // Fast path @@ -304,10 +339,17 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { frame enterSpecial = new_entry_frame(); intptr_t* sp = enterSpecial.sp(); + // We only need to set the return pc. rfp will be restored back in gen_continuation_enter(). sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); - sp[-2] = (intptr_t)enterSpecial.fp(); + return sp; +} + +inline intptr_t* ThawBase::push_preempt_adapter() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); - log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + // We only need to set the return pc. rfp will be restored back in generate_cont_preempt_stub(). + sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); return sp; } diff --git a/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp index e39580369db49..19a892f5ff856 100644 --- a/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp @@ -52,6 +52,9 @@ static inline void patch_return_pc_with_preempt_stub(frame& f) { // The target will check for preemption once it returns to the interpreter // or the native wrapper code and will manually jump to the preempt stub. JavaThread *thread = JavaThread::current(); + DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();) + assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0, + "preemptable VM call not using call_VM_preemptable"); thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); } } diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index 6f8795494a2bb..cf4d5a63496dd 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -479,6 +479,14 @@ void InterpreterMacroAssembler::remove_activation(TosState state, // result check if synchronized method Label unlocked, unlock, no_unlock; +#ifdef ASSERT + Label not_preempted; + ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); + cbz(rscratch1, not_preempted); + stop("remove_activation: should not have alternate return address set"); + bind(not_preempted); +#endif /* ASSERT */ + // get the value of _do_not_unlock_if_synchronized into r3 const Address do_not_unlock_if_synchronized(rthread, in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); @@ -1371,6 +1379,7 @@ void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point, void InterpreterMacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions) { @@ -1394,26 +1403,34 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, #endif /* ASSERT */ // super call MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp, - entry_point, number_of_arguments, - check_exceptions); + return_pc, entry_point, + number_of_arguments, check_exceptions); // interpreter specific restore_bcp(); restore_locals(); } -void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, - address entry_point, - Register arg_1) { - assert(arg_1 == c_rarg1, ""); +void InterpreterMacroAssembler::call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + assert(InterpreterRuntime::is_preemptable_call(entry_point), "VM call not preemptable, should use call_VM()"); Label resume_pc, not_preempted; #ifdef ASSERT { - Label L; + Label L1, L2; ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); - cbz(rscratch1, L); - stop("Should not have alternate return address set"); - bind(L); + cbz(rscratch1, L1); + stop("call_VM_preemptable_helper: Should not have alternate return address set"); + bind(L1); + // We check this counter in patch_return_pc_with_preempt_stub() during freeze. + incrementw(Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + ldrw(rscratch1, Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + cmpw(rscratch1, 0); + br(Assembler::GT, L2); + stop("call_VM_preemptable_helper: should be > 0"); + bind(L2); } #endif /* ASSERT */ @@ -1421,12 +1438,23 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, push_cont_fastpath(); // Make VM call. In case of preemption set last_pc to the one we want to resume to. - adr(rscratch1, resume_pc); - str(rscratch1, Address(rthread, JavaThread::last_Java_pc_offset())); - call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/); + // Note: call_VM_base will use resume_pc label to set last_Java_pc. + call_VM_base(noreg, noreg, noreg, &resume_pc, entry_point, number_of_arguments, false /*check_exceptions*/); pop_cont_fastpath(); +#ifdef ASSERT + { + Label L; + decrementw(Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + ldrw(rscratch1, Address(rthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + cmpw(rscratch1, 0); + br(Assembler::GE, L); + stop("call_VM_preemptable_helper: should be >= 0"); + bind(L); + } +#endif /* ASSERT */ + // Check if preempted. ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset())); cbz(rscratch1, not_preempted); @@ -1438,6 +1466,51 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, restore_after_resume(false /* is_native */); bind(not_preempted); + if (check_exceptions) { + // check for pending exceptions + ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset()))); + Label ok; + cbz(rscratch1, ok); + lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry())); + br(rscratch1); + bind(ok); + } + + // get oop result if there is one and reset the value in the thread + if (oop_result->is_valid()) { + get_vm_result_oop(oop_result, rthread); + } +} + +static void pass_arg1(MacroAssembler* masm, Register arg) { + if (c_rarg1 != arg ) { + masm->mov(c_rarg1, arg); + } +} + +static void pass_arg2(MacroAssembler* masm, Register arg) { + if (c_rarg2 != arg ) { + masm->mov(c_rarg2, arg); + } +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + bool check_exceptions) { + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 1, check_exceptions); +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions) { + LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 2, check_exceptions); } void InterpreterMacroAssembler::restore_after_resume(bool is_native) { diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp index e07e6e49f535d..2b230a3b73e60 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp @@ -45,6 +45,7 @@ class InterpreterMacroAssembler: public MacroAssembler { virtual void call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions); @@ -58,11 +59,24 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + // Use for vthread preemption void call_VM_preemptable(Register oop_result, address entry_point, - Register arg_1); + Register arg_1, + bool check_exceptions = true); + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions = true); void restore_after_resume(bool is_native); + private: + void call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions); + public: void jump_to_entry(address entry); virtual void check_and_handle_popframe(Register java_thread); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 1400978931986..e6dd29f105ab8 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -743,13 +743,10 @@ static void pass_arg3(MacroAssembler* masm, Register arg) { } } -static bool is_preemptable(address entry_point) { - return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); -} - void MacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions) { @@ -782,12 +779,7 @@ void MacroAssembler::call_VM_base(Register oop_result, assert(last_java_sp != rfp, "can't use rfp"); Label l; - if (is_preemptable(entry_point)) { - // skip setting last_pc since we already set it to desired value. - set_last_Java_frame(last_java_sp, rfp, noreg, rscratch1); - } else { - set_last_Java_frame(last_java_sp, rfp, l, rscratch1); - } + set_last_Java_frame(last_java_sp, rfp, return_pc != nullptr ? *return_pc : l, rscratch1); // do the call, remove parameters MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); @@ -822,7 +814,7 @@ void MacroAssembler::call_VM_base(Register oop_result, } void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { - call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); + call_VM_base(oop_result, noreg, noreg, nullptr, entry_point, number_of_arguments, check_exceptions); } // Check the entry target is always reachable from any branch. @@ -1080,7 +1072,7 @@ void MacroAssembler::call_VM(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { - call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); + call_VM_base(oop_result, rthread, last_java_sp, nullptr, entry_point, number_of_arguments, check_exceptions); } void MacroAssembler::call_VM(Register oop_result, diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index d5a16e424e428..7bfc6c562e3a4 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -81,6 +81,7 @@ class MacroAssembler: public Assembler { Register oop_result, // where an oop-result ends up if any; use noreg otherwise Register java_thread, // the thread if computed before ; use noreg otherwise Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise + Label* return_pc, // to set up last_Java_frame; use nullptr otherwise address entry_point, // the entry point int number_of_arguments, // the number of arguments (w/o thread) to pop after the call bool check_exceptions // whether to check for pending exceptions after return diff --git a/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp index 45e8f2f4202ed..dd00e765eea68 100644 --- a/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp @@ -28,18 +28,17 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; + + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } -private: static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ assert (r == rfp->as_VMReg() || r == rfp->as_VMReg()->next(), "Reg: %s", r->name()); }) public: @@ -71,7 +70,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp index 8a221f1377268..8d2f013e116b5 100644 --- a/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp @@ -108,17 +108,14 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { assert_is_interpreted_and_frame_type_mixed(); ResourceMark rm; - InterpreterOopMap mask; frame f = to_frame(); - f.interpreted_frame_oop_map(&mask); - return mask.num_oops() - + 1 // for the mirror oop - + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot - + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); + InterpreterOopCount closure; + f.oops_interpreted_do(&closure, map); + return closure.count(); } template<> diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index f4774f31bbd42..cde142b39ac6c 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -2304,7 +2304,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ mov(temp, (int) code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_method_entry(Rcache, index); @@ -2356,7 +2356,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ mov(temp, (int) code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_field_entry(Rcache, index); @@ -3700,7 +3700,7 @@ void TemplateTable::_new() { __ bind(slow_case); __ get_constant_pool(c_rarg1); __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1); - call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); + __ call_VM_preemptable(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); __ verify_oop(r0); // continue diff --git a/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp index 389cf4dc936a2..05172ffeba334 100644 --- a/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp +++ b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp @@ -68,6 +68,15 @@ inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { Unimplemented(); } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + Unimplemented(); + return nullptr; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + Unimplemented(); +} + inline frame ThawBase::new_entry_frame() { Unimplemented(); return frame(); @@ -100,6 +109,11 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { return nullptr; } +inline intptr_t* ThawBase::push_preempt_adapter() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp b/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp index 903a71aab5342..1223384d3b781 100644 --- a/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp +++ b/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp @@ -28,18 +28,17 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; + + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } -private: static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ Unimplemented(); }) public: @@ -69,7 +68,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp b/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp index ae97ec60f325c..2da68d6d5ee0d 100644 --- a/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp +++ b/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp @@ -85,7 +85,8 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { Unimplemented(); return 0; } diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp index fa878b07d43bd..e35bdae70d6fa 100644 --- a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp @@ -72,9 +72,16 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { } inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) { - // nothing to do - DEBUG_ONLY( intptr_t* lspp = (intptr_t*) &(f.get_ijava_state()->top_frame_sp); ) - assert(*lspp == f.unextended_sp() - f.fp(), "should be " INTPTR_FORMAT " usp:" INTPTR_FORMAT " fp:" INTPTR_FORMAT, *lspp, p2i(f.unextended_sp()), p2i(f.fp())); + // Nothing to do. We don't save a last sp since we cannot use sp as esp. + // Instead the top frame is trimmed when making an i2i call. The original + // top_frame_sp is set when the frame is pushed (see generate_fixed_frame()). + // An interpreter top frame that was just thawed is resized to top_frame_sp by the + // resume adapter (see generate_cont_resume_interpreter_adapter()). So the assertion is + // false, if we freeze again right after thawing as we do when redoing a vm call wasn't + // successful. + assert(_thread->interp_redoing_vm_call() || + ((intptr_t*)f.at_relative(ijava_idx(top_frame_sp)) == f.unextended_sp()), + "top_frame_sp:" PTR_FORMAT " usp:" PTR_FORMAT, f.at_relative(ijava_idx(top_frame_sp)), p2i(f.unextended_sp())); } inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { @@ -337,6 +344,15 @@ inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { inline void FreezeBase::patch_pd_unused(intptr_t* sp) { } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + // Nothing to do on PPC because the interpreter does not use SP as expression stack pointer. + // Instead there is a dedicated register R15_esp which is not affected by VM calls. + return _top_frame.sp(); +} + +inline void AnchorMark::anchor_mark_clear_pd() { +} + //////// Thaw // Fast path @@ -566,6 +582,19 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { return enterSpecial.sp(); } +inline intptr_t* ThawBase::push_preempt_adapter() { + frame enterSpecial = new_entry_frame(); + frame::common_abi* enterSpecial_abi = (frame::common_abi*)enterSpecial.sp(); + + enterSpecial_abi->lr = (intptr_t)StubRoutines::cont_preempt_stub(); + + log_develop_trace(continuations, preempt)("push_preempt_adapter enterSpecial sp: " INTPTR_FORMAT " adapter pc: " INTPTR_FORMAT, + p2i(enterSpecial_abi), + p2i(StubRoutines::cont_preempt_stub())); + + return enterSpecial.sp(); +} + inline void ThawBase::patch_pd(frame& f, const frame& caller) { patch_callee_link(caller, caller.fp()); // Prevent assertion if f gets deoptimized right away before it's fully initialized diff --git a/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp index d55bf0da3e3c9..7c6d19d442b4e 100644 --- a/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp @@ -37,6 +37,9 @@ static inline void patch_return_pc_with_preempt_stub(frame& f) { // The target will check for preemption once it returns to the interpreter // or the native wrapper code and will manually jump to the preempt stub. JavaThread *thread = JavaThread::current(); + DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();) + assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0, + "preemptable VM call not using call_VM_preemptable"); thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); } } diff --git a/src/hotspot/cpu/ppc/frame_ppc.hpp b/src/hotspot/cpu/ppc/frame_ppc.hpp index 188015f5cd930..ebe5d24c0720a 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.hpp @@ -215,7 +215,7 @@ uint64_t bcp; uint64_t esp; uint64_t mdx; - uint64_t top_frame_sp; // Maybe define parent_frame_abi and move there. + uint64_t top_frame_sp; // Original sp to be restored when returning from an i2i call uint64_t sender_sp; // Slots only needed for native calls. Maybe better to move elsewhere. uint64_t oop_tmp; diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp index 9140dd7ca4edf..4ea33ebaf6347 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp @@ -49,12 +49,14 @@ class InterpreterMacroAssembler: public MacroAssembler { virtual void check_and_handle_popframe(Register scratch_reg); virtual void check_and_handle_earlyret(Register scratch_reg); + // Use for vthread preemption void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true); + void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true); void restore_after_resume(Register fp); // R22 and R31 are preserved when a vthread gets preempted in the interpreter. // The interpreter already assumes that these registers are nonvolatile across native calls. bool nonvolatile_accross_vthread_preemtion(Register r) const { - return r->is_nonvolatile() && ((r == R22) || (r == R31)); + return r->is_nonvolatile() && ((r == R24) || (r == R31)); } // Base routine for all dispatches. diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp index 503cc25943253..0d32ea8003e8c 100644 --- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp @@ -108,6 +108,8 @@ void InterpreterMacroAssembler::dispatch_prolog(TosState state, int bcp_incr) { // own dispatch. The dispatch address in R24_dispatch_addr is used for the // dispatch. void InterpreterMacroAssembler::dispatch_epilog(TosState state, int bcp_incr) { + assert(nonvolatile_accross_vthread_preemtion(R24_dispatch_addr), + "Requirement of field accesses (e.g. putstatic)"); if (bcp_incr) { addi(R14_bcp, R14_bcp, bcp_incr); } mtctr(R24_dispatch_addr); bcctr(bcondAlways, 0, bhintbhBCCTRisNotPredictable); @@ -862,6 +864,9 @@ void InterpreterMacroAssembler::remove_activation(TosState state, bool install_monitor_exception) { BLOCK_COMMENT("remove_activation {"); + asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread, + "remove_activation: should not have alternate return address set"); + unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception); // The below poll is for the stack watermark barrier. It allows fixing up frames lazily, @@ -2014,35 +2019,67 @@ void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point } void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point, - Register arg_1, bool check_exceptions) { + Register arg_1, + bool check_exceptions) { if (!Continuations::enabled()) { call_VM(oop_result, entry_point, arg_1, check_exceptions); return; } + call_VM_preemptable(oop_result, entry_point, arg_1, noreg /* arg_2 */, check_exceptions); +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point, + Register arg_1, Register arg_2, + bool check_exceptions) { + if (!Continuations::enabled()) { + call_VM(oop_result, entry_point, arg_1, arg_2, check_exceptions); + return; + } Label resume_pc, not_preempted; + Register tmp = R11_scratch1; + assert_different_registers(arg_1, tmp); + assert_different_registers(arg_2, tmp); - DEBUG_ONLY(ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread)); - DEBUG_ONLY(cmpdi(CR0, R0, 0)); - asm_assert_eq("Should not have alternate return address set"); +#ifdef ASSERT + asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread, + "Should not have alternate return address set"); + // We check this counter in patch_return_pc_with_preempt_stub() during freeze. + lwa(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread); + addi(tmp, tmp, 1); + cmpwi(CR0, tmp, 0); + stw(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread); + asm_assert(gt, "call_VM_preemptable: should be > 0"); +#endif // ASSERT // Preserve 2 registers - assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), ""); + assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R24), ""); ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP std(R31, _ijava_state_neg(lresult), R3_ARG1); - std(R22, _ijava_state_neg(fresult), R3_ARG1); + std(R24, _ijava_state_neg(fresult), R3_ARG1); // We set resume_pc as last java pc. It will be saved if the vthread gets preempted. // Later execution will continue right there. mr_if_needed(R4_ARG2, arg_1); + assert(arg_2 != R4_ARG2, "smashed argument"); + mr_if_needed(R5_ARG3, arg_2, true /* allow_noreg */); push_cont_fastpath(); - call_VM(oop_result, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */); + call_VM(noreg /* oop_result */, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */); pop_cont_fastpath(); +#ifdef ASSERT + lwa(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread); + addi(tmp, tmp, -1); + cmpwi(CR0, tmp, 0); + stw(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread); + asm_assert(ge, "call_VM_preemptable: should be >= 0"); +#endif // ASSERT + // Jump to handler if the call was preempted ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); cmpdi(CR0, R0, 0); beq(CR0, not_preempted); + // Preempted. Frames are already frozen on heap. mtlr(R0); li(R0, 0); std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread); @@ -2050,21 +2087,21 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address bind(resume_pc); // Location to resume execution restore_after_resume(noreg /* fp */); + bind(not_preempted); + if (check_exceptions) { + check_and_forward_exception(R11_scratch1, R12_scratch2); + } + if (oop_result->is_valid()) { + get_vm_result_oop(oop_result); + } } void InterpreterMacroAssembler::restore_after_resume(Register fp) { - if (!Continuations::enabled()) return; - const address resume_adapter = TemplateInterpreter::cont_resume_interpreter_adapter(); add_const_optimized(R31, R29_TOC, MacroAssembler::offset_to_global_toc(resume_adapter)); mtctr(R31); bctrl(); - // Restore registers that are preserved across vthread preemption - assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), ""); - ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP - ld(R31, _ijava_state_neg(lresult), R3_ARG1); - ld(R22, _ijava_state_neg(fresult), R3_ARG1); #ifdef ASSERT // Assert FP is in R11_scratch1 (see generate_cont_resume_interpreter_adapter()) { diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 00a46504e145f..5511e88911990 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -757,10 +757,11 @@ void MacroAssembler::clobber_nonvolatile_registers() { R31 }; Register bad = regs[0]; - load_const_optimized(bad, 0xbad0101babe11111); + load_const_optimized(bad, 0xbad0101babe00000); for (uint32_t i = 1; i < (sizeof(regs) / sizeof(Register)); i++) { - mr(regs[i], bad); + addi(regs[i], regs[0], regs[i]->encoding()); } + addi(regs[0], regs[0], regs[0]->encoding()); BLOCK_COMMENT("} clobber nonvolatile registers"); } #endif // ASSERT @@ -4341,21 +4342,36 @@ void MacroAssembler::multiply_to_len(Register x, Register xlen, bind(L_done); } // multiply_to_len -void MacroAssembler::asm_assert(bool check_equal, const char *msg) { #ifdef ASSERT +void MacroAssembler::asm_assert(AsmAssertCond cond, const char *msg) { Label ok; - if (check_equal) { + switch (cond) { + case eq: beq(CR0, ok); - } else { + break; + case ne: bne(CR0, ok); + break; + case ge: + bge(CR0, ok); + break; + case gt: + bgt(CR0, ok); + break; + case lt: + blt(CR0, ok); + break; + case le: + ble(CR0, ok); + break; + default: + assert(false, "unknown cond:%d", cond); } stop(msg); bind(ok); -#endif } -#ifdef ASSERT -void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset, +void MacroAssembler::asm_assert_mems_zero(AsmAssertCond cond, int size, int mem_offset, Register mem_base, const char* msg) { switch (size) { case 4: @@ -4369,7 +4385,7 @@ void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_of default: ShouldNotReachHere(); } - asm_assert(check_equal, msg); + asm_assert(cond, msg); } #endif // ASSERT diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp index 63be608094fe1..61e6a173823ad 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp @@ -68,7 +68,7 @@ class MacroAssembler: public Assembler { void store_sized_value(Register dst, RegisterOrConstant offs, Register base, size_t size_in_bytes); // Move register if destination register and target register are different - inline void mr_if_needed(Register rd, Register rs); + inline void mr_if_needed(Register rd, Register rs, bool allow_invalid = false); inline void fmr_if_needed(FloatRegister rd, FloatRegister rs); // This is dedicated for emitting scheduled mach nodes. For better // readability of the ad file I put it here. @@ -942,21 +942,29 @@ class MacroAssembler: public Assembler { // // assert on cr0 - void asm_assert(bool check_equal, const char* msg); - void asm_assert_eq(const char* msg) { asm_assert(true, msg); } - void asm_assert_ne(const char* msg) { asm_assert(false, msg); } + enum AsmAssertCond { + eq, + ne, + ge, + gt, + lt, + le + }; + void asm_assert(AsmAssertCond cond, const char* msg) PRODUCT_RETURN; + void asm_assert_eq(const char* msg) { asm_assert(eq, msg); } + void asm_assert_ne(const char* msg) { asm_assert(ne, msg); } private: - void asm_assert_mems_zero(bool check_equal, int size, int mem_offset, Register mem_base, + void asm_assert_mems_zero(AsmAssertCond cond, int size, int mem_offset, Register mem_base, const char* msg) NOT_DEBUG_RETURN; public: void asm_assert_mem8_is_zero(int mem_offset, Register mem_base, const char* msg) { - asm_assert_mems_zero(true, 8, mem_offset, mem_base, msg); + asm_assert_mems_zero(eq, 8, mem_offset, mem_base, msg); } void asm_assert_mem8_isnot_zero(int mem_offset, Register mem_base, const char* msg) { - asm_assert_mems_zero(false, 8, mem_offset, mem_base, msg); + asm_assert_mems_zero(ne, 8, mem_offset, mem_base, msg); } // Calls verify_oop. If UseCompressedOops is on, decodes the oop. diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp index d27011112e2ba..2b19d84c69c64 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp @@ -65,7 +65,8 @@ inline void MacroAssembler::round_to(Register r, int modulus) { } // Move register if destination register and target register are different. -inline void MacroAssembler::mr_if_needed(Register rd, Register rs) { +inline void MacroAssembler::mr_if_needed(Register rd, Register rs, bool allow_noreg) { + if (allow_noreg && (rs == noreg)) return; if (rs != rd) mr(rd, rs); } inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) { diff --git a/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp b/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp index fd352a5371664..221a3c0134148 100644 --- a/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp @@ -28,18 +28,18 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers, so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); + public: // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap // Consider enhancing SmallRegisterMap to support those cases const RegisterMap* as_RegisterMap() const { return nullptr; } @@ -61,14 +61,14 @@ class SmallRegisterMap { JavaThread* thread() const { #ifndef ASSERT - guarantee (false, ""); + guarantee (false, "unreachable"); #endif return nullptr; } bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } @@ -76,7 +76,7 @@ class SmallRegisterMap { #ifdef ASSERT bool should_skip_missing() const { return false; } VMReg find_register_spilled_here(void* p, intptr_t* sp) { - Unimplemented(); + assert(false, "Shouldn't reach here! p:" PTR_FORMAT " sp:" PTR_FORMAT, p2i(p), p2i(p)); return nullptr; } void print() const { print_on(tty); } diff --git a/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp index 07f1c9c1c6f16..a14ec9303f621 100644 --- a/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp @@ -176,17 +176,14 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { assert_is_interpreted_and_frame_type_mixed(); ResourceMark rm; - InterpreterOopMap mask; frame f = to_frame(); - f.interpreted_frame_oop_map(&mask); - return mask.num_oops() - + 1 // for the mirror oop - + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot - + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); + InterpreterOopCount closure; + f.oops_interpreted_do(&closure, map); + return closure.count(); } template<> diff --git a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp index 199b578a36f23..3fe7d35396235 100644 --- a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp @@ -702,6 +702,11 @@ address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter() __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R12_scratch2); __ restore_interpreter_state(R11_scratch1, false, true /*restore_top_frame_sp*/); + // Restore registers that are preserved across vthread preemption + assert(__ nonvolatile_accross_vthread_preemtion(R31) && __ nonvolatile_accross_vthread_preemtion(R24), ""); + __ ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP + __ ld(R31, _ijava_state_neg(lresult), R3_ARG1); + __ ld(R24, _ijava_state_neg(fresult), R3_ARG1); __ blr(); return start; @@ -1249,7 +1254,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { const Register pending_exception = R0; const Register result_handler_addr = R31; const Register native_method_fd = R12_scratch2; // preferred in MacroAssembler::branch_to - const Register access_flags = R22_tmp2; + const Register access_flags = R24_tmp4; const Register active_handles = R11_scratch1; // R26_monitor saved to state. const Register sync_state = R12_scratch2; const Register sync_state_addr = sync_state; // Address is dead after use. diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp index 09acd1c067da9..8d61ba1b2d7d9 100644 --- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp +++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp @@ -2214,7 +2214,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca __ bind(L_clinit_barrier_slow); address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ li(R4_ARG2, code); - __ call_VM(noreg, entry, R4_ARG2); + __ call_VM_preemptable(noreg, entry, R4_ARG2); // Update registers with resolved info. __ load_method_entry(Rcache, Rindex); @@ -2262,7 +2262,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, Register Rcac __ bind(L_clinit_barrier_slow); address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ li(R4_ARG2, code); - __ call_VM(noreg, entry, R4_ARG2); + __ call_VM_preemptable(noreg, entry, R4_ARG2); // Update registers with resolved info __ load_field_entry(Rcache, index); @@ -3864,7 +3864,7 @@ void TemplateTable::_new() { // -------------------------------------------------------------------------- // slow case __ bind(Lslow_case); - call_VM(R17_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Rcpool, Rindex); + __ call_VM_preemptable(R17_tos, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), Rcpool, Rindex); // continue __ bind(Ldone); diff --git a/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp index 461dc19f38341..d6586617a63e9 100644 --- a/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp @@ -207,6 +207,41 @@ inline void ThawBase::prefetch_chunk_pd(void* start, int size) { Prefetch::read(start, size - 64); } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + intptr_t* sp = _top_frame.sp(); + if (_top_frame.is_interpreted_frame()) { + // In case the top frame is interpreted we need to set up the anchor using + // the last_sp saved in the frame (remove possible alignment added while + // thawing, see ThawBase::finish_thaw()). We also clear last_sp to match + // the behavior when calling the VM from the interpreter (we check for this + // in FreezeBase::prepare_freeze_interpreted_top_frame, which can be reached + // if preempting again at redo_vmcall()). + _last_sp_from_frame = _top_frame.interpreter_frame_last_sp(); + assert(_last_sp_from_frame != nullptr, ""); + _top_frame.interpreter_frame_set_last_sp(nullptr); + if (sp != _last_sp_from_frame) { + // We need to move up return pc and fp. They will be read next in + // set_anchor() and set as _last_Java_pc and _last_Java_fp respectively. + _last_sp_from_frame[-1] = (intptr_t)_top_frame.pc(); + _last_sp_from_frame[-2] = (intptr_t)_top_frame.fp(); + } + _is_interpreted = true; + sp = _last_sp_from_frame; + } + return sp; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + if (_is_interpreted) { + // Restore last_sp_from_frame and possibly overwritten pc. + _top_frame.interpreter_frame_set_last_sp(_last_sp_from_frame); + intptr_t* sp = _top_frame.sp(); + if (sp != _last_sp_from_frame) { + sp[-1] = (intptr_t)_top_frame.pc(); + } + } +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { // Fast path depends on !PreserveFramePointer. See can_thaw_fast(). @@ -305,10 +340,17 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { frame enterSpecial = new_entry_frame(); intptr_t* sp = enterSpecial.sp(); + // We only need to set the return pc. fp will be restored back in gen_continuation_enter(). sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); - sp[-2] = (intptr_t)enterSpecial.fp(); + return sp; +} + +inline intptr_t* ThawBase::push_preempt_adapter() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); - log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + // We only need to set the return pc. fp will be restored back in generate_cont_preempt_stub(). + sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); return sp; } diff --git a/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp index 424d56edf5a6c..918bf3db8cc41 100644 --- a/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp @@ -52,6 +52,9 @@ static inline void patch_return_pc_with_preempt_stub(frame& f) { // The target will check for preemption once it returns to the interpreter // or the native wrapper code and will manually jump to the preempt stub. JavaThread *thread = JavaThread::current(); + DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();) + assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0, + "preemptable VM call not using call_VM_preemptable"); thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); } } diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp index 549c9cda7b611..5af8ea1da37bb 100644 --- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp @@ -518,6 +518,14 @@ void InterpreterMacroAssembler::remove_activation(TosState state, // result check if synchronized method Label unlocked, unlock, no_unlock; +#ifdef ASSERT + Label not_preempted; + ld(t0, Address(xthread, JavaThread::preempt_alternate_return_offset())); + beqz(t0, not_preempted); + stop("remove_activation: should not have alternate return address set"); + bind(not_preempted); +#endif /* ASSERT */ + // get the value of _do_not_unlock_if_synchronized into x13 const Address do_not_unlock_if_synchronized(xthread, in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); @@ -1441,6 +1449,7 @@ void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point, void InterpreterMacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions) { @@ -1463,26 +1472,34 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, #endif /* ASSERT */ // super call MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp, - entry_point, number_of_arguments, - check_exceptions); -// interpreter specific + return_pc, entry_point, + number_of_arguments, check_exceptions); + // interpreter specific restore_bcp(); restore_locals(); } -void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, - address entry_point, - Register arg_1) { - assert(arg_1 == c_rarg1, ""); +void InterpreterMacroAssembler::call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + assert(InterpreterRuntime::is_preemptable_call(entry_point), + "VM call not preemptable, should use call_VM()"); Label resume_pc, not_preempted; #ifdef ASSERT { - Label L; + Label L1, L2; ld(t0, Address(xthread, JavaThread::preempt_alternate_return_offset())); - beqz(t0, L); - stop("Should not have alternate return address set"); - bind(L); + beqz(t0, L1); + stop("call_VM_preemptable_helper: Should not have alternate return address set"); + bind(L1); + // We check this counter in patch_return_pc_with_preempt_stub() during freeze. + incrementw(Address(xthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + lw(t0, Address(xthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + bgtz(t0, L2); + stop("call_VM_preemptable_helper: should be > 0"); + bind(L2); } #endif /* ASSERT */ @@ -1490,12 +1507,22 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, push_cont_fastpath(); // Make VM call. In case of preemption set last_pc to the one we want to resume to. - la(t0, resume_pc); - sd(t0, Address(xthread, JavaThread::last_Java_pc_offset())); - call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/); + // Note: call_VM_base will use resume_pc label to set last_Java_pc. + call_VM_base(noreg, noreg, noreg, &resume_pc, entry_point, number_of_arguments, false /*check_exceptions*/); pop_cont_fastpath(); +#ifdef ASSERT + { + Label L; + decrementw(Address(xthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + lw(t0, Address(xthread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + bgez(t0, L); + stop("call_VM_preemptable_helper: should be >= 0"); + bind(L); + } +#endif /* ASSERT */ + // Check if preempted. ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset())); beqz(t1, not_preempted); @@ -1507,6 +1534,51 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, restore_after_resume(false /* is_native */); bind(not_preempted); + if (check_exceptions) { + // check for pending exceptions + ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + Label ok; + beqz(t0, ok); + la(t1, RuntimeAddress(StubRoutines::forward_exception_entry())); + jr(t1); + bind(ok); + } + + // get oop result if there is one and reset the value in the thread + if (oop_result->is_valid()) { + get_vm_result_oop(oop_result, xthread); + } +} + +static void pass_arg1(MacroAssembler* masm, Register arg) { + if (c_rarg1 != arg) { + masm->mv(c_rarg1, arg); + } +} + +static void pass_arg2(MacroAssembler* masm, Register arg) { + if (c_rarg2 != arg) { + masm->mv(c_rarg2, arg); + } +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + bool check_exceptions) { + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 1, check_exceptions); +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions) { + LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 2, check_exceptions); } void InterpreterMacroAssembler::restore_after_resume(bool is_native) { diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp index 295f1b221916c..89b2ed17291fc 100644 --- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp @@ -46,6 +46,7 @@ class InterpreterMacroAssembler: public MacroAssembler { virtual void call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions); @@ -59,11 +60,27 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + // Use for vthread preemption void call_VM_preemptable(Register oop_result, address entry_point, - Register arg_1); + Register arg_1, + bool check_exceptions = true); + + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions = true); + void restore_after_resume(bool is_native); + private: + void call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions); + + public: void jump_to_entry(address entry); virtual void check_and_handle_popframe(Register java_thread); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 700e42e6194ce..54304ec648d24 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -233,7 +233,7 @@ int MacroAssembler::align(int modulus, int extra_offset) { } void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { - call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); + call_VM_base(oop_result, noreg, noreg, nullptr, entry_point, number_of_arguments, check_exceptions); } // Implementation of call_VM versions @@ -284,7 +284,7 @@ void MacroAssembler::call_VM(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { - call_VM_base(oop_result, xthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); + call_VM_base(oop_result, xthread, last_java_sp, nullptr, entry_point, number_of_arguments, check_exceptions); } void MacroAssembler::call_VM(Register oop_result, @@ -409,13 +409,10 @@ void MacroAssembler::reset_last_Java_frame(bool clear_fp) { sd(zr, Address(xthread, JavaThread::last_Java_pc_offset())); } -static bool is_preemptable(address entry_point) { - return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); -} - void MacroAssembler::call_VM_base(Register oop_result, Register java_thread, Register last_java_sp, + Label* return_pc, address entry_point, int number_of_arguments, bool check_exceptions) { @@ -423,6 +420,7 @@ void MacroAssembler::call_VM_base(Register oop_result, if (!java_thread->is_valid()) { java_thread = xthread; } + // determine last_java_sp register if (!last_java_sp->is_valid()) { last_java_sp = esp; @@ -442,12 +440,7 @@ void MacroAssembler::call_VM_base(Register oop_result, assert(last_java_sp != fp, "can't use fp"); Label l; - if (is_preemptable(entry_point)) { - // skip setting last_pc since we already set it to desired value. - set_last_Java_frame(last_java_sp, fp, noreg); - } else { - set_last_Java_frame(last_java_sp, fp, l, t0); - } + set_last_Java_frame(last_java_sp, fp, return_pc != nullptr ? *return_pc : l, t0); // do the call, remove parameters MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index 9e713e9027022..e0e610ff49a5e 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -168,6 +168,7 @@ class MacroAssembler: public Assembler { Register oop_result, // where an oop-result ends up if any; use noreg otherwise Register java_thread, // the thread if computed before ; use noreg otherwise Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise + Label* return_pc, // to set up last_Java_frame; use nullptr otherwise address entry_point, // the entry point int number_of_arguments, // the number of arguments (w/o thread) to pop after the call bool check_exceptions // whether to check for pending exceptions after return diff --git a/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp b/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp index 541cb0cbcb54c..ea5aef14ae1e7 100644 --- a/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp @@ -28,20 +28,20 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers (except for fp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; + + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } -private: static void assert_is_fp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ assert (r == fp->as_VMReg() || r == fp->as_VMReg()->next(), "Reg: %s", r->name()); }) + public: // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap // Consider enhancing SmallRegisterMap to support those cases @@ -71,7 +71,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp index fa8a8fb47f022..9e2b8003dc472 100644 --- a/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp @@ -106,17 +106,14 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { assert_is_interpreted_and_frame_type_mixed(); ResourceMark rm; - InterpreterOopMap mask; frame f = to_frame(); - f.interpreted_frame_oop_map(&mask); - return mask.num_oops() - + 1 // for the mirror oop - + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot - + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), - (intptr_t*)f.interpreter_frame_monitor_end()) / BasicObjectLock::size(); + InterpreterOopCount closure; + f.oops_interpreted_do(&closure, map); + return closure.count(); } template<> diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp index bd4a89d819967..ca41583e4bcdf 100644 --- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp +++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp @@ -2206,7 +2206,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ mv(temp, (int) code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_method_entry(Rcache, index); @@ -2259,7 +2259,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ mv(temp, (int) code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_field_entry(Rcache, index); @@ -3612,7 +3612,7 @@ void TemplateTable::_new() { __ bind(slow_case); __ get_constant_pool(c_rarg1); __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1); - call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); + __ call_VM_preemptable(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); __ verify_oop(x10); // continue diff --git a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp index d39821bd03422..33ac17effa7c1 100644 --- a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp +++ b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp @@ -68,6 +68,15 @@ inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { Unimplemented(); } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + Unimplemented(); + return nullptr; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + Unimplemented(); +} + inline frame ThawBase::new_entry_frame() { Unimplemented(); return frame(); @@ -100,6 +109,11 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { return nullptr; } +inline intptr_t* ThawBase::push_preempt_adapter() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp index 3a0afb2726007..43363ccd93390 100644 --- a/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp +++ b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp @@ -28,18 +28,17 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" -// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +class SmallRegisterMap; + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller SmallRegisterMapType +template +class SmallRegisterMapType { + friend SmallRegisterMap; + + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } -private: static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ Unimplemented(); }) public: @@ -69,7 +68,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp index d94dea33e5555..4fff0846a8044 100644 --- a/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp +++ b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp @@ -85,7 +85,8 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { Unimplemented(); return 0; } diff --git a/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp index 126f4043cadc9..7691a84a9fe83 100644 --- a/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp +++ b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp @@ -191,6 +191,41 @@ inline void FreezeBase::patch_pd_unused(intptr_t* sp) { *fp_addr = badAddressVal; } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + intptr_t* sp = _top_frame.sp(); + if (_top_frame.is_interpreted_frame()) { + // In case the top frame is interpreted we need to set up the anchor using + // the last_sp saved in the frame (remove possible alignment added while + // thawing, see ThawBase::finish_thaw()). We also clear last_sp to match + // the behavior when calling the VM from the interpreter (we check for this + // in FreezeBase::prepare_freeze_interpreted_top_frame, which can be reached + // if preempting again at redo_vmcall()). + _last_sp_from_frame = _top_frame.interpreter_frame_last_sp(); + assert(_last_sp_from_frame != nullptr, ""); + _top_frame.interpreter_frame_set_last_sp(nullptr); + if (sp != _last_sp_from_frame) { + // We need to move up return pc and fp. They will be read next in + // set_anchor() and set as _last_Java_pc and _last_Java_fp respectively. + _last_sp_from_frame[-1] = (intptr_t)_top_frame.pc(); + _last_sp_from_frame[-2] = (intptr_t)_top_frame.fp(); + } + _is_interpreted = true; + sp = _last_sp_from_frame; + } + return sp; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + if (_is_interpreted) { + // Restore last_sp_from_frame and possibly overwritten pc. + _top_frame.interpreter_frame_set_last_sp(_last_sp_from_frame); + intptr_t* sp = _top_frame.sp(); + if (sp != _last_sp_from_frame) { + sp[-1] = (intptr_t)_top_frame.pc(); + } + } +} + //////// Thaw // Fast path @@ -291,10 +326,17 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { frame enterSpecial = new_entry_frame(); intptr_t* sp = enterSpecial.sp(); + // We only need to set the return pc. rbp will be restored back in gen_continuation_enter(). sp[-1] = (intptr_t)ContinuationEntry::cleanup_pc(); - sp[-2] = (intptr_t)enterSpecial.fp(); + return sp; +} + +inline intptr_t* ThawBase::push_preempt_adapter() { + frame enterSpecial = new_entry_frame(); + intptr_t* sp = enterSpecial.sp(); - log_develop_trace(continuations, preempt)("push_cleanup_continuation initial sp: " INTPTR_FORMAT " final sp: " INTPTR_FORMAT, p2i(sp + 2 * frame::metadata_words), p2i(sp)); + // We only need to set the return pc. rbp will be restored back in generate_cont_preempt_stub(). + sp[-1] = (intptr_t)StubRoutines::cont_preempt_stub(); return sp; } diff --git a/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp index 6d72e1b80e893..bb32e55db75c4 100644 --- a/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp +++ b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp @@ -50,6 +50,9 @@ static inline void patch_return_pc_with_preempt_stub(frame& f) { // The target will check for preemption once it returns to the interpreter // or the native wrapper code and will manually jump to the preempt stub. JavaThread *thread = JavaThread::current(); + DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();) + assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0, + "preemptable VM call not using call_VM_preemptable"); thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub()); } } diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index a6b4efbe4f2e3..6cfa33e5a0d65 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -326,19 +326,26 @@ void InterpreterMacroAssembler::call_VM_base(Register oop_result, restore_locals(); } -void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, - address entry_point, - Register arg_1) { - assert(arg_1 == c_rarg1, ""); +void InterpreterMacroAssembler::call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + assert(InterpreterRuntime::is_preemptable_call(entry_point), "VM call not preemptable, should use call_VM()"); Label resume_pc, not_preempted; #ifdef ASSERT { - Label L; + Label L1, L2; cmpptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); - jcc(Assembler::equal, L); - stop("Should not have alternate return address set"); - bind(L); + jcc(Assembler::equal, L1); + stop("call_VM_preemptable_helper: should not have alternate return address set"); + bind(L1); + // We check this counter in patch_return_pc_with_preempt_stub() during freeze. + incrementl(Address(r15_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + cmpl(Address(r15_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()), 0); + jcc(Assembler::greater, L2); + stop("call_VM_preemptable_helper: should be > 0"); + bind(L2); } #endif /* ASSERT */ @@ -346,14 +353,25 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, push_cont_fastpath(); // Make VM call. In case of preemption set last_pc to the one we want to resume to. - // Note: call_VM_helper requires last_Java_pc for anchor to be at the top of the stack. lea(rscratch1, resume_pc); push(rscratch1); - MacroAssembler::call_VM_helper(oop_result, entry_point, 1, false /*check_exceptions*/); + lea(rax, Address(rsp, wordSize)); + call_VM_base(noreg, rax, entry_point, number_of_arguments, false); pop(rscratch1); pop_cont_fastpath(); +#ifdef ASSERT + { + Label L; + decrementl(Address(r15_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset())); + cmpl(Address(r15_thread, JavaThread::interp_at_preemptable_vmcall_cnt_offset()), 0); + jcc(Assembler::greaterEqual, L); + stop("call_VM_preemptable_helper: should be >= 0"); + bind(L); + } +#endif /* ASSERT */ + // Check if preempted. movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset())); cmpptr(rscratch1, NULL_WORD); @@ -366,6 +384,54 @@ void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, restore_after_resume(false /* is_native */); bind(not_preempted); + if (check_exceptions) { + // check for pending exceptions + cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); + Label ok; + jcc(Assembler::equal, ok); + // Exception stub expects return pc to be at top of stack. We only need + // it to check Interpreter::contains(return_address) so anything will do. + lea(rscratch1, resume_pc); + push(rscratch1); + jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + bind(ok); + } + + // get oop result if there is one and reset the value in the thread + if (oop_result->is_valid()) { + get_vm_result_oop(oop_result); + } +} + +static void pass_arg1(MacroAssembler* masm, Register arg) { + if (c_rarg1 != arg ) { + masm->mov(c_rarg1, arg); + } +} + +static void pass_arg2(MacroAssembler* masm, Register arg) { + if (c_rarg2 != arg ) { + masm->mov(c_rarg2, arg); + } +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + bool check_exceptions) { + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 1, check_exceptions); +} + +void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions) { + LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM_preemptable_helper(oop_result, entry_point, 2, check_exceptions); } void InterpreterMacroAssembler::restore_after_resume(bool is_native) { @@ -800,6 +866,14 @@ void InterpreterMacroAssembler::remove_activation(TosState state, // result check if synchronized method Label unlocked, unlock, no_unlock; +#ifdef ASSERT + Label not_preempted; + cmpptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD); + jcc(Assembler::equal, not_preempted); + stop("remove_activation: should not have alternate return address set"); + bind(not_preempted); +#endif /* ASSERT */ + const Register rthread = r15_thread; const Register robj = c_rarg1; const Register rmon = c_rarg1; diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index a36a697eebf5f..1d2080cd542bc 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -62,11 +62,24 @@ class InterpreterMacroAssembler: public MacroAssembler { void load_earlyret_value(TosState state); + // Use for vthread preemption void call_VM_preemptable(Register oop_result, address entry_point, - Register arg_1); + Register arg_1, + bool check_exceptions = true); + void call_VM_preemptable(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions = true); void restore_after_resume(bool is_native); + private: + void call_VM_preemptable_helper(Register oop_result, + address entry_point, + int number_of_arguments, + bool check_exceptions); + public: // Interpreter-specific registers void save_bcp() { movptr(Address(rbp, frame::interpreter_frame_bcp_offset * wordSize), _bcp_register); diff --git a/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp b/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp index 930a8af07941f..78b604c16fc1d 100644 --- a/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp +++ b/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp @@ -28,19 +28,17 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers (except for rbp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -private: static void assert_is_rbp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ assert(r == rbp->as_VMReg() || r == rbp->as_VMReg()->next(), "Reg: %s", r->name()); }) public: @@ -72,7 +70,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp index 6289b903ab1e4..ebdae8a7eac62 100644 --- a/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp +++ b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp @@ -106,17 +106,14 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { assert_is_interpreted_and_frame_type_mixed(); ResourceMark rm; - InterpreterOopMap mask; frame f = to_frame(); - f.interpreted_frame_oop_map(&mask); - return mask.num_oops() - + 1 // for the mirror oop - + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot - + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(), - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); + InterpreterOopCount closure; + f.oops_interpreted_do(&closure, map); + return closure.count(); } template<> diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index c8dafb2d023e0..7065f653e1982 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -2233,7 +2233,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ movl(temp, code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_method_entry(cache, index); __ bind(L_done); @@ -2280,7 +2280,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no, // Class initialization barrier slow path lands here as well. address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); __ movl(temp, code); - __ call_VM(noreg, entry, temp); + __ call_VM_preemptable(noreg, entry, temp); // Update registers with resolved info __ load_field_entry(cache, index); __ bind(L_done); @@ -3644,8 +3644,8 @@ void TemplateTable::_new() { __ get_constant_pool(c_rarg1); __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1); - call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); - __ verify_oop(rax); + __ call_VM_preemptable(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); + __ verify_oop(rax); // continue __ bind(done); diff --git a/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp index eb6be8772f962..22f596b161ac7 100644 --- a/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp +++ b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp @@ -68,6 +68,15 @@ inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { Unimplemented(); } +inline intptr_t* AnchorMark::anchor_mark_set_pd() { + Unimplemented(); + return nullptr; +} + +inline void AnchorMark::anchor_mark_clear_pd() { + Unimplemented(); +} + inline frame ThawBase::new_entry_frame() { Unimplemented(); return frame(); @@ -100,6 +109,11 @@ inline intptr_t* ThawBase::push_cleanup_continuation() { return nullptr; } +inline intptr_t* ThawBase::push_preempt_adapter() { + Unimplemented(); + return nullptr; +} + template inline void Thaw::patch_caller_links(intptr_t* sp, intptr_t* bottom) { Unimplemented(); diff --git a/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp b/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp index bbb70fa20657c..906575ab669ac 100644 --- a/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp +++ b/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp @@ -28,18 +28,17 @@ #include "runtime/frame.inline.hpp" #include "runtime/registerMap.hpp" +class SmallRegisterMap; + // Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap -class SmallRegisterMap { - constexpr SmallRegisterMap() = default; - ~SmallRegisterMap() = default; - NONCOPYABLE(SmallRegisterMap); +template +class SmallRegisterMapType { + friend SmallRegisterMap; + + constexpr SmallRegisterMapType() = default; + ~SmallRegisterMapType() = default; + NONCOPYABLE(SmallRegisterMapType); -public: - static const SmallRegisterMap* instance() { - static constexpr SmallRegisterMap the_instance{}; - return &the_instance; - } -private: static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN DEBUG_ONLY({ Unimplemented(); }) public: @@ -69,7 +68,7 @@ class SmallRegisterMap { bool update_map() const { return false; } bool walk_cont() const { return false; } - bool include_argument_oops() const { return false; } + bool include_argument_oops() const { return IncludeArgs; } void set_include_argument_oops(bool f) {} bool in_cont() const { return false; } stackChunkHandle stack_chunk() const { return stackChunkHandle(); } diff --git a/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp b/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp index 84d6dee91490d..3b9643fb3f4b1 100644 --- a/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp +++ b/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp @@ -85,7 +85,8 @@ inline int StackChunkFrameStream::interpreter_frame_stack_argsize() } template -inline int StackChunkFrameStream::interpreter_frame_num_oops() const { +template +inline int StackChunkFrameStream::interpreter_frame_num_oops(RegisterMapT* map) const { Unimplemented(); return 0; } diff --git a/src/hotspot/share/cds/aotConstantPoolResolver.cpp b/src/hotspot/share/cds/aotConstantPoolResolver.cpp index ff47d00c4847d..ddf7d32ed70ac 100644 --- a/src/hotspot/share/cds/aotConstantPoolResolver.cpp +++ b/src/hotspot/share/cds/aotConstantPoolResolver.cpp @@ -318,13 +318,13 @@ void AOTConstantPoolResolver::maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m if (!VM_Version::supports_fast_class_init_checks()) { return; // Do not resolve since interpreter lacks fast clinit barriers support } - InterpreterRuntime::resolve_get_put(bc, raw_index, mh, cp, false /*initialize_holder*/, CHECK); + InterpreterRuntime::resolve_get_put(bc, raw_index, mh, cp, ClassInitMode::dont_init, CHECK); is_static = " *** static"; break; case Bytecodes::_getfield: case Bytecodes::_putfield: - InterpreterRuntime::resolve_get_put(bc, raw_index, mh, cp, false /*initialize_holder*/, CHECK); + InterpreterRuntime::resolve_get_put(bc, raw_index, mh, cp, ClassInitMode::dont_init, CHECK); break; case Bytecodes::_invokestatic: diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp index de4d1c8a72947..146ee4af1e046 100644 --- a/src/hotspot/share/cds/heapShared.cpp +++ b/src/hotspot/share/cds/heapShared.cpp @@ -1809,7 +1809,8 @@ void HeapShared::check_special_subgraph_classes() { name != vmSymbols::java_lang_ArrayStoreException() && name != vmSymbols::java_lang_ClassCastException() && name != vmSymbols::java_lang_InternalError() && - name != vmSymbols::java_lang_NullPointerException()) { + name != vmSymbols::java_lang_NullPointerException() && + name != vmSymbols::jdk_internal_vm_PreemptedException()) { ResourceMark rm; fatal("special subgraph cannot have objects of type %s", subgraph_k->external_name()); } diff --git a/src/hotspot/share/ci/ciField.cpp b/src/hotspot/share/ci/ciField.cpp index cbe0cadbc9304..d47c4c508d775 100644 --- a/src/hotspot/share/ci/ciField.cpp +++ b/src/hotspot/share/ci/ciField.cpp @@ -400,7 +400,7 @@ bool ciField::will_link(ciMethod* accessing_method, _name->get_symbol(), _signature->get_symbol(), methodHandle(THREAD, accessing_method->get_Method())); fieldDescriptor result; - LinkResolver::resolve_field(result, link_info, bc, false, CHECK_AND_CLEAR_(false)); + LinkResolver::resolve_field(result, link_info, bc, ClassInitMode::dont_init, CHECK_AND_CLEAR_(false)); // update the hit-cache, unless there is a problem with memory scoping: if (accessing_method->holder()->is_shared() || !is_shared()) { diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp index cfe55bf7f762b..ac701e8364f59 100644 --- a/src/hotspot/share/classfile/javaClasses.cpp +++ b/src/hotspot/share/classfile/javaClasses.cpp @@ -2115,6 +2115,7 @@ int java_lang_VirtualThread::_state_offset; int java_lang_VirtualThread::_next_offset; int java_lang_VirtualThread::_onWaitingList_offset; int java_lang_VirtualThread::_notified_offset; +int java_lang_VirtualThread::_interruptible_wait_offset; int java_lang_VirtualThread::_timeout_offset; int java_lang_VirtualThread::_objectWaiter_offset; @@ -2126,6 +2127,7 @@ int java_lang_VirtualThread::_objectWaiter_offset; macro(_next_offset, k, "next", vthread_signature, false); \ macro(_onWaitingList_offset, k, "onWaitingList", bool_signature, false); \ macro(_notified_offset, k, "notified", bool_signature, false); \ + macro(_interruptible_wait_offset, k, "interruptibleWait", bool_signature, false); \ macro(_timeout_offset, k, "timeout", long_signature, false); @@ -2195,6 +2197,10 @@ void java_lang_VirtualThread::set_notified(oop vthread, jboolean value) { vthread->bool_field_put_volatile(_notified_offset, value); } +void java_lang_VirtualThread::set_interruptible_wait(oop vthread, jboolean value) { + vthread->bool_field_put_volatile(_interruptible_wait_offset, value); +} + jlong java_lang_VirtualThread::timeout(oop vthread) { return vthread->long_field(_timeout_offset); } diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp index 750f27fcf47c6..28f8c0a3b8cf3 100644 --- a/src/hotspot/share/classfile/javaClasses.hpp +++ b/src/hotspot/share/classfile/javaClasses.hpp @@ -563,6 +563,7 @@ class java_lang_VirtualThread : AllStatic { static int _next_offset; static int _onWaitingList_offset; static int _notified_offset; + static int _interruptible_wait_offset; static int _recheckInterval_offset; static int _timeout_offset; static int _objectWaiter_offset; @@ -615,6 +616,7 @@ class java_lang_VirtualThread : AllStatic { static jlong timeout(oop vthread); static void set_timeout(oop vthread, jlong value); static void set_notified(oop vthread, jboolean value); + static void set_interruptible_wait(oop vthread, jboolean value); static bool is_preempted(oop vthread); static JavaThreadStatus map_state_to_thread_status(int state); diff --git a/src/hotspot/share/classfile/vmClassMacros.hpp b/src/hotspot/share/classfile/vmClassMacros.hpp index 1edc13054d474..04f0aaaaa44f8 100644 --- a/src/hotspot/share/classfile/vmClassMacros.hpp +++ b/src/hotspot/share/classfile/vmClassMacros.hpp @@ -75,6 +75,7 @@ do_klass(IllegalMonitorStateException_klass, java_lang_IllegalMonitorStateException ) \ do_klass(Reference_klass, java_lang_ref_Reference ) \ do_klass(IllegalCallerException_klass, java_lang_IllegalCallerException ) \ + do_klass(PreemptedException_klass, jdk_internal_vm_PreemptedException ) \ \ /* ref klasses and set reference types */ \ do_klass(SoftReference_klass, java_lang_ref_SoftReference ) \ diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index 06f27f09c5c01..d22700b8d1f05 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -220,6 +220,7 @@ class SerializeClosure; template(java_lang_Exception, "java/lang/Exception") \ template(java_lang_RuntimeException, "java/lang/RuntimeException") \ template(java_io_IOException, "java/io/IOException") \ + template(jdk_internal_vm_PreemptedException, "jdk/internal/vm/PreemptedException") \ \ /* error klasses: at least all errors thrown by the VM have entries here */ \ template(java_lang_AbstractMethodError, "java/lang/AbstractMethodError") \ @@ -512,6 +513,8 @@ class SerializeClosure; template(maxThawingSize_name, "maxThawingSize") \ template(lockStackSize_name, "lockStackSize") \ template(objectWaiter_name, "objectWaiter") \ + template(atKlassInit_name, "atKlassInit") \ + template(hasArgsAtTop_name, "hasArgsAtTop") \ \ /* name symbols needed by intrinsics */ \ VM_INTRINSICS_DO(VM_INTRINSIC_IGNORE, VM_SYMBOL_IGNORE, template, VM_SYMBOL_IGNORE, VM_ALIAS_IGNORE) \ diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp index 6e067c77287ff..5f4cc6c450d0d 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.cpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp @@ -219,7 +219,7 @@ JRT_ENTRY(void, InterpreterRuntime::_new(JavaThread* current, ConstantPool* pool klass->check_valid_for_instantiation(true, CHECK); // Make sure klass is initialized - klass->initialize(CHECK); + klass->initialize_preemptable(CHECK_AND_CLEAR_PREEMPTED); oop obj = klass->allocate_instance(CHECK); current->set_vm_result_oop(obj); @@ -646,18 +646,19 @@ JRT_END // Fields // -void InterpreterRuntime::resolve_get_put(JavaThread* current, Bytecodes::Code bytecode) { +void InterpreterRuntime::resolve_get_put(Bytecodes::Code bytecode, TRAPS) { + JavaThread* current = THREAD; LastFrameAccessor last_frame(current); constantPoolHandle pool(current, last_frame.method()->constants()); methodHandle m(current, last_frame.method()); - resolve_get_put(bytecode, last_frame.get_index_u2(bytecode), m, pool, true /*initialize_holder*/, current); + resolve_get_put(bytecode, last_frame.get_index_u2(bytecode), m, pool, ClassInitMode::init_preemptable, THREAD); } void InterpreterRuntime::resolve_get_put(Bytecodes::Code bytecode, int field_index, methodHandle& m, constantPoolHandle& pool, - bool initialize_holder, TRAPS) { + ClassInitMode init_mode, TRAPS) { fieldDescriptor info; bool is_put = (bytecode == Bytecodes::_putfield || bytecode == Bytecodes::_nofast_putfield || bytecode == Bytecodes::_putstatic); @@ -665,8 +666,7 @@ void InterpreterRuntime::resolve_get_put(Bytecodes::Code bytecode, int field_ind { JvmtiHideSingleStepping jhss(THREAD); - LinkResolver::resolve_field_access(info, pool, field_index, - m, bytecode, initialize_holder, CHECK); + LinkResolver::resolve_field_access(info, pool, field_index, m, bytecode, init_mode, CHECK); } // end JvmtiHideSingleStepping // check if link resolution caused cpCache to be updated @@ -794,7 +794,8 @@ JRT_ENTRY(void, InterpreterRuntime::_breakpoint(JavaThread* current, Method* met JvmtiExport::post_raw_breakpoint(current, method, bcp); JRT_END -void InterpreterRuntime::resolve_invoke(JavaThread* current, Bytecodes::Code bytecode) { +void InterpreterRuntime::resolve_invoke(Bytecodes::Code bytecode, TRAPS) { + JavaThread* current = THREAD; LastFrameAccessor last_frame(current); // extract receiver from the outgoing argument list if necessary Handle receiver(current, nullptr); @@ -822,10 +823,9 @@ void InterpreterRuntime::resolve_invoke(JavaThread* current, Bytecodes::Code byt int method_index = last_frame.get_index_u2(bytecode); { JvmtiHideSingleStepping jhss(current); - JavaThread* THREAD = current; // For exception macros. LinkResolver::resolve_invoke(info, receiver, pool, method_index, bytecode, - THREAD); + ClassInitMode::init_preemptable, THREAD); if (HAS_PENDING_EXCEPTION) { if (ProfileTraps && PENDING_EXCEPTION->klass()->name() == vmSymbols::java_lang_NullPointerException()) { @@ -933,7 +933,8 @@ void InterpreterRuntime::cds_resolve_invoke(Bytecodes::Code bytecode, int method } // First time execution: Resolve symbols, create a permanent MethodType object. -void InterpreterRuntime::resolve_invokehandle(JavaThread* current) { +void InterpreterRuntime::resolve_invokehandle(TRAPS) { + JavaThread* current = THREAD; const Bytecodes::Code bytecode = Bytecodes::_invokehandle; LastFrameAccessor last_frame(current); @@ -962,7 +963,8 @@ void InterpreterRuntime::cds_resolve_invokehandle(int raw_index, } // First time execution: Resolve symbols, create a permanent CallSite object. -void InterpreterRuntime::resolve_invokedynamic(JavaThread* current) { +void InterpreterRuntime::resolve_invokedynamic(TRAPS) { + JavaThread* current = THREAD; LastFrameAccessor last_frame(current); const Bytecodes::Code bytecode = Bytecodes::_invokedynamic; @@ -997,19 +999,19 @@ JRT_ENTRY(void, InterpreterRuntime::resolve_from_cache(JavaThread* current, Byte case Bytecodes::_putstatic: case Bytecodes::_getfield: case Bytecodes::_putfield: - resolve_get_put(current, bytecode); + resolve_get_put(bytecode, CHECK_AND_CLEAR_PREEMPTED); break; case Bytecodes::_invokevirtual: case Bytecodes::_invokespecial: case Bytecodes::_invokestatic: case Bytecodes::_invokeinterface: - resolve_invoke(current, bytecode); + resolve_invoke(bytecode, CHECK_AND_CLEAR_PREEMPTED); break; case Bytecodes::_invokehandle: - resolve_invokehandle(current); + resolve_invokehandle(THREAD); break; case Bytecodes::_invokedynamic: - resolve_invokedynamic(current); + resolve_invokedynamic(THREAD); break; default: fatal("unexpected bytecode: %s", Bytecodes::name(bytecode)); @@ -1524,3 +1526,11 @@ JRT_LEAF(intptr_t, InterpreterRuntime::trace_bytecode(JavaThread* current, intpt return preserve_this_value; JRT_END #endif // !PRODUCT + +#ifdef ASSERT +bool InterpreterRuntime::is_preemptable_call(address entry_point) { + return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter) || + entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache) || + entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::_new); +} +#endif // ASSERT diff --git a/src/hotspot/share/interpreter/interpreterRuntime.hpp b/src/hotspot/share/interpreter/interpreterRuntime.hpp index cdee0c9daa71e..f553debcd7506 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.hpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.hpp @@ -94,7 +94,7 @@ class InterpreterRuntime: AllStatic { // Used by AOTConstantPoolResolver static void resolve_get_put(Bytecodes::Code bytecode, int field_index, - methodHandle& m, constantPoolHandle& pool, bool initialize_holder, TRAPS); + methodHandle& m, constantPoolHandle& pool, ClassInitMode init_mode, TRAPS); static void cds_resolve_invoke(Bytecodes::Code bytecode, int method_index, constantPoolHandle& pool, TRAPS); static void cds_resolve_invokehandle(int raw_index, @@ -103,12 +103,12 @@ class InterpreterRuntime: AllStatic { constantPoolHandle& pool, TRAPS); private: // Statics & fields - static void resolve_get_put(JavaThread* current, Bytecodes::Code bytecode); + static void resolve_get_put(Bytecodes::Code bytecode, TRAPS); // Calls - static void resolve_invoke(JavaThread* current, Bytecodes::Code bytecode); - static void resolve_invokehandle (JavaThread* current); - static void resolve_invokedynamic(JavaThread* current); + static void resolve_invoke(Bytecodes::Code bytecode, TRAPS); + static void resolve_invokehandle (TRAPS); + static void resolve_invokedynamic(TRAPS); static void update_invoke_cp_cache_entry(CallInfo& info, Bytecodes::Code bytecode, methodHandle& resolved_method, @@ -170,6 +170,9 @@ class InterpreterRuntime: AllStatic { static void verify_mdp(Method* method, address bcp, address mdp); #endif // ASSERT static MethodCounters* build_method_counters(JavaThread* current, Method* m); + + // Virtual Thread Preemption + DEBUG_ONLY(static bool is_preemptable_call(address entry_point);) }; diff --git a/src/hotspot/share/interpreter/linkResolver.cpp b/src/hotspot/share/interpreter/linkResolver.cpp index d46ccdb4d1c47..c82398b654c83 100644 --- a/src/hotspot/share/interpreter/linkResolver.cpp +++ b/src/hotspot/share/interpreter/linkResolver.cpp @@ -986,14 +986,14 @@ void LinkResolver::resolve_field_access(fieldDescriptor& fd, int index, const methodHandle& method, Bytecodes::Code byte, - bool initialize_class, TRAPS) { + ClassInitMode init_mode, TRAPS) { LinkInfo link_info(pool, index, method, byte, CHECK); - resolve_field(fd, link_info, byte, initialize_class, CHECK); + resolve_field(fd, link_info, byte, init_mode, CHECK); } void LinkResolver::resolve_field(fieldDescriptor& fd, const LinkInfo& link_info, - Bytecodes::Code byte, bool initialize_class, + Bytecodes::Code byte, ClassInitMode init_mode, TRAPS) { assert(byte == Bytecodes::_getstatic || byte == Bytecodes::_putstatic || byte == Bytecodes::_getfield || byte == Bytecodes::_putfield || @@ -1077,8 +1077,12 @@ void LinkResolver::resolve_field(fieldDescriptor& fd, // // note 2: we don't want to force initialization if we are just checking // if the field access is legal; e.g., during compilation - if (is_static && initialize_class) { - sel_klass->initialize(CHECK); + if (is_static) { + if (init_mode == ClassInitMode::init) { + sel_klass->initialize(CHECK); + } else if (init_mode == ClassInitMode::init_preemptable) { + sel_klass->initialize_preemptable(CHECK); + } } } @@ -1104,15 +1108,19 @@ void LinkResolver::resolve_field(fieldDescriptor& fd, void LinkResolver::resolve_static_call(CallInfo& result, const LinkInfo& link_info, - bool initialize_class, TRAPS) { + ClassInitMode init_mode, TRAPS) { Method* resolved_method = linktime_resolve_static_method(link_info, CHECK); // The resolved class can change as a result of this resolution. Klass* resolved_klass = resolved_method->method_holder(); // Initialize klass (this should only happen if everything is ok) - if (initialize_class && resolved_klass->should_be_initialized()) { - resolved_klass->initialize(CHECK); + if (init_mode != ClassInitMode::dont_init && resolved_klass->should_be_initialized()) { + if (init_mode == ClassInitMode::init) { + resolved_klass->initialize(CHECK); + } else if (init_mode == ClassInitMode::init_preemptable) { + resolved_klass->initialize_preemptable(CHECK); + } // Use updated LinkInfo to reresolve with resolved method holder LinkInfo new_info(resolved_klass, link_info.name(), link_info.signature(), link_info.current_klass(), @@ -1127,7 +1135,7 @@ void LinkResolver::resolve_static_call(CallInfo& result, } void LinkResolver::cds_resolve_static_call(CallInfo& result, const LinkInfo& link_info, TRAPS) { - resolve_static_call(result, link_info, /*initialize_class*/false, CHECK); + resolve_static_call(result, link_info, ClassInitMode::dont_init, CHECK); } // throws linktime exceptions @@ -1678,7 +1686,7 @@ int LinkResolver::resolve_virtual_vtable_index(Klass* receiver_klass, Method* LinkResolver::resolve_static_call_or_null(const LinkInfo& link_info) { EXCEPTION_MARK; CallInfo info; - resolve_static_call(info, link_info, /*initialize_class*/false, THREAD); + resolve_static_call(info, link_info, ClassInitMode::dont_init, THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_EXCEPTION; return nullptr; @@ -1702,15 +1710,15 @@ Method* LinkResolver::resolve_special_call_or_null(const LinkInfo& link_info) { //------------------------------------------------------------------------------------------------------------------------ // ConstantPool entries -void LinkResolver::resolve_invoke(CallInfo& result, Handle recv, const constantPoolHandle& pool, int index, Bytecodes::Code byte, TRAPS) { +void LinkResolver::resolve_invoke(CallInfo& result, Handle recv, const constantPoolHandle& pool, int index, Bytecodes::Code byte, ClassInitMode init_mode, TRAPS) { switch (byte) { - case Bytecodes::_invokestatic : resolve_invokestatic (result, pool, index, CHECK); break; - case Bytecodes::_invokespecial : resolve_invokespecial (result, recv, pool, index, CHECK); break; - case Bytecodes::_invokevirtual : resolve_invokevirtual (result, recv, pool, index, CHECK); break; - case Bytecodes::_invokehandle : resolve_invokehandle (result, pool, index, CHECK); break; - case Bytecodes::_invokedynamic : resolve_invokedynamic (result, pool, index, CHECK); break; - case Bytecodes::_invokeinterface: resolve_invokeinterface(result, recv, pool, index, CHECK); break; - default : break; + case Bytecodes::_invokestatic : resolve_invokestatic (result, pool, index, init_mode, CHECK); break; + case Bytecodes::_invokespecial : resolve_invokespecial (result, recv, pool, index, CHECK); break; + case Bytecodes::_invokevirtual : resolve_invokevirtual (result, recv, pool, index, CHECK); break; + case Bytecodes::_invokehandle : resolve_invokehandle (result, pool, index, CHECK); break; + case Bytecodes::_invokedynamic : resolve_invokedynamic (result, pool, index, CHECK); break; + case Bytecodes::_invokeinterface: resolve_invokeinterface(result, recv, pool, index, CHECK); break; + default : break; } return; } @@ -1732,7 +1740,7 @@ void LinkResolver::resolve_invoke(CallInfo& result, Handle& recv, /*check_null_and_abstract=*/true, CHECK); break; case Bytecodes::_invokestatic: - resolve_static_call(result, link_info, /*initialize_class=*/false, CHECK); + resolve_static_call(result, link_info, ClassInitMode::dont_init, CHECK); break; case Bytecodes::_invokespecial: resolve_special_call(result, recv, link_info, CHECK); @@ -1743,9 +1751,9 @@ void LinkResolver::resolve_invoke(CallInfo& result, Handle& recv, } } -void LinkResolver::resolve_invokestatic(CallInfo& result, const constantPoolHandle& pool, int index, TRAPS) { +void LinkResolver::resolve_invokestatic(CallInfo& result, const constantPoolHandle& pool, int index, ClassInitMode init_mode, TRAPS) { LinkInfo link_info(pool, index, Bytecodes::_invokestatic, CHECK); - resolve_static_call(result, link_info, /*initialize_class*/true, CHECK); + resolve_static_call(result, link_info, init_mode, CHECK); } diff --git a/src/hotspot/share/interpreter/linkResolver.hpp b/src/hotspot/share/interpreter/linkResolver.hpp index 18fb4ee6ccb54..4d28e712c5f4b 100644 --- a/src/hotspot/share/interpreter/linkResolver.hpp +++ b/src/hotspot/share/interpreter/linkResolver.hpp @@ -189,6 +189,12 @@ class LinkInfo : public StackObj { void print() PRODUCT_RETURN; }; +enum class ClassInitMode { + dont_init, + init, + init_preemptable +}; + // Link information for getfield/putfield & getstatic/putstatic bytecodes // is represented using a fieldDescriptor. @@ -267,7 +273,7 @@ class LinkResolver: AllStatic { // runtime resolving from constant pool static void resolve_invokestatic (CallInfo& result, - const constantPoolHandle& pool, int index, TRAPS); + const constantPoolHandle& pool, int index, ClassInitMode mode, TRAPS); static void resolve_invokespecial (CallInfo& result, Handle recv, const constantPoolHandle& pool, int index, TRAPS); static void resolve_invokevirtual (CallInfo& result, Handle recv, @@ -295,22 +301,21 @@ class LinkResolver: AllStatic { int index, const methodHandle& method, Bytecodes::Code byte, - bool initialize_class, TRAPS); + ClassInitMode mode, TRAPS); static void resolve_field_access(fieldDescriptor& result, const constantPoolHandle& pool, int index, const methodHandle& method, Bytecodes::Code byte, TRAPS) { - resolve_field_access(result, pool, index, method, byte, - /* initialize_class*/true, THREAD); + resolve_field_access(result, pool, index, method, byte, ClassInitMode::init, THREAD); } static void resolve_field(fieldDescriptor& result, const LinkInfo& link_info, Bytecodes::Code access_kind, - bool initialize_class, TRAPS); + ClassInitMode mode, TRAPS); static void resolve_static_call (CallInfo& result, const LinkInfo& link_info, - bool initialize_klass, TRAPS); + ClassInitMode mode, TRAPS); static void resolve_special_call (CallInfo& result, Handle recv, const LinkInfo& link_info, @@ -353,7 +358,12 @@ class LinkResolver: AllStatic { // runtime resolving from constant pool static void resolve_invoke(CallInfo& result, Handle recv, const constantPoolHandle& pool, int index, - Bytecodes::Code byte, TRAPS); + Bytecodes::Code byte, ClassInitMode static_mode, TRAPS); + static void resolve_invoke(CallInfo& result, Handle recv, + const constantPoolHandle& pool, int index, + Bytecodes::Code byte, TRAPS) { + resolve_invoke(result, recv, pool, index, byte, ClassInitMode::init, THREAD); + } // runtime resolving from attached method static void resolve_invoke(CallInfo& result, Handle& recv, diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp index 36a6fae794a91..5891bec9c8a5f 100644 --- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp +++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp @@ -1008,7 +1008,7 @@ C2V_VMENTRY_NULL(jobject, resolveFieldInPool, (JNIEnv* env, jobject, ARGUMENT_PA } LinkInfo link_info(cp, index, mh, code, CHECK_NULL); - LinkResolver::resolve_field(fd, link_info, Bytecodes::java_code(code), false, CHECK_NULL); + LinkResolver::resolve_field(fd, link_info, Bytecodes::java_code(code), ClassInitMode::dont_init, CHECK_NULL); JVMCIPrimitiveArray info = JVMCIENV->wrap(info_handle); if (info.is_null() || JVMCIENV->get_length(info) != 4) { JVMCI_ERROR_NULL("info must not be null and have a length of 4"); diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 756619bff3386..69afe5c6450ed 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -238,6 +238,7 @@ static BuiltinException _internal_error; static BuiltinException _array_index_out_of_bounds_exception; static BuiltinException _array_store_exception; static BuiltinException _class_cast_exception; +static BuiltinException _preempted_exception; objArrayOop Universe::the_empty_class_array () { return (objArrayOop)_the_empty_class_array.resolve(); @@ -258,6 +259,7 @@ oop Universe::internal_error_instance() { return _internal_error.insta oop Universe::array_index_out_of_bounds_exception_instance() { return _array_index_out_of_bounds_exception.instance(); } oop Universe::array_store_exception_instance() { return _array_store_exception.instance(); } oop Universe::class_cast_exception_instance() { return _class_cast_exception.instance(); } +oop Universe::preempted_exception_instance() { return _preempted_exception.instance(); } oop Universe::the_null_sentinel() { return _the_null_sentinel.resolve(); } @@ -317,6 +319,7 @@ void Universe::archive_exception_instances() { _array_index_out_of_bounds_exception.store_in_cds(); _array_store_exception.store_in_cds(); _class_cast_exception.store_in_cds(); + _preempted_exception.store_in_cds(); } void Universe::load_archived_object_instances() { @@ -336,6 +339,7 @@ void Universe::load_archived_object_instances() { _array_index_out_of_bounds_exception.load_from_cds(); _array_store_exception.load_from_cds(); _class_cast_exception.load_from_cds(); + _preempted_exception.load_from_cds(); } } #endif @@ -355,6 +359,7 @@ void Universe::serialize(SerializeClosure* f) { _array_index_out_of_bounds_exception.serialize(f); _array_store_exception.serialize(f); _class_cast_exception.serialize(f); + _preempted_exception.serialize(f); #endif f->do_ptr(&_fillerArrayKlass); @@ -1139,6 +1144,7 @@ bool universe_post_init() { _array_index_out_of_bounds_exception.init_if_empty(vmSymbols::java_lang_ArrayIndexOutOfBoundsException(), CHECK_false); _array_store_exception.init_if_empty(vmSymbols::java_lang_ArrayStoreException(), CHECK_false); _class_cast_exception.init_if_empty(vmSymbols::java_lang_ClassCastException(), CHECK_false); + _preempted_exception.init_if_empty(vmSymbols::jdk_internal_vm_PreemptedException(), CHECK_false); // Virtual Machine Error for when we get into a situation we can't resolve Klass* k = vmClasses::InternalError_klass(); diff --git a/src/hotspot/share/memory/universe.hpp b/src/hotspot/share/memory/universe.hpp index 37ca965062e80..df2c1d66d3c9c 100644 --- a/src/hotspot/share/memory/universe.hpp +++ b/src/hotspot/share/memory/universe.hpp @@ -244,6 +244,7 @@ class Universe: AllStatic { static oop array_index_out_of_bounds_exception_instance(); static oop array_store_exception_instance(); static oop class_cast_exception_instance(); + static oop preempted_exception_instance(); static oop vm_exception() { return internal_error_instance(); } static Array* the_array_interfaces_array() { return _the_array_interfaces_array; } diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index e13f484945405..2d03b69ee92e9 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -793,10 +793,11 @@ oop InstanceKlass::init_lock() const { return lock; } -// Set the initialization lock to null so the object can be GC'ed. Any racing +// Set the initialization lock to null so the object can be GC'ed. Any racing // threads to get this lock will see a null lock and will not lock. // That's okay because they all check for initialized state after getting -// the lock and return. +// the lock and return. For preempted vthreads we keep the oop protected +// in the ObjectMonitor (see ObjectMonitor::set_object_strong()). void InstanceKlass::fence_and_clear_init_lock() { // make sure previous stores are all done, notably the init_state. OrderAccess::storestore(); @@ -804,6 +805,31 @@ void InstanceKlass::fence_and_clear_init_lock() { assert(!is_not_initialized(), "class must be initialized now"); } +class PreemptableInitCall { + JavaThread* _thread; + bool _previous; + DEBUG_ONLY(InstanceKlass* _previous_klass;) + public: + PreemptableInitCall(JavaThread* thread, InstanceKlass* ik) : _thread(thread) { + _previous = thread->at_preemptable_init(); + _thread->set_at_preemptable_init(true); + DEBUG_ONLY(_previous_klass = _thread->preempt_init_klass();) + DEBUG_ONLY(_thread->set_preempt_init_klass(ik)); + } + ~PreemptableInitCall() { + _thread->set_at_preemptable_init(_previous); + DEBUG_ONLY(_thread->set_preempt_init_klass(_previous_klass)); + } +}; + +void InstanceKlass::initialize_preemptable(TRAPS) { + if (this->should_be_initialized()) { + PreemptableInitCall pic(THREAD, this); + initialize_impl(THREAD); + } else { + assert(is_initialized(), "sanity check"); + } +} // See "The Virtual Machine Specification" section 2.16.5 for a detailed explanation of the class initialization // process. The step comments refers to the procedure described in that section. @@ -980,7 +1006,12 @@ bool InstanceKlass::link_class_impl(TRAPS) { { HandleMark hm(THREAD); Handle h_init_lock(THREAD, init_lock()); - ObjectLocker ol(h_init_lock, jt); + ObjectLocker ol(h_init_lock, CHECK_PREEMPTABLE_false); + // Don't allow preemption if we link/initialize classes below, + // since that would release this monitor while we are in the + // middle of linking this class. + NoPreemptMark npm(THREAD); + // rewritten will have been set if loader constraint error found // on an earlier link attempt // don't verify or rewrite if already rewritten @@ -1174,6 +1205,17 @@ void InstanceKlass::clean_initialization_error_table() { } } +class ThreadWaitingForClassInit : public StackObj { + JavaThread* _thread; + public: + ThreadWaitingForClassInit(JavaThread* thread, InstanceKlass* ik) : _thread(thread) { + _thread->set_class_to_be_initialized(ik); + } + ~ThreadWaitingForClassInit() { + _thread->set_class_to_be_initialized(nullptr); + } +}; + void InstanceKlass::initialize_impl(TRAPS) { HandleMark hm(THREAD); @@ -1193,7 +1235,7 @@ void InstanceKlass::initialize_impl(TRAPS) { // Step 1 { Handle h_init_lock(THREAD, init_lock()); - ObjectLocker ol(h_init_lock, jt); + ObjectLocker ol(h_init_lock, CHECK_PREEMPTABLE); // Step 2 // If we were to use wait() instead of waitInterruptibly() then @@ -1206,9 +1248,8 @@ void InstanceKlass::initialize_impl(TRAPS) { jt->name(), external_name(), init_thread_name()); } wait = true; - jt->set_class_to_be_initialized(this); - ol.wait_uninterruptibly(jt); - jt->set_class_to_be_initialized(nullptr); + ThreadWaitingForClassInit twcl(THREAD, this); + ol.wait_uninterruptibly(CHECK_PREEMPTABLE); } // Step 3 @@ -1266,6 +1307,10 @@ void InstanceKlass::initialize_impl(TRAPS) { } } + // Block preemption once we are the initializer thread. Unmounting now + // would complicate the reentrant case (identity is platform thread). + NoPreemptMark npm(THREAD); + // Step 7 // Next, if C is a class rather than an interface, initialize it's super class and super // interfaces. diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 82c82714f9be4..8bb9741af9f42 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -538,6 +538,7 @@ class InstanceKlass: public Klass { void initialize_with_aot_initialized_mirror(TRAPS); void assert_no_clinit_will_run_for_aot_initialized_class() const NOT_DEBUG_RETURN; void initialize(TRAPS); + void initialize_preemptable(TRAPS); void link_class(TRAPS); bool link_class_or_fail(TRAPS); // returns false on failure void rewrite_class(TRAPS); diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.cpp b/src/hotspot/share/oops/instanceStackChunkKlass.cpp index db60e74013f08..aaa3eaa4f6b31 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.cpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.cpp @@ -195,7 +195,8 @@ class DescribeStackChunkClosure { } const RegisterMap* get_map(const RegisterMap* map, intptr_t* sp) { return map; } - const RegisterMap* get_map(const SmallRegisterMap* map, intptr_t* sp) { return map->copy_to_RegisterMap(&_map, sp); } + template + const RegisterMap* get_map(const SmallRegisterMapT map, intptr_t* sp) { return map->copy_to_RegisterMap(&_map, sp); } template bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 208a274d766c7..d9041cc54f41b 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -252,6 +252,10 @@ void Klass::initialize(TRAPS) { ShouldNotReachHere(); } +void Klass::initialize_preemptable(TRAPS) { + ShouldNotReachHere(); +} + Klass* Klass::find_field(Symbol* name, Symbol* sig, fieldDescriptor* fd) const { #ifdef ASSERT tty->print_cr("Error: find_field called on a klass oop." diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index db3360b080e41..5ac393d761409 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -580,6 +580,7 @@ class Klass : public Metadata { virtual bool should_be_initialized() const { return false; } // initializes the klass virtual void initialize(TRAPS); + virtual void initialize_preemptable(TRAPS); virtual Klass* find_field(Symbol* name, Symbol* signature, fieldDescriptor* fd) const; virtual Method* uncached_lookup_method(const Symbol* name, const Symbol* signature, OverpassLookupMode overpass_mode, diff --git a/src/hotspot/share/oops/stackChunkOop.cpp b/src/hotspot/share/oops/stackChunkOop.cpp index 13a94d65f8ebe..67e5e474cceca 100644 --- a/src/hotspot/share/oops/stackChunkOop.cpp +++ b/src/hotspot/share/oops/stackChunkOop.cpp @@ -56,7 +56,7 @@ class FrameOopIterator : public OopIterator { virtual void oops_do(OopClosure* cl) override { if (_f.is_interpreted_frame()) { - _f.oops_interpreted_do(cl, nullptr); + _f.oops_interpreted_do(cl, _map); } else { OopMapDo visitor(cl, nullptr); visitor.oops_do(&_f, _map, _f.oop_map()); @@ -139,7 +139,7 @@ static int num_java_frames(const StackChunkFrameStream& f) { int stackChunkOopDesc::num_java_frames() const { int n = 0; for (StackChunkFrameStream f(const_cast(this)); !f.is_done(); - f.next(SmallRegisterMap::instance())) { + f.next(SmallRegisterMap::instance_no_args())) { if (!f.is_stub()) { n += ::num_java_frames(f); } @@ -415,10 +415,12 @@ template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const RegisterMap* map); template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const RegisterMap* map); template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const RegisterMap* map); -template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMap* map); -template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMap* map); -template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMap* map); -template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMap* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMapNoArgs* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMapNoArgs* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMapNoArgs* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMapNoArgs* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMapWithArgs* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMapWithArgs* map); template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMapT* map) { @@ -438,7 +440,8 @@ void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMapT* map } template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMap* map); -template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallRegisterMap* map); +template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallRegisterMapNoArgs* map); +template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallRegisterMapWithArgs* map); void stackChunkOopDesc::transfer_lockstack(oop* dst, bool requires_barriers) { const bool requires_gc_barriers = is_gc_mode() || requires_barriers; @@ -527,7 +530,7 @@ class VerifyStackChunkFrameClosure { _cb = f.cb(); int fsize = f.frame_size() - ((f.is_interpreted() == _callee_interpreted) ? _argsize : 0); - int num_oops = f.num_oops(); + int num_oops = f.num_oops(map); assert(num_oops >= 0, ""); _argsize = f.stack_argsize() + frame::metadata_words_at_top; diff --git a/src/hotspot/share/oops/stackChunkOop.hpp b/src/hotspot/share/oops/stackChunkOop.hpp index 57ab6316c2a17..79944f0326eca 100644 --- a/src/hotspot/share/oops/stackChunkOop.hpp +++ b/src/hotspot/share/oops/stackChunkOop.hpp @@ -137,6 +137,12 @@ class stackChunkOopDesc : public instanceOopDesc { inline bool preempted() const; inline void set_preempted(bool value); + inline bool at_klass_init() const; + inline void set_at_klass_init(bool value); + + inline bool has_args_at_top() const; + inline void set_has_args_at_top(bool value); + inline bool has_lockstack() const; inline void set_has_lockstack(bool value); diff --git a/src/hotspot/share/oops/stackChunkOop.inline.hpp b/src/hotspot/share/oops/stackChunkOop.inline.hpp index 9c91e93432881..4fe5b913adebe 100644 --- a/src/hotspot/share/oops/stackChunkOop.inline.hpp +++ b/src/hotspot/share/oops/stackChunkOop.inline.hpp @@ -168,6 +168,18 @@ inline void stackChunkOopDesc::set_preempted(bool value) { set_flag(FLAG_PREEMPTED, value); } +inline bool stackChunkOopDesc::at_klass_init() const { return jdk_internal_vm_StackChunk::atKlassInit(as_oop()); } +inline void stackChunkOopDesc::set_at_klass_init(bool value) { + assert(at_klass_init() != value, ""); + jdk_internal_vm_StackChunk::set_atKlassInit(this, value); +} + +inline bool stackChunkOopDesc::has_args_at_top() const { return jdk_internal_vm_StackChunk::hasArgsAtTop(as_oop()); } +inline void stackChunkOopDesc::set_has_args_at_top(bool value) { + assert(has_args_at_top() != value, ""); + jdk_internal_vm_StackChunk::set_hasArgsAtTop(this, value); +} + inline bool stackChunkOopDesc::has_lockstack() const { return is_flag(FLAG_HAS_LOCKSTACK); } inline void stackChunkOopDesc::set_has_lockstack(bool value) { set_flag(FLAG_HAS_LOCKSTACK, value); } @@ -210,7 +222,7 @@ inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure template inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { - const SmallRegisterMap* map = SmallRegisterMap::instance(); + const auto* map = SmallRegisterMap::instance_no_args(); assert(!map->in_cont(), ""); StackChunkFrameStream f(this); @@ -230,6 +242,9 @@ inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure should_continue = closure->do_frame(f, &full_map); f.next(map); + } else if (frame_kind == ChunkFrames::Mixed && f.is_interpreted() && has_args_at_top()) { + should_continue = closure->do_frame(f, SmallRegisterMap::instance_with_args()); + f.next(map); } assert(!f.is_stub(), ""); diff --git a/src/hotspot/share/prims/methodHandles.cpp b/src/hotspot/share/prims/methodHandles.cpp index b13bd392eaa5b..c243cae20ab3d 100644 --- a/src/hotspot/share/prims/methodHandles.cpp +++ b/src/hotspot/share/prims/methodHandles.cpp @@ -771,7 +771,7 @@ Handle MethodHandles::resolve_MemberName(Handle mname, Klass* caller, int lookup assert(!HAS_PENDING_EXCEPTION, ""); if (ref_kind == JVM_REF_invokeStatic) { LinkResolver::resolve_static_call(result, - link_info, false, THREAD); + link_info, ClassInitMode::dont_init, THREAD); } else if (ref_kind == JVM_REF_invokeInterface) { LinkResolver::resolve_interface_call(result, Handle(), defc, link_info, false, THREAD); @@ -833,7 +833,7 @@ Handle MethodHandles::resolve_MemberName(Handle mname, Klass* caller, int lookup { assert(!HAS_PENDING_EXCEPTION, ""); LinkInfo link_info(defc, name, type, caller, LinkInfo::AccessCheck::skip, loader_constraint_check); - LinkResolver::resolve_field(result, link_info, Bytecodes::_nop, false, THREAD); + LinkResolver::resolve_field(result, link_info, Bytecodes::_nop, ClassInitMode::dont_init, THREAD); if (HAS_PENDING_EXCEPTION) { if (speculative_resolve) { CLEAR_PENDING_EXCEPTION; diff --git a/src/hotspot/share/runtime/continuation.hpp b/src/hotspot/share/runtime/continuation.hpp index 0cfd484361dfd..c54a56ce7eb18 100644 --- a/src/hotspot/share/runtime/continuation.hpp +++ b/src/hotspot/share/runtime/continuation.hpp @@ -62,8 +62,9 @@ class Continuation : AllStatic { public: enum preempt_kind { - freeze_on_monitorenter, - freeze_on_wait + monitorenter, + object_wait, + object_locker }; enum thaw_kind { diff --git a/src/hotspot/share/runtime/continuationFreezeThaw.cpp b/src/hotspot/share/runtime/continuationFreezeThaw.cpp index 3e509e71551da..381e806e95d5a 100644 --- a/src/hotspot/share/runtime/continuationFreezeThaw.cpp +++ b/src/hotspot/share/runtime/continuationFreezeThaw.cpp @@ -33,11 +33,14 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shared/memAllocator.hpp" #include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "interpreter/bytecodeStream.hpp" #include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" #include "jfr/jfrEvents.hpp" #include "logging/log.hpp" #include "logging/logStream.hpp" #include "oops/access.inline.hpp" +#include "oops/constantPool.inline.hpp" #include "oops/method.inline.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oopsHierarchy.hpp" @@ -63,6 +66,8 @@ #include "runtime/stackFrameStream.inline.hpp" #include "runtime/stackOverflow.hpp" #include "runtime/stackWatermarkSet.inline.hpp" +#include "runtime/vframe.inline.hpp" +#include "runtime/vframe_hp.hpp" #include "utilities/debug.hpp" #include "utilities/exceptions.hpp" #include "utilities/macros.hpp" @@ -73,6 +78,12 @@ #if INCLUDE_JFR #include "jfr/jfr.inline.hpp" #endif +#ifdef COMPILER1 +#include "c1/c1_Runtime1.hpp" +#endif +#ifdef COMPILER2 +#include "opto/runtime.hpp" +#endif #include @@ -183,8 +194,9 @@ static void verify_continuation(oop continuation) { Continuation::debug_verify_c static void do_deopt_after_thaw(JavaThread* thread); static bool do_verify_after_thaw(JavaThread* thread, stackChunkOop chunk, outputStream* st); static void log_frames(JavaThread* thread); -static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted); +static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp); static void print_frame_layout(const frame& f, bool callee_complete, outputStream* st = tty); +static void verify_frame_kind(frame& top, Continuation::preempt_kind preempt_kind, Method** m_ptr = nullptr, const char** code_name_ptr = nullptr, int* bci_ptr = nullptr, stackChunkOop chunk = nullptr); #define assert_pfl(p, ...) \ do { \ @@ -462,7 +474,7 @@ class FreezeBase : public StackObj { inline void patch_pd(frame& callee, const frame& caller); inline void patch_pd_unused(intptr_t* sp); void adjust_interpreted_frame_unextended_sp(frame& f); - static inline void prepare_freeze_interpreted_top_frame(frame& f); + inline void prepare_freeze_interpreted_top_frame(frame& f); static inline void relativize_interpreted_frame_metadata(const frame& f, const frame& hf); protected: @@ -1090,13 +1102,27 @@ freeze_result FreezeBase::finalize_freeze(const frame& callee, frame& caller, in assert(!chunk->is_empty() || StackChunkFrameStream(chunk).to_frame().is_empty(), ""); if (_preempt) { - frame f = _thread->last_frame(); - if (f.is_interpreted_frame()) { + frame top_frame = _thread->last_frame(); + if (top_frame.is_interpreted_frame()) { // Some platforms do not save the last_sp in the top interpreter frame on VM calls. // We need it so that on resume we can restore the sp to the right place, since // thawing might add an alignment word to the expression stack (see finish_thaw()). // We do it now that we know freezing will be successful. - prepare_freeze_interpreted_top_frame(f); + prepare_freeze_interpreted_top_frame(top_frame); + } + + // Do this now so should_process_args_at_top() is set before calling finish_freeze + // in case we might need to apply GC barriers to frames in this stackChunk. + if (_thread->at_preemptable_init()) { + assert(top_frame.is_interpreted_frame(), "only InterpreterRuntime::_new/resolve_from_cache allowed"); + chunk->set_at_klass_init(true); + methodHandle m(_thread, top_frame.interpreter_frame_method()); + Bytecode_invoke call = Bytecode_invoke_check(m, top_frame.interpreter_frame_bci()); + assert(!call.is_valid() || call.is_invokestatic(), "only invokestatic allowed"); + if (call.is_invokestatic() && call.size_of_parameters() > 0) { + assert(top_frame.interpreter_frame_expression_stack_size() > 0, "should have parameters in exp stack"); + chunk->set_has_args_at_top(true); + } } } @@ -1608,6 +1634,25 @@ void FreezeBase::throw_stack_overflow_on_humongous_chunk() { Exceptions::_throw_msg(_thread, __FILE__, __LINE__, vmSymbols::java_lang_StackOverflowError(), "Humongous stack chunk"); } +class AnchorMark : public StackObj { + JavaThread* _current; + frame& _top_frame; + intptr_t* _last_sp_from_frame; + bool _is_interpreted; + + public: + AnchorMark(JavaThread* current, frame& f) : _current(current), _top_frame(f), _is_interpreted(false) { + intptr_t* sp = anchor_mark_set_pd(); + set_anchor(_current, sp); + } + ~AnchorMark() { + clear_anchor(_current); + anchor_mark_clear_pd(); + } + inline intptr_t* anchor_mark_set_pd(); + inline void anchor_mark_clear_pd(); +}; + #if INCLUDE_JVMTI static int num_java_frames(ContinuationWrapper& cont) { ResourceMark rm; // used for scope traversal in num_java_frames(nmethod*, address) @@ -1635,27 +1680,25 @@ static void jvmti_yield_cleanup(JavaThread* thread, ContinuationWrapper& cont) { invalidate_jvmti_stack(thread); } -static void jvmti_mount_end(JavaThread* current, ContinuationWrapper& cont, frame top) { +static void jvmti_mount_end(JavaThread* current, ContinuationWrapper& cont, frame top, Continuation::preempt_kind pk) { assert(current->vthread() != nullptr, "must be"); - HandleMarkCleaner hm(current); + HandleMarkCleaner hm(current); // Cleanup all handles (including so._conth) before returning to Java. Handle vth(current, current->vthread()); - ContinuationWrapper::SafepointOp so(current, cont); - - // Since we might safepoint set the anchor so that the stack can be walked. - set_anchor(current, top.sp()); + AnchorMark am(current, top); // Set anchor so that the stack is walkable. JRT_BLOCK JvmtiVTMSTransitionDisabler::VTMS_vthread_mount((jthread)vth.raw_value(), false); if (current->pending_contended_entered_event()) { - JvmtiExport::post_monitor_contended_entered(current, current->contended_entered_monitor()); + // No monitor JVMTI events for ObjectLocker case. + if (pk != Continuation::object_locker) { + JvmtiExport::post_monitor_contended_entered(current, current->contended_entered_monitor()); + } current->set_contended_entered_monitor(nullptr); } JRT_BLOCK_END - - clear_anchor(current); } #endif // INCLUDE_JVMTI @@ -1678,6 +1721,109 @@ bool FreezeBase::check_valid_fast_path() { } return true; } + +static void verify_frame_kind(frame& top, Continuation::preempt_kind preempt_kind, Method** m_ptr, const char** code_name_ptr, int* bci_ptr, stackChunkOop chunk) { + Method* m; + const char* code_name; + int bci; + if (preempt_kind == Continuation::monitorenter) { + assert(top.is_interpreted_frame() || top.is_runtime_frame(), "unexpected %sframe", + top.is_compiled_frame() ? "compiled " : top.is_native_frame() ? "native " : ""); + bool at_sync_method; + if (top.is_interpreted_frame()) { + m = top.interpreter_frame_method(); + assert(!m->is_native() || m->is_synchronized(), "invalid method %s", m->external_name()); + address bcp = top.interpreter_frame_bcp(); + assert(bcp != 0 || m->is_native(), ""); + at_sync_method = m->is_synchronized() && (bcp == 0 || bcp == m->code_base()); + // bcp is advanced on monitorenter before making the VM call, adjust for that. + bool at_sync_bytecode = bcp > m->code_base() && Bytecode(m, bcp - 1).code() == Bytecodes::Code::_monitorenter; + assert(at_sync_method || at_sync_bytecode, ""); + bci = at_sync_method ? -1 : top.interpreter_frame_bci(); + } else { + JavaThread* current = JavaThread::current(); + ResourceMark rm(current); + CodeBlob* cb = top.cb(); + RegisterMap reg_map(current, + RegisterMap::UpdateMap::skip, + RegisterMap::ProcessFrames::skip, + RegisterMap::WalkContinuation::include); + if (top.is_heap_frame()) { + assert(chunk != nullptr, ""); + reg_map.set_stack_chunk(chunk); + top = chunk->relativize(top); + top.set_frame_index(0); + } + frame fr = top.sender(®_map); + vframe* vf = vframe::new_vframe(&fr, ®_map, current); + compiledVFrame* cvf = compiledVFrame::cast(vf); + m = cvf->method(); + bci = cvf->scope()->bci(); + at_sync_method = bci == SynchronizationEntryBCI; + assert(!at_sync_method || m->is_synchronized(), "bci is %d but method %s is not synchronized", bci, m->external_name()); + bool is_c1_monitorenter = false, is_c2_monitorenter = false; + COMPILER1_PRESENT(is_c1_monitorenter = cb == Runtime1::blob_for(StubId::c1_monitorenter_id) || + cb == Runtime1::blob_for(StubId::c1_monitorenter_nofpu_id);) + COMPILER2_PRESENT(is_c2_monitorenter = cb == CodeCache::find_blob(OptoRuntime::complete_monitor_locking_Java());) + assert(is_c1_monitorenter || is_c2_monitorenter, "wrong runtime stub frame"); + } + code_name = at_sync_method ? "synchronized method" : "monitorenter"; + } else if (preempt_kind == Continuation::object_wait) { + assert(top.is_interpreted_frame() || top.is_native_frame(), ""); + m = top.is_interpreted_frame() ? top.interpreter_frame_method() : top.cb()->as_nmethod()->method(); + assert(m->is_object_wait0(), ""); + bci = 0; + code_name = ""; + } else { + assert(preempt_kind == Continuation::object_locker, "invalid preempt kind"); + assert(top.is_interpreted_frame(), ""); + m = top.interpreter_frame_method(); + Bytecode current_bytecode = Bytecode(m, top.interpreter_frame_bcp()); + Bytecodes::Code code = current_bytecode.code(); + assert(code == Bytecodes::Code::_new || code == Bytecodes::Code::_invokestatic || + (code == Bytecodes::Code::_getstatic || code == Bytecodes::Code::_putstatic), "invalid bytecode"); + bci = top.interpreter_frame_bci(); + code_name = Bytecodes::name(current_bytecode.code()); + } + assert(bci >= 0 || m->is_synchronized(), "invalid bci:%d at method %s", bci, m->external_name()); + + if (m_ptr != nullptr) { + *m_ptr = m; + *code_name_ptr = code_name; + *bci_ptr = bci; + } +} + +static void log_preempt_after_freeze(const ContinuationWrapper& cont) { + JavaThread* current = cont.thread(); + int64_t tid = current->monitor_owner_id(); + + StackChunkFrameStream sfs(cont.tail()); + frame top_frame = sfs.to_frame(); + bool at_init = current->at_preemptable_init(); + bool at_enter = current->current_pending_monitor() != nullptr; + bool at_wait = current->current_waiting_monitor() != nullptr; + assert((at_enter && !at_wait) || (!at_enter && at_wait), ""); + Continuation::preempt_kind pk = at_init ? Continuation::object_locker : at_enter ? Continuation::monitorenter : Continuation::object_wait; + + Method* m = nullptr; + const char* code_name = nullptr; + int bci = InvalidFrameStateBci; + verify_frame_kind(top_frame, pk, &m, &code_name, &bci, cont.tail()); + assert(m != nullptr && code_name != nullptr && bci != InvalidFrameStateBci, "should be set"); + + ResourceMark rm(current); + if (bci < 0) { + log_trace(continuations, preempt)("Preempted " INT64_FORMAT " while synchronizing on %smethod %s", tid, m->is_native() ? "native " : "", m->external_name()); + } else if (m->is_object_wait0()) { + log_trace(continuations, preempt)("Preempted " INT64_FORMAT " at native method %s", tid, m->external_name()); + } else { + Klass* k = current->preempt_init_klass(); + assert(k != nullptr || !at_init, ""); + log_trace(continuations, preempt)("Preempted " INT64_FORMAT " at %s(bci:%d) in method %s %s%s", tid, code_name, bci, + m->external_name(), at_init ? "trying to initialize klass " : "", at_init ? k->external_name() : ""); + } +} #endif // ASSERT static inline freeze_result freeze_epilog(ContinuationWrapper& cont) { @@ -1707,9 +1853,10 @@ static freeze_result preempt_epilog(ContinuationWrapper& cont, freeze_result res return res; } + // Set up things so that on return to Java we jump to preempt stub. patch_return_pc_with_preempt_stub(old_last_frame); cont.tail()->set_preempted(true); - + DEBUG_ONLY(log_preempt_after_freeze(cont);) return freeze_epilog(cont); } @@ -1905,10 +2052,14 @@ class ThawBase : public StackObj { intptr_t* _fastpath; bool _barriers; bool _preempted_case; + bool _process_args_at_top; intptr_t* _top_unextended_sp_before_thaw; int _align_size; DEBUG_ONLY(intptr_t* _top_stack_address); + // Only used for preemption on ObjectLocker + ObjectMonitor* _init_lock; + StackChunkFrameStream _stream; NOT_PRODUCT(int _frames;) @@ -1935,6 +2086,8 @@ class ThawBase : public StackObj { intptr_t* handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case); inline intptr_t* push_cleanup_continuation(); + inline intptr_t* push_preempt_adapter(); + intptr_t* redo_vmcall(JavaThread* current, frame& top); void throw_interrupted_exception(JavaThread* current, frame& top); void recurse_thaw(const frame& heap_frame, frame& caller, int num_frames, bool top_on_preempt_case); @@ -1951,12 +2104,12 @@ class ThawBase : public StackObj { inline void patch(frame& f, const frame& caller, bool bottom); void clear_bitmap_bits(address start, address end); - NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames); + NOINLINE void recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames, bool is_top); void recurse_thaw_compiled_frame(const frame& hf, frame& caller, int num_frames, bool stub_caller); void recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_frames); void recurse_thaw_native_frame(const frame& hf, frame& caller, int num_frames); - void push_return_frame(frame& f); + void push_return_frame(const frame& f); inline frame new_entry_frame(); template frame new_stack_frame(const frame& hf, frame& caller, bool bottom); inline void patch_pd(frame& f, const frame& sender); @@ -2051,7 +2204,7 @@ int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &arg // If we don't thaw the top compiled frame too, after restoring the saved // registers back in Java, we would hit the return barrier to thaw one more // frame effectively overwriting the restored registers during that call. - f.next(SmallRegisterMap::instance(), true /* stop */); + f.next(SmallRegisterMap::instance_no_args(), true /* stop */); assert(!f.is_done(), ""); f.get_cb(); @@ -2067,7 +2220,7 @@ int ThawBase::remove_top_compiled_frame_from_chunk(stackChunkOop chunk, int &arg } } - f.next(SmallRegisterMap::instance(), true /* stop */); + f.next(SmallRegisterMap::instance_no_args(), true /* stop */); empty = f.is_done(); assert(!empty || argsize == chunk->argsize(), ""); @@ -2199,12 +2352,12 @@ NOINLINE intptr_t* Thaw::thaw_fast(stackChunkOop chunk) { #endif #ifdef ASSERT - set_anchor(_thread, rs.sp()); - log_frames(_thread); if (LoomDeoptAfterThaw) { + frame top(rs.sp()); + AnchorMark am(_thread, top); + log_frames(_thread); do_deopt_after_thaw(_thread); } - clear_anchor(_thread); #endif return rs.sp(); @@ -2227,29 +2380,49 @@ NOINLINE intptr_t* Thaw::thaw_slow(stackChunkOop chunk, Continuation::t Continuation::preempt_kind preempt_kind; bool retry_fast_path = false; + _process_args_at_top = false; _preempted_case = chunk->preempted(); if (_preempted_case) { + ObjectMonitor* mon = nullptr; ObjectWaiter* waiter = java_lang_VirtualThread::objectWaiter(_thread->vthread()); if (waiter != nullptr) { // Mounted again after preemption. Resume the pending monitor operation, // which will be either a monitorenter or Object.wait() call. - ObjectMonitor* mon = waiter->monitor(); - preempt_kind = waiter->is_wait() ? Continuation::freeze_on_wait : Continuation::freeze_on_monitorenter; + mon = waiter->monitor(); + preempt_kind = waiter->is_wait() ? Continuation::object_wait : Continuation::monitorenter; bool mon_acquired = mon->resume_operation(_thread, waiter, _cont); assert(!mon_acquired || mon->has_owner(_thread), "invariant"); if (!mon_acquired) { // Failed to acquire monitor. Return to enterSpecial to unmount again. + log_develop_trace(continuations, preempt)("Failed to acquire monitor, unmounting again"); return push_cleanup_continuation(); } chunk = _cont.tail(); // reload oop in case of safepoint in resume_operation (if posting JVMTI events). + JVMTI_ONLY(assert(_thread->contended_entered_monitor() == nullptr || _thread->contended_entered_monitor() == mon, "")); } else { - // Preemption cancelled in moniterenter case. We actually acquired - // the monitor after freezing all frames so nothing to do. - preempt_kind = Continuation::freeze_on_monitorenter; + // Preemption cancelled on moniterenter or ObjectLocker case. We + // actually acquired the monitor after freezing all frames so no + // need to call resume_operation. If this is the ObjectLocker case + // we released the monitor already at ~ObjectLocker, so _init_lock + // will be set to nullptr below since there is no monitor to release. + preempt_kind = Continuation::monitorenter; } + // Call this first to avoid racing with GC threads later when modifying the chunk flags. relativize_chunk_concurrently(chunk); + + if (chunk->at_klass_init()) { + preempt_kind = Continuation::object_locker; + chunk->set_at_klass_init(false); + _process_args_at_top = chunk->has_args_at_top(); + if (_process_args_at_top) { + // Only needed for the top frame which will be thawed. + chunk->set_has_args_at_top(false); + } + assert(waiter == nullptr || mon != nullptr, "should have a monitor"); + _init_lock = mon; // remember monitor since we will need it on handle_preempted_continuation() + } chunk->set_preempted(false); retry_fast_path = true; } else { @@ -2333,7 +2506,7 @@ void ThawBase::recurse_thaw(const frame& heap_frame, frame& caller, int num_fram } else if (!heap_frame.is_interpreted_frame()) { recurse_thaw_compiled_frame(heap_frame, caller, num_frames, false); } else { - recurse_thaw_interpreted_frame(heap_frame, caller, num_frames); + recurse_thaw_interpreted_frame(heap_frame, caller, num_frames, top_on_preempt_case); } } @@ -2345,7 +2518,7 @@ bool ThawBase::recurse_thaw_java_frame(frame& caller, int num_frames) { int argsize = _stream.stack_argsize(); - _stream.next(SmallRegisterMap::instance()); + _stream.next(SmallRegisterMap::instance_no_args()); assert(_stream.to_frame().is_empty() == _stream.is_done(), ""); // we never leave a compiled caller of an interpreted frame as the top frame in the chunk @@ -2450,9 +2623,10 @@ void ThawBase::clear_bitmap_bits(address start, address end) { } intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::preempt_kind preempt_kind, bool fast_case) { - assert(preempt_kind == Continuation::freeze_on_wait || preempt_kind == Continuation::freeze_on_monitorenter, ""); frame top(sp); assert(top.pc() == *(address*)(sp - frame::sender_sp_ret_address_offset()), ""); + DEBUG_ONLY(verify_frame_kind(top, preempt_kind);) + NOT_PRODUCT(int64_t tid = _thread->monitor_owner_id();) #if INCLUDE_JVMTI // Finish the VTMS transition. @@ -2460,7 +2634,7 @@ intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::pr bool is_vthread = Continuation::continuation_scope(_cont.continuation()) == java_lang_VirtualThread::vthread_scope(); if (is_vthread) { if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) { - jvmti_mount_end(_thread, _cont, top); + jvmti_mount_end(_thread, _cont, top, preempt_kind); } else { _thread->set_is_in_VTMS_transition(false); java_lang_Thread::set_is_in_VTMS_transition(_thread->vthread(), false); @@ -2477,36 +2651,103 @@ intptr_t* ThawBase::handle_preempted_continuation(intptr_t* sp, Continuation::pr patch_pd(top, sp + fsize); } - if (preempt_kind == Continuation::freeze_on_wait) { + if (preempt_kind == Continuation::object_wait) { // Check now if we need to throw IE exception. - if (_thread->pending_interrupted_exception()) { + bool throw_ie = _thread->pending_interrupted_exception(); + if (throw_ie) { throw_interrupted_exception(_thread, top); _thread->set_pending_interrupted_exception(false); } - } else if (top.is_runtime_frame()) { - // The continuation might now run on a different platform thread than the previous time so - // we need to adjust the current thread saved in the stub frame before restoring registers. - JavaThread** thread_addr = frame::saved_thread_address(top); - if (thread_addr != nullptr) *thread_addr = _thread; + log_develop_trace(continuations, preempt)("Resuming " INT64_FORMAT" after preemption on Object.wait%s", tid, throw_ie ? "(throwing IE)" : ""); + } else if (preempt_kind == Continuation::monitorenter) { + if (top.is_runtime_frame()) { + // The continuation might now run on a different platform thread than the previous time so + // we need to adjust the current thread saved in the stub frame before restoring registers. + JavaThread** thread_addr = frame::saved_thread_address(top); + if (thread_addr != nullptr) *thread_addr = _thread; + } + log_develop_trace(continuations, preempt)("Resuming " INT64_FORMAT " after preemption on monitorenter", tid); + } else { + // We need to redo the original call into the VM. First though, we need + // to exit the monitor we just acquired (except on preemption cancelled + // case where it was already released). + assert(preempt_kind == Continuation::object_locker, ""); + if (_init_lock != nullptr) _init_lock->exit(_thread); + sp = redo_vmcall(_thread, top); + } + return sp; +} + +intptr_t* ThawBase::redo_vmcall(JavaThread* current, frame& top) { + assert(!current->preempting(), ""); + NOT_PRODUCT(int64_t tid = current->monitor_owner_id();) + intptr_t* sp = top.sp(); + + { + HandleMarkCleaner hmc(current); // Cleanup all handles (including so._conth) before returning to Java. + ContinuationWrapper::SafepointOp so(current, _cont); + AnchorMark am(current, top); // Set the anchor so that the stack is walkable. + + Method* m = top.interpreter_frame_method(); + Bytecode current_bytecode = Bytecode(m, top.interpreter_frame_bcp()); + Bytecodes::Code code = current_bytecode.code(); + log_develop_trace(continuations, preempt)("Redoing InterpreterRuntime::%s for " INT64_FORMAT, code == Bytecodes::Code::_new ? "_new" : "resolve_from_cache", tid); + + // These InterpreterRuntime entry points use JRT_ENTRY which uses a HandleMarkCleaner. + // Create a HandeMark to avoid destroying so._conth. + HandleMark hm(current); + DEBUG_ONLY(JavaThread::AtRedoVMCall apvmc(current);) + if (code == Bytecodes::Code::_new) { + InterpreterRuntime::_new(current, m->constants(), current_bytecode.get_index_u2(code)); + } else { + InterpreterRuntime::resolve_from_cache(current, code); + } + } + + if (current->preempting()) { + // Preempted again so we just arrange to return to preempt stub to unmount. + sp = push_preempt_adapter(); + current->set_preempt_alternate_return(nullptr); + bool cancelled = current->preemption_cancelled(); + if (cancelled) { + // Since preemption was cancelled, the thread will call thaw again from the preempt + // stub. These retries could happen several times due to contention on the init_lock, + // so just let the vthread umount to give a chance for other vthreads to run. + current->set_preemption_cancelled(false); + oop vthread = current->vthread(); + assert(java_lang_VirtualThread::state(vthread) == java_lang_VirtualThread::RUNNING, "wrong state for vthread"); + java_lang_VirtualThread::set_state(vthread, java_lang_VirtualThread::YIELDING); +#if INCLUDE_JVMTI + if (current->contended_entered_monitor() != nullptr) { + current->set_contended_entered_monitor(nullptr); + } +#endif + } + log_develop_trace(continuations, preempt)("Preempted " INT64_FORMAT " again%s", tid, cancelled ? "(preemption cancelled, setting state to YIELDING)" : ""); + } else { + log_develop_trace(continuations, preempt)("Call succesful, resuming " INT64_FORMAT, tid); } return sp; } void ThawBase::throw_interrupted_exception(JavaThread* current, frame& top) { + HandleMarkCleaner hm(current); // Cleanup all handles (including so._conth) before returning to Java. ContinuationWrapper::SafepointOp so(current, _cont); - // Since we might safepoint set the anchor so that the stack can be walked. - set_anchor(current, top.sp()); + AnchorMark am(current, top); // Set the anchor so that the stack is walkable. JRT_BLOCK THROW(vmSymbols::java_lang_InterruptedException()); JRT_BLOCK_END - clear_anchor(current); } -NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames) { +NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& caller, int num_frames, bool is_top) { assert(hf.is_interpreted_frame(), ""); if (UNLIKELY(seen_by_gc())) { - _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance()); + if (is_top && _process_args_at_top) { + _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance_with_args()); + } else { + _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance_no_args()); + } } const bool is_bottom_frame = recurse_thaw_java_frame(caller, num_frames); @@ -2551,7 +2792,7 @@ NOINLINE void ThawBase::recurse_thaw_interpreted_frame(const frame& hf, frame& c if (!is_bottom_frame) { // can only fix caller once this frame is thawed (due to callee saved regs) - _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance()); + _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance_no_args()); } else if (_cont.tail()->has_bitmap() && locals > 0) { assert(hf.is_heap_frame(), "should be"); address start = (address)(heap_frame_bottom - locals); @@ -2568,7 +2809,7 @@ void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int n assert(_preempted_case || !stub_caller, "stub caller not at preemption"); if (!stub_caller && UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap - _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance()); + _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance_no_args()); } const bool is_bottom_frame = recurse_thaw_java_frame(caller, num_frames); @@ -2627,7 +2868,7 @@ void ThawBase::recurse_thaw_compiled_frame(const frame& hf, frame& caller, int n if (!is_bottom_frame) { // can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack - _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance()); + _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance_no_args()); } else if (_cont.tail()->has_bitmap() && added_argsize > 0) { address start = (address)(heap_frame_top + ContinuationHelper::CompiledFrame::size(hf) + frame::metadata_words_at_top); int stack_args_slots = f.cb()->as_nmethod()->num_stack_arg_slots(false /* rounded */); @@ -2653,7 +2894,7 @@ void ThawBase::recurse_thaw_stub_frame(const frame& hf, frame& caller, int num_f assert(!_stream.is_done(), ""); _cont.tail()->do_barriers(_stream, &map); } else { - _stream.next(SmallRegisterMap::instance()); + _stream.next(SmallRegisterMap::instance_no_args()); assert(!_stream.is_done(), ""); } @@ -2693,7 +2934,7 @@ void ThawBase::recurse_thaw_native_frame(const frame& hf, frame& caller, int num assert(_preempted_case && hf.cb()->as_nmethod()->method()->is_object_wait0(), ""); if (UNLIKELY(seen_by_gc())) { // recurse_thaw_stub_frame already invoked our barriers with a full regmap - _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance()); + _cont.tail()->do_barriers(_stream, SmallRegisterMap::instance_no_args()); } const bool is_bottom_frame = recurse_thaw_java_frame(caller, num_frames); @@ -2731,7 +2972,7 @@ void ThawBase::recurse_thaw_native_frame(const frame& hf, frame& caller, int num assert(!f.cb()->as_nmethod()->is_marked_for_deoptimization(), ""); // can only fix caller once this frame is thawed (due to callee saved regs); this happens on the stack - _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance()); + _cont.tail()->fix_thawed_frame(caller, SmallRegisterMap::instance_no_args()); DEBUG_ONLY(after_thaw_java_frame(f, false /* bottom */);) caller = f; @@ -2758,7 +2999,12 @@ void ThawBase::finish_thaw(frame& f) { f.set_sp(align_down(f.sp(), frame::frame_alignment)); } push_return_frame(f); - chunk->fix_thawed_frame(f, SmallRegisterMap::instance()); // can only fix caller after push_return_frame (due to callee saved regs) + // can only fix caller after push_return_frame (due to callee saved regs) + if (_process_args_at_top) { + chunk->fix_thawed_frame(f, SmallRegisterMap::instance_with_args()); + } else { + chunk->fix_thawed_frame(f, SmallRegisterMap::instance_no_args()); + } assert(_cont.is_empty() == _cont.last_frame().is_empty(), ""); @@ -2772,7 +3018,7 @@ void ThawBase::finish_thaw(frame& f) { } } -void ThawBase::push_return_frame(frame& f) { // see generate_cont_thaw +void ThawBase::push_return_frame(const frame& f) { // see generate_cont_thaw assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == f.cb()->as_nmethod()->is_deopt_pc(f.raw_pc()), ""); assert(!f.is_compiled_frame() || f.is_deoptimized_frame() == (f.pc() != f.raw_pc()), ""); @@ -2820,11 +3066,10 @@ static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::th clear_anchor(thread); #endif - DEBUG_ONLY(bool preempted = cont.tail()->preempted();) Thaw thw(thread, cont); intptr_t* const sp = thw.thaw(kind); assert(is_aligned(sp, frame::frame_alignment), ""); - DEBUG_ONLY(log_frames_after_thaw(thread, cont, sp, preempted);) + DEBUG_ONLY(log_frames_after_thaw(thread, cont, sp);) CONT_JFR_ONLY(thw.jfr_info().post_jfr_event(&event, cont.continuation(), thread);) @@ -2966,14 +3211,16 @@ static void log_frames(JavaThread* thread) { ls.print_cr("======= end frames ========="); } -static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp, bool preempted) { +static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, intptr_t* sp) { intptr_t* sp0 = sp; address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset()); - if (preempted && sp0 == cont.entrySP()) { + bool preempted = false; + stackChunkOop tail = cont.tail(); + if (tail != nullptr && tail->preempted()) { // Still preempted (monitor not acquired) so no frames were thawed. - assert(cont.tail()->preempted(), ""); set_anchor(thread, cont.entrySP(), cont.entryPC()); + preempted = true; } else { set_anchor(thread, sp0); } @@ -2982,7 +3229,7 @@ static void log_frames_after_thaw(JavaThread* thread, ContinuationWrapper& cont, if (LoomVerifyAfterThaw) { assert(do_verify_after_thaw(thread, cont.tail(), tty), ""); } - assert(ContinuationEntry::assert_entry_frame_laid_out(thread), ""); + assert(preempted || ContinuationEntry::assert_entry_frame_laid_out(thread), ""); clear_anchor(thread); LogTarget(Trace, continuations) lt; diff --git a/src/hotspot/share/runtime/continuationJavaClasses.cpp b/src/hotspot/share/runtime/continuationJavaClasses.cpp index a5b7ea4ad9368..6e4e52e010773 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.cpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.cpp @@ -87,6 +87,8 @@ int jdk_internal_vm_StackChunk::_bottom_offset; int jdk_internal_vm_StackChunk::_flags_offset; int jdk_internal_vm_StackChunk::_maxThawingSize_offset; int jdk_internal_vm_StackChunk::_lockStackSize_offset; +int jdk_internal_vm_StackChunk::_atKlassInit_offset; +int jdk_internal_vm_StackChunk::_hasArgsAtTop_offset; int jdk_internal_vm_StackChunk::_cont_offset; #define STACKCHUNK_FIELDS_DO(macro) \ diff --git a/src/hotspot/share/runtime/continuationJavaClasses.hpp b/src/hotspot/share/runtime/continuationJavaClasses.hpp index f9cd53ff9f05b..91224b94e1ec7 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.hpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.hpp @@ -76,6 +76,8 @@ class jdk_internal_vm_Continuation: AllStatic { macro(jdk_internal_vm_StackChunk, pc, intptr_signature, false) \ macro(jdk_internal_vm_StackChunk, maxThawingSize, int_signature, false) \ macro(jdk_internal_vm_StackChunk, lockStackSize, byte_signature, false) \ + macro(jdk_internal_vm_StackChunk, atKlassInit, bool_signature, false) \ + macro(jdk_internal_vm_StackChunk, hasArgsAtTop, bool_signature, false) \ class jdk_internal_vm_StackChunk: AllStatic { friend class JavaClasses; @@ -88,6 +90,8 @@ class jdk_internal_vm_StackChunk: AllStatic { static int _flags_offset; static int _maxThawingSize_offset; static int _lockStackSize_offset; + static int _atKlassInit_offset; + static int _hasArgsAtTop_offset; static int _cont_offset; @@ -129,6 +133,12 @@ class jdk_internal_vm_StackChunk: AllStatic { static inline uint8_t lockStackSize(oop chunk); static inline void set_lockStackSize(oop chunk, uint8_t value); + static inline bool atKlassInit(oop chunk); + static inline void set_atKlassInit(oop chunk, bool value); + + static inline bool hasArgsAtTop(oop chunk); + static inline void set_hasArgsAtTop(oop chunk, bool value); + // cont oop's processing is essential for the chunk's GC protocol static inline oop cont(oop chunk); template diff --git a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp index f464648cdc3fa..c6d8065218055 100644 --- a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp +++ b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp @@ -194,4 +194,20 @@ inline void jdk_internal_vm_StackChunk::set_lockStackSize(oop chunk, uint8_t val AtomicAccess::store(chunk->field_addr(_lockStackSize_offset), value); } +inline bool jdk_internal_vm_StackChunk::atKlassInit(oop chunk) { + return chunk->bool_field(_atKlassInit_offset); +} + +inline void jdk_internal_vm_StackChunk::set_atKlassInit(oop chunk, bool value) { + chunk->bool_field_put(_atKlassInit_offset, (jboolean)value); +} + +inline bool jdk_internal_vm_StackChunk::hasArgsAtTop(oop chunk) { + return chunk->bool_field(_hasArgsAtTop_offset); +} + +inline void jdk_internal_vm_StackChunk::set_hasArgsAtTop(oop chunk, bool value) { + chunk->bool_field_put(_hasArgsAtTop_offset, (jboolean)value); +} + #endif // SHARE_RUNTIME_CONTINUATIONJAVACLASSES_INLINE_HPP diff --git a/src/hotspot/share/runtime/frame.cpp b/src/hotspot/share/runtime/frame.cpp index e578e614440ae..b5cd4acc75d44 100644 --- a/src/hotspot/share/runtime/frame.cpp +++ b/src/hotspot/share/runtime/frame.cpp @@ -891,7 +891,8 @@ oop frame::interpreter_callee_receiver(Symbol* signature) { return *interpreter_callee_receiver_addr(signature); } -void frame::oops_interpreted_do(OopClosure* f, const RegisterMap* map, bool query_oop_map_cache) const { +template +void frame::oops_interpreted_do(OopClosure* f, const RegisterMapT* map, bool query_oop_map_cache) const { assert(is_interpreted_frame(), "Not an interpreted frame"); Thread *thread = Thread::current(); methodHandle m (thread, interpreter_frame_method()); @@ -928,33 +929,19 @@ void frame::oops_interpreted_do(OopClosure* f, const RegisterMap* map, bool quer int max_locals = m->is_native() ? m->size_of_parameters() : m->max_locals(); - Symbol* signature = nullptr; - bool has_receiver = false; - // Process a callee's arguments if we are at a call site // (i.e., if we are at an invoke bytecode) // This is used sometimes for calling into the VM, not for another // interpreted or compiled frame. - if (!m->is_native()) { + if (!m->is_native() && map != nullptr && map->include_argument_oops()) { Bytecode_invoke call = Bytecode_invoke_check(m, bci); - if (map != nullptr && call.is_valid()) { - signature = call.signature(); - has_receiver = call.has_receiver(); - if (map->include_argument_oops() && - interpreter_frame_expression_stack_size() > 0) { - ResourceMark rm(thread); // is this right ??? - // we are at a call site & the expression stack is not empty - // => process callee's arguments - // - // Note: The expression stack can be empty if an exception - // occurred during method resolution/execution. In all - // cases we empty the expression stack completely be- - // fore handling the exception (the exception handling - // code in the interpreter calls a blocking runtime - // routine which can cause this code to be executed). - // (was bug gri 7/27/98) - oops_interpreted_arguments_do(signature, has_receiver, f); - } + if (call.is_valid() && interpreter_frame_expression_stack_size() > 0) { + ResourceMark rm(thread); // is this right ??? + Symbol* signature = call.signature(); + bool has_receiver = call.has_receiver(); + // We are at a call site & the expression stack is not empty + // so we might have callee arguments we need to process. + oops_interpreted_arguments_do(signature, has_receiver, f); } } @@ -970,6 +957,9 @@ void frame::oops_interpreted_do(OopClosure* f, const RegisterMap* map, bool quer mask.iterate_oop(&blk); } +template void frame::oops_interpreted_do(OopClosure* f, const RegisterMap* map, bool query_oop_map_cache) const; +template void frame::oops_interpreted_do(OopClosure* f, const SmallRegisterMapNoArgs* map, bool query_oop_map_cache) const; +template void frame::oops_interpreted_do(OopClosure* f, const SmallRegisterMapWithArgs* map, bool query_oop_map_cache) const; void frame::oops_interpreted_arguments_do(Symbol* signature, bool has_receiver, OopClosure* f) const { InterpretedArgumentOopFinder finder(signature, has_receiver, this, f); diff --git a/src/hotspot/share/runtime/frame.hpp b/src/hotspot/share/runtime/frame.hpp index fbe7310f3ae8d..f54553086f6b6 100644 --- a/src/hotspot/share/runtime/frame.hpp +++ b/src/hotspot/share/runtime/frame.hpp @@ -456,7 +456,8 @@ class frame { // Oops-do's void oops_compiled_arguments_do(Symbol* signature, bool has_receiver, bool has_appendix, const RegisterMap* reg_map, OopClosure* f) const; - void oops_interpreted_do(OopClosure* f, const RegisterMap* map, bool query_oop_map_cache = true) const; + template + void oops_interpreted_do(OopClosure* f, const RegisterMapT* map, bool query_oop_map_cache = true) const; private: void oops_interpreted_arguments_do(Symbol* signature, bool has_receiver, OopClosure* f) const; diff --git a/src/hotspot/share/runtime/javaCalls.cpp b/src/hotspot/share/runtime/javaCalls.cpp index ec3132220d967..3ed678284e4a7 100644 --- a/src/hotspot/share/runtime/javaCalls.cpp +++ b/src/hotspot/share/runtime/javaCalls.cpp @@ -58,6 +58,7 @@ JavaCallWrapper::JavaCallWrapper(const methodHandle& callee_method, Handle recei guarantee(thread->is_Java_thread(), "crucial check - the VM thread cannot and must not escape to Java code"); assert(!thread->owns_locks(), "must release all locks when leaving VM"); guarantee(thread->can_call_java(), "cannot make java calls from the native compiler"); + assert(!thread->preempting(), "Unexpected Java upcall whilst processing preemption"); _result = result; // Allocate handle block for Java code. This must be done before we change thread_state to _thread_in_Java_or_stub, @@ -242,7 +243,7 @@ void JavaCalls::call_special(JavaValue* result, Handle receiver, Klass* klass, S void JavaCalls::call_static(JavaValue* result, Klass* klass, Symbol* name, Symbol* signature, JavaCallArguments* args, TRAPS) { CallInfo callinfo; LinkInfo link_info(klass, name, signature); - LinkResolver::resolve_static_call(callinfo, link_info, true, CHECK); + LinkResolver::resolve_static_call(callinfo, link_info, ClassInitMode::init, CHECK); methodHandle method(THREAD, callinfo.selected_method()); assert(method.not_null(), "should have thrown exception"); diff --git a/src/hotspot/share/runtime/javaThread.cpp b/src/hotspot/share/runtime/javaThread.cpp index 36544cf1118ea..dcf92c387be8e 100644 --- a/src/hotspot/share/runtime/javaThread.cpp +++ b/src/hotspot/share/runtime/javaThread.cpp @@ -493,6 +493,10 @@ JavaThread::JavaThread(MemTag mem_tag) : _preempt_alternate_return(nullptr), _preemption_cancelled(false), _pending_interrupted_exception(false), + _at_preemptable_init(false), + DEBUG_ONLY(_preempt_init_klass(nullptr) COMMA) + DEBUG_ONLY(_interp_at_preemptable_vmcall_cnt(0) COMMA) + DEBUG_ONLY(_interp_redoing_vm_call(false) COMMA) _handshake(this), _suspend_resume_manager(this, &_handshake._lock), diff --git a/src/hotspot/share/runtime/javaThread.hpp b/src/hotspot/share/runtime/javaThread.hpp index a6a00bfbd033e..612a82ee09fbc 100644 --- a/src/hotspot/share/runtime/javaThread.hpp +++ b/src/hotspot/share/runtime/javaThread.hpp @@ -484,6 +484,9 @@ class JavaThread: public Thread { // For Object.wait() we set this field to know if we need to // throw IE at the end of thawing before returning to Java. bool _pending_interrupted_exception; + // We allow preemption on some klass initialization calls. + // We use this boolean to mark such calls. + bool _at_preemptable_init; public: bool preemption_cancelled() { return _preemption_cancelled; } @@ -492,11 +495,46 @@ class JavaThread: public Thread { bool pending_interrupted_exception() { return _pending_interrupted_exception; } void set_pending_interrupted_exception(bool b) { _pending_interrupted_exception = b; } - bool preempting() { return _preempt_alternate_return != nullptr; } + bool preempting() { return _preempt_alternate_return != nullptr; } void set_preempt_alternate_return(address val) { _preempt_alternate_return = val; } -private: + bool at_preemptable_init() { return _at_preemptable_init; } + void set_at_preemptable_init(bool b) { _at_preemptable_init = b; } + +#ifdef ASSERT + // Used for extra logging with -Xlog:continuation+preempt + InstanceKlass* _preempt_init_klass; + + InstanceKlass* preempt_init_klass() { return _preempt_init_klass; } + void set_preempt_init_klass(InstanceKlass* ik) { _preempt_init_klass = ik; } + int _interp_at_preemptable_vmcall_cnt; + int interp_at_preemptable_vmcall_cnt() { return _interp_at_preemptable_vmcall_cnt; } + + bool _interp_redoing_vm_call; + bool interp_redoing_vm_call() const { return _interp_redoing_vm_call; }; + + class AtRedoVMCall : public StackObj { + JavaThread* _thread; + public: + AtRedoVMCall(JavaThread* t) : _thread(t) { + assert(!_thread->_interp_redoing_vm_call, ""); + _thread->_interp_redoing_vm_call = true; + _thread->_interp_at_preemptable_vmcall_cnt++; + assert(_thread->_interp_at_preemptable_vmcall_cnt > 0, "Unexpected count: %d", + _thread->_interp_at_preemptable_vmcall_cnt); + } + ~AtRedoVMCall() { + assert(_thread->_interp_redoing_vm_call, ""); + _thread->_interp_redoing_vm_call = false; + _thread->_interp_at_preemptable_vmcall_cnt--; + assert(_thread->_interp_at_preemptable_vmcall_cnt >= 0, "Unexpected count: %d", + _thread->_interp_at_preemptable_vmcall_cnt); + } + }; +#endif // ASSERT + +private: friend class VMThread; friend class ThreadWaitTransition; friend class VM_Exit; @@ -881,6 +919,7 @@ class JavaThread: public Thread { static ByteSize cont_fastpath_offset() { return byte_offset_of(JavaThread, _cont_fastpath); } static ByteSize preemption_cancelled_offset() { return byte_offset_of(JavaThread, _preemption_cancelled); } static ByteSize preempt_alternate_return_offset() { return byte_offset_of(JavaThread, _preempt_alternate_return); } + DEBUG_ONLY(static ByteSize interp_at_preemptable_vmcall_cnt_offset() { return byte_offset_of(JavaThread, _interp_at_preemptable_vmcall_cnt); }) static ByteSize unlocked_inflated_monitor_offset() { return byte_offset_of(JavaThread, _unlocked_inflated_monitor); } #if INCLUDE_JVMTI @@ -1326,8 +1365,8 @@ class NoPreemptMark { ContinuationEntry* _ce; bool _unpin; public: - NoPreemptMark(JavaThread* thread) : _ce(thread->last_continuation()), _unpin(false) { - if (_ce != nullptr) _unpin = _ce->pin(); + NoPreemptMark(JavaThread* thread, bool ignore_mark = false) : _ce(thread->last_continuation()), _unpin(false) { + if (_ce != nullptr && !ignore_mark) _unpin = _ce->pin(); } ~NoPreemptMark() { if (_unpin) _ce->unpin(); } }; diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp index f6d569b1b7a50..6a99568ba44a2 100644 --- a/src/hotspot/share/runtime/objectMonitor.cpp +++ b/src/hotspot/share/runtime/objectMonitor.cpp @@ -304,6 +304,7 @@ ObjectMonitor::ObjectMonitor(oop object) : ObjectMonitor::~ObjectMonitor() { _object.release(_oop_storage); + _object_strong.release(JavaThread::thread_oop_storage()); } oop ObjectMonitor::object() const { @@ -311,6 +312,20 @@ oop ObjectMonitor::object() const { return _object.resolve(); } +// Keep object protected during ObjectLocker preemption. +void ObjectMonitor::set_object_strong() { + check_object_context(); + if (_object_strong.is_empty()) { + if (AtomicAccess::cmpxchg(&_object_strong_lock, 0, 1) == 0) { + if (_object_strong.is_empty()) { + assert(_object.resolve() != nullptr, ""); + _object_strong = OopHandle(JavaThread::thread_oop_storage(), _object.resolve()); + } + AtomicAccess::release_store(&_object_strong_lock, 0); + } + } +} + void ObjectMonitor::ExitOnSuspend::operator()(JavaThread* current) { if (current->is_suspended()) { _om->_recursions = 0; @@ -1813,7 +1828,7 @@ void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { current->set_current_waiting_monitor(this); result = Continuation::try_preempt(current, ce->cont_oop(current)); if (result == freeze_ok) { - vthread_wait(current, millis); + vthread_wait(current, millis, interruptible); current->set_current_waiting_monitor(nullptr); return; } @@ -2167,12 +2182,14 @@ void ObjectMonitor::quick_notifyAll(JavaThread* current) { } } -void ObjectMonitor::vthread_wait(JavaThread* current, jlong millis) { +void ObjectMonitor::vthread_wait(JavaThread* current, jlong millis, bool interruptible) { oop vthread = current->vthread(); ObjectWaiter* node = new ObjectWaiter(vthread, this); node->_is_wait = true; + node->_interruptible = interruptible; node->TState = ObjectWaiter::TS_WAIT; java_lang_VirtualThread::set_notified(vthread, false); // Reset notified flag + java_lang_VirtualThread::set_interruptible_wait(vthread, interruptible); // Enter the waiting queue, which is a circular doubly linked list in this case // but it could be a priority queue or any data structure. @@ -2217,11 +2234,12 @@ bool ObjectMonitor::vthread_wait_reenter(JavaThread* current, ObjectWaiter* node ObjectWaiter::TStates state = node->TState; bool was_notified = state == ObjectWaiter::TS_ENTER; assert(was_notified || state == ObjectWaiter::TS_RUN, ""); - node->_interrupted = !was_notified && current->is_interrupted(false); + node->_interrupted = node->_interruptible && !was_notified && current->is_interrupted(false); - // Post JFR and JVMTI events. + // Post JFR and JVMTI events. If non-interruptible we are in + // ObjectLocker case so we don't post anything. EventJavaMonitorWait wait_event; - if (wait_event.should_commit() || JvmtiExport::should_post_monitor_waited()) { + if (node->_interruptible && (wait_event.should_commit() || JvmtiExport::should_post_monitor_waited())) { vthread_monitor_waited_event(current, node, cont, &wait_event, !was_notified && !node->_interrupted); } diff --git a/src/hotspot/share/runtime/objectMonitor.hpp b/src/hotspot/share/runtime/objectMonitor.hpp index 77919d9995586..2da41786309b6 100644 --- a/src/hotspot/share/runtime/objectMonitor.hpp +++ b/src/hotspot/share/runtime/objectMonitor.hpp @@ -55,6 +55,7 @@ class ObjectWaiter : public CHeapObj { bool _is_wait; bool _at_reenter; bool _interrupted; + bool _interruptible; bool _do_timed_park; bool _active; // Contention monitoring is enabled public: @@ -200,10 +201,12 @@ class ObjectMonitor : public CHeapObj { // ObjectMonitor::deflate_monitor(). int64_t _unmounted_vthreads; // Number of nodes in the _entry_list associated with unmounted vthreads. // It might be temporarily more than the actual number but never less. + OopHandle _object_strong; // Used to protect object during preemption on class initialization ObjectWaiter* volatile _wait_set; // LL of threads waiting on the monitor - wait() volatile int _waiters; // number of waiting threads volatile int _wait_set_lock; // protects wait set queue - simple spinlock + volatile int _object_strong_lock; // protects setting of _object_strong public: @@ -343,6 +346,7 @@ class ObjectMonitor : public CHeapObj { oop object_peek() const; bool object_is_dead() const; bool object_refers_to(oop obj) const; + void set_object_strong(); // Returns true if the specified thread owns the ObjectMonitor. Otherwise // returns false and throws IllegalMonitorStateException (IMSE). @@ -405,7 +409,7 @@ class ObjectMonitor : public CHeapObj { ObjectWaiter* entry_list_tail(JavaThread* current); bool vthread_monitor_enter(JavaThread* current, ObjectWaiter* node = nullptr); - void vthread_wait(JavaThread* current, jlong millis); + void vthread_wait(JavaThread* current, jlong millis, bool interruptible); bool vthread_wait_reenter(JavaThread* current, ObjectWaiter* node, ContinuationWrapper& cont); void vthread_epilog(JavaThread* current, ObjectWaiter* node); diff --git a/src/hotspot/share/runtime/smallRegisterMap.inline.hpp b/src/hotspot/share/runtime/smallRegisterMap.inline.hpp index e0de6acbd6d9d..09febdaf450cd 100644 --- a/src/hotspot/share/runtime/smallRegisterMap.inline.hpp +++ b/src/hotspot/share/runtime/smallRegisterMap.inline.hpp @@ -29,4 +29,19 @@ #include CPU_HEADER_INLINE(smallRegisterMap) +typedef SmallRegisterMapType SmallRegisterMapNoArgs; +typedef SmallRegisterMapType SmallRegisterMapWithArgs; + +class SmallRegisterMap : AllStatic { +public: + static const SmallRegisterMapNoArgs* instance_no_args() { + static constexpr SmallRegisterMapNoArgs the_instance{}; + return &the_instance; + } + static const SmallRegisterMapWithArgs* instance_with_args() { + static constexpr SmallRegisterMapWithArgs the_instance_with_args{}; + return &the_instance_with_args; + } +}; + #endif // SHARE_RUNTIME_SMALLREGISTERMAP_HPP diff --git a/src/hotspot/share/runtime/stackChunkFrameStream.hpp b/src/hotspot/share/runtime/stackChunkFrameStream.hpp index 207203dbb3509..cb46254d1251a 100644 --- a/src/hotspot/share/runtime/stackChunkFrameStream.hpp +++ b/src/hotspot/share/runtime/stackChunkFrameStream.hpp @@ -26,6 +26,7 @@ #define SHARE_RUNTIME_STACKCHUNKFRAMESTREAM_HPP #include "memory/allocation.hpp" +#include "memory/iterator.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" @@ -79,7 +80,8 @@ class StackChunkFrameStream : public StackObj { const ImmutableOopMap* oopmap() const { if (_oopmap == nullptr) get_oopmap(); return _oopmap; } inline int frame_size() const; inline int stack_argsize() const; - inline int num_oops() const; + template + inline int num_oops(RegisterMapT* map) const; inline void initialize_register_map(RegisterMap* map); template inline void next(RegisterMapT* map, bool stop = false); @@ -101,7 +103,8 @@ class StackChunkFrameStream : public StackObj { inline address get_pc() const; inline int interpreter_frame_size() const; - inline int interpreter_frame_num_oops() const; + template + inline int interpreter_frame_num_oops(RegisterMapT* map) const; inline int interpreter_frame_stack_argsize() const; inline void next_for_interpreter_frame(); inline intptr_t* unextended_sp_for_interpreter_frame() const; @@ -123,4 +126,13 @@ class StackChunkFrameStream : public StackObj { inline void iterate_derived_pointers(DerivedOopClosureType* closure, const RegisterMapT* map) const; }; +class InterpreterOopCount : public OopClosure { + int _count; +public: + InterpreterOopCount() : _count(0) {} + void do_oop(oop* p) override { _count++; } + void do_oop(narrowOop* p) override { _count++; } + int count() { return _count; } +}; + #endif // SHARE_RUNTIME_STACKCHUNKFRAMESTREAM_HPP diff --git a/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp b/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp index 09c267b408b6f..97166950752ad 100644 --- a/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp +++ b/src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp @@ -195,9 +195,10 @@ inline int StackChunkFrameStream::stack_argsize() const { } template -inline int StackChunkFrameStream::num_oops() const { +template +inline int StackChunkFrameStream::num_oops(RegisterMapT* map) const { if (is_interpreted()) { - return interpreter_frame_num_oops(); + return interpreter_frame_num_oops(map); } else if (is_compiled()) { return oopmap()->num_oops(); } else { @@ -365,7 +366,7 @@ template inline void StackChunkFrameStream::iterate_oops(OopClosureType* closure, const RegisterMapT* map) const { if (is_interpreted()) { frame f = to_frame(); - f.oops_interpreted_do(closure, nullptr, true); + f.oops_interpreted_do(closure, map, true); } else { DEBUG_ONLY(int oops = 0;) for (OopMapStream oms(oopmap()); !oms.is_done(); oms.next()) { diff --git a/src/hotspot/share/runtime/stackValue.cpp b/src/hotspot/share/runtime/stackValue.cpp index 52d2631f3c94f..fc810a1fa80f6 100644 --- a/src/hotspot/share/runtime/stackValue.cpp +++ b/src/hotspot/share/runtime/stackValue.cpp @@ -38,7 +38,7 @@ class RegisterMap; class SmallRegisterMap; template StackValue* StackValue::create_stack_value(const frame* fr, const RegisterMap* reg_map, ScopeValue* sv); -template StackValue* StackValue::create_stack_value(const frame* fr, const SmallRegisterMap* reg_map, ScopeValue* sv); +template StackValue* StackValue::create_stack_value(const frame* fr, const SmallRegisterMapNoArgs* reg_map, ScopeValue* sv); template StackValue* StackValue::create_stack_value(const frame* fr, const RegisterMapT* reg_map, ScopeValue* sv) { @@ -257,7 +257,7 @@ StackValue* StackValue::create_stack_value(ScopeValue* sv, address value_addr, c } template address StackValue::stack_value_address(const frame* fr, const RegisterMap* reg_map, ScopeValue* sv); -template address StackValue::stack_value_address(const frame* fr, const SmallRegisterMap* reg_map, ScopeValue* sv); +template address StackValue::stack_value_address(const frame* fr, const SmallRegisterMapNoArgs* reg_map, ScopeValue* sv); template address StackValue::stack_value_address(const frame* fr, const RegisterMapT* reg_map, ScopeValue* sv) { diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index e513c57fe069f..36e38b4dd35c2 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -475,22 +475,45 @@ void ObjectSynchronizer::jni_exit(oop obj, TRAPS) { // ----------------------------------------------------------------------------- // Internal VM locks on java objects // standard constructor, allows locking failures -ObjectLocker::ObjectLocker(Handle obj, JavaThread* thread) : _npm(thread) { - _thread = thread; +ObjectLocker::ObjectLocker(Handle obj, TRAPS) : _thread(THREAD), _obj(obj), + _npm(_thread, _thread->at_preemptable_init() /* ignore_mark */), _skip_exit(false) { + assert(!_thread->preempting(), ""); + _thread->check_for_valid_safepoint_state(); - _obj = obj; if (_obj() != nullptr) { ObjectSynchronizer::enter(_obj, &_lock, _thread); + + if (_thread->preempting()) { + // If preemption was cancelled we acquired the monitor after freezing + // the frames. Redoing the vm call later in thaw will require us to + // release it since the call should look like the original one. We + // do it in ~ObjectLocker to reduce the window of time we hold the + // monitor since we can't do anything useful with it now, and would + // otherwise just force other vthreads to preempt in case they try + // to acquire this monitor. + _skip_exit = !_thread->preemption_cancelled(); + ObjectSynchronizer::read_monitor(_thread, _obj())->set_object_strong(); + _thread->set_pending_preempted_exception(); + + } } } ObjectLocker::~ObjectLocker() { - if (_obj() != nullptr) { + if (_obj() != nullptr && !_skip_exit) { ObjectSynchronizer::exit(_obj(), &_lock, _thread); } } +void ObjectLocker::wait_uninterruptibly(TRAPS) { + ObjectSynchronizer::waitUninterruptibly(_obj, 0, _thread); + if (_thread->preempting()) { + _skip_exit = true; + ObjectSynchronizer::read_monitor(_thread, _obj())->set_object_strong(); + _thread->set_pending_preempted_exception(); + } +} // ----------------------------------------------------------------------------- // Wait/Notify/NotifyAll @@ -517,9 +540,7 @@ int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) { } void ObjectSynchronizer::waitUninterruptibly(Handle obj, jlong millis, TRAPS) { - if (millis < 0) { - THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); - } + assert(millis >= 0, "timeout value is negative"); ObjectMonitor* monitor; monitor = LightweightSynchronizer::inflate_locked_or_imse(obj(), inflate_cause_wait, CHECK); diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp index 419cf2bf5bb94..2337569176e7d 100644 --- a/src/hotspot/share/runtime/synchronizer.hpp +++ b/src/hotspot/share/runtime/synchronizer.hpp @@ -126,6 +126,7 @@ class ObjectSynchronizer : AllStatic { static const char* inflate_cause_name(const InflateCause cause); inline static ObjectMonitor* read_monitor(markWord mark); + inline static ObjectMonitor* read_monitor(Thread* current, oop obj); inline static ObjectMonitor* read_monitor(Thread* current, oop obj, markWord mark); // Returns the identity hash value for an oop @@ -224,13 +225,13 @@ class ObjectLocker : public StackObj { Handle _obj; BasicLock _lock; NoPreemptMark _npm; + bool _skip_exit; public: - ObjectLocker(Handle obj, JavaThread* current); + ObjectLocker(Handle obj, TRAPS); ~ObjectLocker(); // Monitor behavior - void wait(TRAPS) { ObjectSynchronizer::wait(_obj, 0, CHECK); } // wait forever - void wait_uninterruptibly(TRAPS) { ObjectSynchronizer::waitUninterruptibly(_obj, 0, CHECK); } // wait forever + void wait_uninterruptibly(TRAPS); void notify_all(TRAPS) { ObjectSynchronizer::notifyall(_obj, CHECK); } }; diff --git a/src/hotspot/share/runtime/synchronizer.inline.hpp b/src/hotspot/share/runtime/synchronizer.inline.hpp index cdbeb1daf5b84..7bcbd91eda7ea 100644 --- a/src/hotspot/share/runtime/synchronizer.inline.hpp +++ b/src/hotspot/share/runtime/synchronizer.inline.hpp @@ -34,6 +34,10 @@ inline ObjectMonitor* ObjectSynchronizer::read_monitor(markWord mark) { return mark.monitor(); } +inline ObjectMonitor* ObjectSynchronizer::read_monitor(Thread* current, oop obj) { + return ObjectSynchronizer::read_monitor(current, obj, obj->mark()); +} + inline ObjectMonitor* ObjectSynchronizer::read_monitor(Thread* current, oop obj, markWord mark) { if (!UseObjectMonitorTable) { return read_monitor(mark); diff --git a/src/hotspot/share/runtime/threads.cpp b/src/hotspot/share/runtime/threads.cpp index 299905ff0a2a3..10c3636273d1a 100644 --- a/src/hotspot/share/runtime/threads.cpp +++ b/src/hotspot/share/runtime/threads.cpp @@ -410,6 +410,7 @@ void Threads::initialize_java_lang_classes(JavaThread* main_thread, TRAPS) { initialize_class(vmSymbols::java_lang_ClassCastException(), CHECK); initialize_class(vmSymbols::java_lang_ArrayStoreException(), CHECK); initialize_class(vmSymbols::java_lang_ArithmeticException(), CHECK); + initialize_class(vmSymbols::jdk_internal_vm_PreemptedException(), CHECK); initialize_class(vmSymbols::java_lang_ArrayIndexOutOfBoundsException(), CHECK); initialize_class(vmSymbols::java_lang_StackOverflowError(), CHECK); initialize_class(vmSymbols::java_lang_IllegalMonitorStateException(), CHECK); diff --git a/src/hotspot/share/utilities/exceptions.cpp b/src/hotspot/share/utilities/exceptions.cpp index e7acb4387edaf..22a8cd6e0ef12 100644 --- a/src/hotspot/share/utilities/exceptions.cpp +++ b/src/hotspot/share/utilities/exceptions.cpp @@ -62,6 +62,8 @@ void ThreadShadow::set_pending_exception(oop exception, const char* file, int li } void ThreadShadow::clear_pending_exception() { + assert(_pending_exception == nullptr || !_pending_exception->is_a(vmClasses::PreemptedException_klass()), + "unexpected PreemptedException, missing NoPreemptMark?"); LogTarget(Debug, exceptions) lt; if (_pending_exception != nullptr && lt.is_enabled()) { ResourceMark rm; @@ -82,6 +84,30 @@ void ThreadShadow::clear_pending_nonasync_exception() { } } +void ThreadShadow::set_pending_preempted_exception() { + assert(!has_pending_exception(), ""); + // We always install the same pre-allocated exception since we only + // want to use the TRAPS mechanism to bail out from all methods until + // reaching the one using the CHECK_AND_CLEAR_PREEMPTED macro. + set_pending_exception(Universe::preempted_exception_instance(), __FILE__, __LINE__); +} + +void ThreadShadow::clear_pending_preempted_exception() { + assert(has_pending_exception(), ""); + if (pending_exception()->is_a(vmClasses::PreemptedException_klass())) { + _pending_exception = nullptr; + _exception_file = nullptr; + _exception_line = 0; + } +} + +#ifdef ASSERT +void ThreadShadow::check_preempted_exception() { + assert(has_pending_exception(), ""); + assert(pending_exception()->is_a(vmClasses::PreemptedException_klass()), "should only be PreemptedException"); +} +#endif + // Implementation of Exceptions bool Exceptions::special_exception(JavaThread* thread, const char* file, int line, Handle h_exception, Symbol* h_name, const char* message) { @@ -317,6 +343,11 @@ Handle Exceptions::new_exception(JavaThread* thread, Symbol* name, if (!thread->has_pending_exception()) { assert(klass != nullptr, "klass must exist"); + // We could get here while linking or initializing a klass + // from a preemptable call. Don't preempt here since before + // the PreemptedException is propagated we might make an upcall + // to Java to initialize the object with the cause of exception. + NoPreemptMark npm(thread); h_exception = JavaCalls::construct_new_instance(InstanceKlass::cast(klass), signature, args, diff --git a/src/hotspot/share/utilities/exceptions.hpp b/src/hotspot/share/utilities/exceptions.hpp index 94f4a04546d0b..e76a0041d2007 100644 --- a/src/hotspot/share/utilities/exceptions.hpp +++ b/src/hotspot/share/utilities/exceptions.hpp @@ -94,6 +94,10 @@ class ThreadShadow: public CHeapObj { // use CLEAR_PENDING_NONASYNC_EXCEPTION to clear probable nonasync exception. void clear_pending_nonasync_exception(); + void set_pending_preempted_exception(); + void clear_pending_preempted_exception(); + void check_preempted_exception() NOT_DEBUG_RETURN; + ThreadShadow() : _pending_exception(nullptr), _exception_file(nullptr), _exception_line(0) {} }; @@ -230,6 +234,8 @@ class Exceptions { #define CHECK_NULL CHECK_(nullptr) #define CHECK_false CHECK_(false) #define CHECK_JNI_ERR CHECK_(JNI_ERR) +#define CHECK_PREEMPTABLE THREAD); if (HAS_PENDING_EXCEPTION) { THREAD->check_preempted_exception(); return; } (void)(0 +#define CHECK_PREEMPTABLE_false THREAD); if (HAS_PENDING_EXCEPTION) { THREAD->check_preempted_exception(); return false; } (void)(0 // CAUTION: These macros clears all exceptions including async exceptions, use it with caution. #define CHECK_AND_CLEAR THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_EXCEPTION; return; } (void)(0 @@ -249,6 +255,10 @@ class Exceptions { #define CHECK_AND_CLEAR_NONASYNC_NULL CHECK_AND_CLEAR_NONASYNC_(nullptr) #define CHECK_AND_CLEAR_NONASYNC_false CHECK_AND_CLEAR_NONASYNC_(false) +#define CLEAR_PENDING_PREEMPTED_EXCEPTION (((ThreadShadow*)THREAD)->clear_pending_preempted_exception()) +#define CHECK_AND_CLEAR_PREEMPTED THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_PREEMPTED_EXCEPTION; return; } (void)(0 + + // The THROW... macros should be used to throw an exception. They require a THREAD variable to be // visible within the scope containing the THROW. Usually this is achieved by declaring the function // with a TRAPS argument. diff --git a/src/java.base/share/classes/java/lang/VirtualThread.java b/src/java.base/share/classes/java/lang/VirtualThread.java index a23cbb72a6c8a..6064b46d50a16 100644 --- a/src/java.base/share/classes/java/lang/VirtualThread.java +++ b/src/java.base/share/classes/java/lang/VirtualThread.java @@ -168,6 +168,9 @@ final class VirtualThread extends BaseVirtualThread { // notified by Object.notify/notifyAll while waiting in Object.wait private volatile boolean notified; + // true when waiting in Object.wait, false for VM internal uninterruptible Object.wait + private volatile boolean interruptibleWait; + // timed-wait support private byte timedWaitSeqNo; @@ -599,6 +602,7 @@ private void afterYield() { // Object.wait if (s == WAITING || s == TIMED_WAITING) { int newState; + boolean interruptible = interruptibleWait; if (s == WAITING) { setState(newState = WAIT); } else { @@ -628,7 +632,7 @@ private void afterYield() { } // may have been interrupted while in transition to wait state - if (interrupted && compareAndSetState(newState, UNBLOCKED)) { + if (interruptible && interrupted && compareAndSetState(newState, UNBLOCKED)) { submitRunContinuation(); return; } diff --git a/src/java.base/share/classes/jdk/internal/vm/PreemptedException.java b/src/java.base/share/classes/jdk/internal/vm/PreemptedException.java new file mode 100644 index 0000000000000..56b3764f2fd0a --- /dev/null +++ b/src/java.base/share/classes/jdk/internal/vm/PreemptedException.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.internal.vm; + +/** + * Internal exception used only by the VM. + */ +public class PreemptedException extends RuntimeException { + @java.io.Serial + private static final long serialVersionUID = 6700691236100628123L; + + /** + * Constructs an {@code PreemptedException} with no detail message. + */ + public PreemptedException() { + super(); + } +} diff --git a/test/hotspot/gtest/oops/test_markWord.cpp b/test/hotspot/gtest/oops/test_markWord.cpp index 226d5a2dd74a0..f894acb5d1b9b 100644 --- a/test/hotspot/gtest/oops/test_markWord.cpp +++ b/test/hotspot/gtest/oops/test_markWord.cpp @@ -102,7 +102,7 @@ TEST_VM(markWord, printing) { st = new LockerThread(&done, h_obj()); st->doit(); - ol.wait(THREAD); + ol.wait_uninterruptibly(THREAD); assert_test_pattern(h_obj, "monitor"); done.wait_with_safepoint_check(THREAD); // wait till the thread is done. } diff --git a/test/jdk/java/lang/Thread/virtual/JfrEvents.java b/test/jdk/java/lang/Thread/virtual/JfrEvents.java index 0c96781148161..a0b2077a2aa5c 100644 --- a/test/jdk/java/lang/Thread/virtual/JfrEvents.java +++ b/test/jdk/java/lang/Thread/virtual/JfrEvents.java @@ -302,10 +302,10 @@ static void m() { } /** - * Test jdk.VirtualThreadPinned event when waiting for a class initializer. + * Test jdk.VirtualThreadPinned event when waiting for a class initializer while pinned. */ @Test - void testWaitingForClassInitializer() throws Exception { + void testWaitingForClassInitializerWhenPinned() throws Exception { class TestClass { static { LockSupport.park(); @@ -328,7 +328,9 @@ static void m() { }); Thread vthread2 = Thread.ofVirtual().unstarted(() -> { started2.set(true); - TestClass.m(); + VThreadPinner.runPinned(() -> { + TestClass.m(); + }); }); try { @@ -341,7 +343,7 @@ static void m() { vthread2.start(); awaitTrue(started2); - // give time for second virtual thread to wait on the MutexLocker + // give time for second virtual thread to wait in VM Thread.sleep(3000); } finally { diff --git a/test/jdk/java/lang/Thread/virtual/KlassInit.java b/test/jdk/java/lang/Thread/virtual/KlassInit.java new file mode 100644 index 0000000000000..4f11d667a2bd6 --- /dev/null +++ b/test/jdk/java/lang/Thread/virtual/KlassInit.java @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test id=default + * @library /test/lib + * @requires vm.continuations + * @run junit/othervm/timeout=480 -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* + * -XX:CompileCommand=exclude,KlassInit$$Lambda*::run -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +/* + * @test id=Xint + * @library /test/lib + * @requires vm.continuations + * @run junit/othervm/timeout=480 -Xint -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* + * -XX:CompileCommand=exclude,KlassInit$$Lambda*::run -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +/* + * @test id=Xcomp + * @library /test/lib + * @requires vm.continuations + * @run junit/othervm/timeout=480 -Xcomp -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* + * -XX:CompileCommand=exclude,KlassInit$$Lambda*::run -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +/* + * @test id=Xcomp-TieredStopAtLevel1 + * @library /test/lib + * @requires vm.continuations + * @run junit/othervm/timeout=480 -Xcomp -XX:TieredStopAtLevel=1 -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* + * -XX:CompileCommand=exclude,KlassInit$$Lambda*::run -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +/* + * @test id=Xcomp-noTieredCompilation + * @library /test/lib + * @requires vm.continuations + * @run junit/othervm/timeout=480 -Xcomp -XX:-TieredCompilation -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* + * -XX:CompileCommand=exclude,KlassInit$$Lambda*::run -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +/* + * @test id=gc + * @library /test/lib + * @requires vm.debug == true & vm.continuations + * @run junit/othervm/timeout=480 -XX:+UnlockDiagnosticVMOptions -XX:+FullGCALot -XX:FullGCALotInterval=1000 + * -XX:CompileCommand=exclude,KlassInit::lambda$testReleaseAtKlassInit* -XX:CompileCommand=exclude,KlassInit$$Lambda*::run + * -XX:CompileCommand=exclude,KlassInit$1Driver::foo KlassInit + */ + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.Arguments; +import static org.junit.jupiter.api.Assertions.*; + +class KlassInit { + private static final int MAX_VTHREAD_COUNT = 8 * Runtime.getRuntime().availableProcessors(); + + private static final CountDownLatch finishInvokeStatic1 = new CountDownLatch(1); + private static final CountDownLatch finishInvokeStatic2 = new CountDownLatch(1); + private static final CountDownLatch finishInvokeStatic3 = new CountDownLatch(1); + private static final CountDownLatch finishNew = new CountDownLatch(1); + private static final CountDownLatch finishGetStatic = new CountDownLatch(1); + private static final CountDownLatch finishPutStatic = new CountDownLatch(1); + private static final CountDownLatch finishFailedInit = new CountDownLatch(1); + + /** + * Test that threads blocked waiting for klass to be initialized + * on invokestatic bytecode release the carrier. + */ + @Test + void testReleaseAtKlassInitInvokeStatic1() throws Exception { + class TestClass { + static { + try { + finishInvokeStatic1.await(); + } catch (InterruptedException e) {} + } + static void m() { + } + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + TestClass.m(); + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishInvokeStatic1.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + } + } + + /** + * Test with static method that takes arguments. + */ + @Test + void testReleaseAtKlassInitInvokeStatic2() throws Exception { + class TestClass { + static { + try { + finishInvokeStatic2.await(); + } catch (InterruptedException e) {} + } + static void m(ArrayList list, int id) { + String str = list.get(0); + if (str != null && str.equals("VThread#" + id)) { + list.add("Success"); + } + } + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + ArrayList[] lists = new ArrayList[MAX_VTHREAD_COUNT]; + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + lists[i] = new ArrayList<>(List.of("VThread#" + i)); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + TestClass.m(lists[id], id); + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + System.gc(); + finishInvokeStatic2.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + assertEquals(lists[i].get(1), "Success"); + } + } + + /** + * Test invokestatic as first bytecode in method. + */ + @Test + void testReleaseAtKlassInitInvokeStatic3() throws Exception { + class TestClass { + static { + try { + finishInvokeStatic3.await(); + } catch (InterruptedException e) {} + } + static void m() { + } + } + class Driver { + static void foo() { + TestClass.m(); + } + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + started[0] = new CountDownLatch(1); + vthreads[0] = Thread.ofVirtual().start(() -> { + started[0].countDown(); + TestClass.m(); + }); + started[0].await(); + await(vthreads[0], Thread.State.WAITING); + + for (int i = 1; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + Driver.foo(); + }); + } + for (int i = 1; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishInvokeStatic3.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + } + } + + /** + * Test that threads blocked waiting for klass to be initialized + * on new bytecode release the carrier. + */ + @Test + void testReleaseAtKlassInitNew() throws Exception { + class TestClass { + static { + try { + finishNew.await(); + } catch (InterruptedException e) {} + } + void m() { + } + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + TestClass x = new TestClass(); + x.m(); + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishNew.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + } + } + + /** + * Test that threads blocked waiting for klass to be initialized + * on getstatic bytecode release the carrier. + */ + @Test + void testReleaseAtKlassInitGetStatic() throws Exception { + class TestClass { + static { + try { + finishGetStatic.await(); + } catch (InterruptedException e) {} + } + public static int NUMBER = 150; + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + AtomicInteger[] result = new AtomicInteger[MAX_VTHREAD_COUNT]; + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + result[i] = new AtomicInteger(); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + result[id].set(TestClass.NUMBER); + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishGetStatic.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + assertEquals(result[i].get(), TestClass.NUMBER); + } + } + + /** + * Test that threads blocked waiting for klass to be initialized + * on putstatic release the carrier. + */ + @Test + void testReleaseAtKlassInitPutStatic() throws Exception { + class TestClass { + static { + try { + finishPutStatic.await(); + } catch (InterruptedException e) {} + } + public static int NUMBER; + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + TestClass.NUMBER = id; + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishPutStatic.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + } + } + + /** + * Test that interruptions during preemption on klass init + * are preserved. + */ + @ParameterizedTest + @MethodSource("interruptTestCases") + void testReleaseAtKlassInitPreserverInterrupt(int timeout, Runnable m, CountDownLatch finish) throws Exception { + // Start vthread1 and wait until it blocks in TestClassX initializer + var vthread1_started = new CountDownLatch(1); + var vthread1 = Thread.ofVirtual().start(() -> { + vthread1_started.countDown(); + m.run(); + }); + vthread1_started.await(); + await(vthread1, Thread.State.WAITING); + + // Start vthread2 and wait until it gets preempted on TestClassX initialization + var lock = new Object(); + var interruptedException = new AtomicBoolean(); + var vthread2_started = new CountDownLatch(1); + var vthread2 = Thread.ofVirtual().start(() -> { + vthread2_started.countDown(); + m.run(); + synchronized (lock) { + try { + if (timeout > 0) { + lock.wait(timeout); + } else { + lock.wait(); + } + } catch (InterruptedException e) { + // check stack trace has the expected frames + Set expected = Set.of("wait0", "wait", "run"); + Set methods = Stream.of(e.getStackTrace()) + .map(StackTraceElement::getMethodName) + .collect(Collectors.toSet()); + assertTrue(methods.containsAll(expected)); + interruptedException.set(true); + } + } + }); + vthread2_started.await(); + await(vthread2, Thread.State.WAITING); + + // Interrupt vthread2 and let initialization of TestClassX finish + vthread2.interrupt(); + finish.countDown(); + vthread1.join(); + vthread2.join(); + assertTrue(interruptedException.get()); + } + + static CountDownLatch finishInterrupt0 = new CountDownLatch(1); + class TestClass0 { + static { + try { + finishInterrupt0.await(); + } catch (InterruptedException e) {} + } + static void m() {} + } + + static CountDownLatch finishInterrupt30000 = new CountDownLatch(1); + class TestClass30000 { + static { + try { + finishInterrupt30000.await(); + } catch (InterruptedException e) {} + } + static void m() {} + } + + static CountDownLatch finishInterruptMax = new CountDownLatch(1); + class TestClassMax { + static { + try { + finishInterruptMax.await(); + } catch (InterruptedException e) {} + } + static void m() {} + } + + static Stream interruptTestCases() { + return Stream.of( + Arguments.of(0, (Runnable) TestClass0::m, finishInterrupt0), + Arguments.of(30000, (Runnable) TestClass30000::m, finishInterrupt30000), + Arguments.of(Integer.MAX_VALUE, (Runnable) TestClassMax::m, finishInterruptMax) + ); + } + + /** + * Test case of threads blocked waiting for klass to be initialized + * when the klass initialization fails. + */ + @Test + void testReleaseAtKlassInitFailedInit() throws Exception { + class TestClass { + static int[] a = {1, 2, 3}; + static { + try { + finishFailedInit.await(); + a[3] = 4; + } catch (InterruptedException e) {} + } + static void m() { + } + } + + Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT]; + CountDownLatch[] started = new CountDownLatch[MAX_VTHREAD_COUNT]; + AtomicInteger failedCount = new AtomicInteger(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + final int id = i; + started[i] = new CountDownLatch(1); + vthreads[i] = Thread.ofVirtual().start(() -> { + started[id].countDown(); + try { + TestClass.m(); + } catch (NoClassDefFoundError e) { + failedCount.getAndIncrement(); + } catch (ExceptionInInitializerError e) { + failedCount.getAndIncrement(); + } + }); + } + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + started[i].await(); + await(vthreads[i], Thread.State.WAITING); + } + + finishFailedInit.countDown(); + for (int i = 0; i < MAX_VTHREAD_COUNT; i++) { + vthreads[i].join(); + } + assertEquals(MAX_VTHREAD_COUNT, failedCount.get()); + } + + /** + * Waits for the given thread to reach a given state. + */ + private void await(Thread thread, Thread.State expectedState) throws InterruptedException { + Thread.State state = thread.getState(); + while (state != expectedState) { + assertTrue(state != Thread.State.TERMINATED, "Thread has terminated"); + Thread.sleep(10); + state = thread.getState(); + } + } +} diff --git a/test/jdk/java/lang/Thread/virtual/YieldQueuing.java b/test/jdk/java/lang/Thread/virtual/YieldQueuing.java index edb0b6e9f7411..7bd2320e2aba3 100644 --- a/test/jdk/java/lang/Thread/virtual/YieldQueuing.java +++ b/test/jdk/java/lang/Thread/virtual/YieldQueuing.java @@ -28,16 +28,24 @@ * @run junit/othervm -Djdk.virtualThreadScheduler.maxPoolSize=1 YieldQueuing */ +import java.lang.invoke.MethodHandles; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; import static org.junit.jupiter.api.Assertions.*; class YieldQueuing { + @BeforeAll + static void setup() throws Exception { + // waiting for LockSupport to be initialized can change the scheduling + MethodHandles.lookup().ensureInitialized(LockSupport.class); + } + /** * Test Thread.yield submits the task for the current virtual thread to a scheduler * submission queue when there are no tasks in the local queue. diff --git a/test/jdk/java/lang/Thread/virtual/stress/LotsOfContendedMonitorEnter.java b/test/jdk/java/lang/Thread/virtual/stress/LotsOfContendedMonitorEnter.java index da45c610e8238..32d4e1b39d9dc 100644 --- a/test/jdk/java/lang/Thread/virtual/stress/LotsOfContendedMonitorEnter.java +++ b/test/jdk/java/lang/Thread/virtual/stress/LotsOfContendedMonitorEnter.java @@ -25,22 +25,34 @@ * @test id=default * @summary Test virtual threads entering a lot of monitors with contention * @library /test/lib - * @run main LotsOfContendedMonitorEnter + * @run main/timeout=480 LotsOfContendedMonitorEnter */ +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.CountDownLatch; import jdk.test.lib.thread.VThreadRunner; public class LotsOfContendedMonitorEnter { + static int depth; public static void main(String[] args) throws Exception { - int depth; if (args.length > 0) { depth = Integer.parseInt(args[0]); } else { depth = 1024; } - VThreadRunner.run(() -> testContendedEnter(depth)); + + var exRef = new AtomicReference(); + var thread = Thread.ofVirtual().start(() -> { + try { + testContendedEnter(depth); + } catch (Exception e) { + exRef.set(e); + } + }); + thread.join(); + assert exRef.get() == null; } /** @@ -48,6 +60,7 @@ public static void main(String[] args) throws Exception { * attempts to enter around the same time, then repeat to the given depth. */ private static void testContendedEnter(int depthRemaining) throws Exception { + boolean doLog = depthRemaining % 10 == 0; if (depthRemaining > 0) { var lock = new Object(); @@ -81,12 +94,18 @@ private static void testContendedEnter(int depthRemaining) throws Exception { // signal thread to enter monitor again, it should block signal.countDown(); await(thread, Thread.State.BLOCKED); + if (doLog) { + System.out.println(Instant.now() + " => at depth: " + (depth - depthRemaining)); + } testContendedEnter(depthRemaining - 1); } } finally { thread.join(); } } + if (doLog) { + System.out.println(Instant.now() + " => returning from depth: " + (depth - depthRemaining)); + } } /**