diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 5e116d82761ac..1385366d8793b 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -1168,8 +1168,8 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { if (op->init_check()) { - __ ldrb(rscratch1, Address(op->klass()->as_register(), - InstanceKlass::init_state_offset())); + __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset())); + __ ldarb(rscratch1, rscratch1); __ cmpw(rscratch1, InstanceKlass::fully_initialized); add_debug_info_for_null_check_here(op->stub()->info()); __ br(Assembler::NE, *op->stub()->entry()); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index c5c02619d446e..16473b09fff42 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -1838,7 +1838,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register scratch, Label* L_f L_slow_path = &L_fallthrough; } // Fast path check: class is fully initialized - ldrb(scratch, Address(klass, InstanceKlass::init_state_offset())); + lea(scratch, Address(klass, InstanceKlass::init_state_offset())); + ldarb(scratch, scratch); subs(zr, scratch, InstanceKlass::fully_initialized); br(Assembler::EQ, *L_fast_path); diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index bb6a93e6f8da7..b14e6f0b4ca0c 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -948,6 +948,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { if (op->init_check()) { Register tmp = op->tmp1()->as_register(); __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset())); + __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp); add_debug_info_for_null_check_here(op->stub()->info()); __ cmp(tmp, InstanceKlass::fully_initialized); __ b(*op->stub()->entry(), ne); diff --git a/src/hotspot/cpu/arm/templateTable_arm.cpp b/src/hotspot/cpu/arm/templateTable_arm.cpp index 80519fd89f426..0974ff1f9a9c3 100644 --- a/src/hotspot/cpu/arm/templateTable_arm.cpp +++ b/src/hotspot/cpu/arm/templateTable_arm.cpp @@ -3974,6 +3974,7 @@ void TemplateTable::_new() { // make sure klass is initialized // make sure klass is fully initialized __ ldrb(Rtemp, Address(Rklass, InstanceKlass::init_state_offset())); + __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp); __ cmp(Rtemp, InstanceKlass::fully_initialized); __ b(slow_case, ne); diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 42934dc7c3179..684c06614a97a 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -2274,6 +2274,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { } __ lbz(op->tmp1()->as_register(), in_bytes(InstanceKlass::init_state_offset()), op->klass()->as_register()); + // acquire barrier included in membar_storestore() which follows the allocation immediately. __ cmpwi(CCR0, op->tmp1()->as_register(), InstanceKlass::fully_initialized); __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), *op->stub()->entry()); } diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 8d8e39b8bbc00..a194c030a6124 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -2410,7 +2410,7 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); - Label L_fallthrough; + Label L_check_thread, L_fallthrough; if (L_fast_path == nullptr) { L_fast_path = &L_fallthrough; } else if (L_slow_path == nullptr) { @@ -2419,10 +2419,14 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa // Fast path check: class is fully initialized lbz(R0, in_bytes(InstanceKlass::init_state_offset()), klass); + // acquire by cmp-branch-isync if fully_initialized cmpwi(CCR0, R0, InstanceKlass::fully_initialized); - beq(CCR0, *L_fast_path); + bne(CCR0, L_check_thread); + isync(); + b(*L_fast_path); // Fast path check: current thread is initializer thread + bind(L_check_thread); ld(R0, in_bytes(InstanceKlass::init_thread_offset()), klass); cmpd(CCR0, thread, R0); if (L_slow_path == &L_fallthrough) { diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp index 940706b0a7376..828f70e4decee 100644 --- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -980,6 +980,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { if (op->init_check()) { __ lbu(t0, Address(op->klass()->as_register(), InstanceKlass::init_state_offset())); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); __ mv(t1, (u1)InstanceKlass::fully_initialized); add_debug_info_for_null_check_here(op->stub()->info()); __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 32a446959a246..f9196fef3f517 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -493,6 +493,7 @@ void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_ // Fast path check: class is fully initialized lbu(tmp, Address(klass, InstanceKlass::init_state_offset())); + membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); sub(tmp, tmp, InstanceKlass::fully_initialized); beqz(tmp, *L_fast_path); diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index d288f4a893d0a..8990cf1663dd5 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -2350,6 +2350,7 @@ void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr de void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { if (op->init_check()) { // Make sure klass is initialized & doesn't have finalizer. + // init_state needs acquire, but S390 is TSO, and so we are already good. const int state_offset = in_bytes(InstanceKlass::init_state_offset()); Register iklass = op->klass()->as_register(); add_debug_info_for_null_check_here(op->stub()->info()); diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index af281345b1477..aaa0e1bff8c7d 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -3458,7 +3458,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa L_slow_path = &L_fallthrough; } - // Fast path check: class is fully initialized + // Fast path check: class is fully initialized. + // init_state needs acquire, but S390 is TSO, and so we are already good. z_cli(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); z_bre(*L_fast_path); diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index c3444d5a5abce..6d9812c11ae6e 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -1578,6 +1578,7 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { if (op->init_check()) { add_debug_info_for_null_check_here(op->stub()->info()); + // init_state needs acquire, but x86 is TSO, and so we are already good. __ cmpb(Address(op->klass()->as_register(), InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 893ae4e844ba4..018258a012e57 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5084,7 +5084,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa L_slow_path = &L_fallthrough; } - // Fast path check: class is fully initialized + // Fast path check: class is fully initialized. + // init_state needs acquire, but x86 is TSO, and so we are already good. cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); jcc(Assembler::equal, *L_fast_path); diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 5e783225fcbfc..527d961259ecc 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -4048,6 +4048,7 @@ void TemplateTable::_new() { __ push(rcx); // save the contexts of klass for initializing the header // make sure klass is initialized + // init_state needs acquire, but x86 is TSO, and so we are already good. #ifdef _LP64 assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks"); __ clinit_barrier(rcx, r15_thread, nullptr /*L_fast_path*/, &slow_case); diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index 5e226a90764ee..6b6d35ee026de 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -4103,7 +4103,7 @@ void InstanceKlass::set_init_state(ClassState state) { assert(good_state || state == allocated, "illegal state transition"); #endif assert(_init_thread == nullptr, "should be cleared before state change"); - _init_state = state; + Atomic::release_store(&_init_state, state); } #if INCLUDE_JVMTI diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index eaffa0250d133..45d65f273c866 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -507,14 +507,14 @@ class InstanceKlass: public Klass { public: // initialization state - bool is_loaded() const { return _init_state >= loaded; } - bool is_linked() const { return _init_state >= linked; } - bool is_initialized() const { return _init_state == fully_initialized; } - bool is_not_initialized() const { return _init_state < being_initialized; } - bool is_being_initialized() const { return _init_state == being_initialized; } - bool is_in_error_state() const { return _init_state == initialization_error; } + bool is_loaded() const { return init_state() >= loaded; } + bool is_linked() const { return init_state() >= linked; } + bool is_initialized() const { return init_state() == fully_initialized; } + bool is_not_initialized() const { return init_state() < being_initialized; } + bool is_being_initialized() const { return init_state() == being_initialized; } + bool is_in_error_state() const { return init_state() == initialization_error; } bool is_reentrant_initialization(Thread *thread) { return thread == _init_thread; } - ClassState init_state() const { return _init_state; } + ClassState init_state() const { return Atomic::load_acquire(&_init_state); } const char* init_state_name() const; bool is_rewritten() const { return _misc_flags.rewritten(); } diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 3bc5b9a8b2a7d..1a0d0bd037515 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3008,7 +3008,7 @@ void GraphKit::guard_klass_being_initialized(Node* klass) { Node* adr = basic_plus_adr(top(), klass, init_state_off); Node* init_state = LoadNode::make(_gvn, nullptr, immutable_memory(), adr, adr->bottom_type()->is_ptr(), TypeInt::BYTE, - T_BYTE, MemNode::unordered); + T_BYTE, MemNode::acquire); init_state = _gvn.transform(init_state); Node* being_initialized_state = makecon(TypeInt::make(InstanceKlass::being_initialized)); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 8bbb2f8115ec4..5dce40ba31d90 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -2899,7 +2899,7 @@ bool LibraryCallKit::inline_unsafe_allocate() { Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset())); // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler // can generate code to load it as unsigned byte. - Node* inst = make_load(nullptr, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); + Node* inst = make_load(nullptr, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::acquire); Node* bits = intcon(InstanceKlass::fully_initialized); test = _gvn.transform(new SubINode(inst, bits)); // The 'test' is non-zero if we need to take a slow path.