diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp index e074dcf60c7..413a25a71c2 100644 --- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp +++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp @@ -662,7 +662,7 @@ bool G1BarrierSetC2::is_gc_barrier_node(Node* node) const { return strcmp(call->_name, "write_ref_field_pre_entry") == 0 || strcmp(call->_name, "write_ref_field_post_entry") == 0; } -void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { +void G1BarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const { assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required"); assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes"); // It could be only one user, URShift node, in Object.clone() intrinsic @@ -690,7 +690,7 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() && cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne, "missing region check in G1 post barrier"); - macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ)); + igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ)); // Remove G1 pre barrier. @@ -708,14 +708,14 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c assert(bol->is_Bool(), ""); cmpx = bol->in(1); if (bol->as_Bool()->_test._test == BoolTest::ne && - cmpx->is_Cmp() && cmpx->in(2) == macro->intcon(0) && + cmpx->is_Cmp() && cmpx->in(2) == igvn->intcon(0) && cmpx->in(1)->is_Load()) { Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address); const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); - if (adr->is_AddP() && adr->in(AddPNode::Base) == macro->top() && + if (adr->is_AddP() && adr->in(AddPNode::Base) == igvn->C->top() && adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && - adr->in(AddPNode::Offset) == macro->MakeConX(marking_offset)) { - macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ)); + adr->in(AddPNode::Offset) == igvn->MakeConX(marking_offset)) { + igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ)); } } } @@ -734,13 +734,13 @@ void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) c assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() && cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne, "missing card value check in G1 post barrier"); - macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ)); + igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ)); // There is no G1 pre barrier in this case } // Now CastP2X can be removed since it is used only on dead path // which currently still alive until igvn optimize it. assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, ""); - macro->replace_node(node, macro->top()); + igvn->replace_node(node, igvn->C->top()); } Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const { diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp index 3002daa5cfc..e661725f56d 100644 --- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp +++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp @@ -86,7 +86,7 @@ class G1BarrierSetC2: public CardTableBarrierSetC2 { public: virtual bool is_gc_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; + virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const; virtual Node* step_over_gc_barrier(Node* c) const; #ifdef ASSERT diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp index 6e96eb611fd..a636262ba28 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp @@ -853,5 +853,5 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP); phase->transform_later(call); - phase->igvn().replace_node(ac, call); + phase->replace_node(ac, call); } diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index 43496d07b0b..66f247ad034 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -267,7 +267,7 @@ class BarrierSetC2: public CHeapObj { // Support for macro expanded GC barriers virtual void register_potential_barrier_node(Node* node) const { } virtual void unregister_potential_barrier_node(Node* node) const { } - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } + virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const { } virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {} virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {} diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp index 06ac0fdf967..30e2d42be94 100644 --- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp @@ -168,7 +168,7 @@ bool CardTableBarrierSetC2::is_gc_barrier_node(Node* node) const { return ModRefBarrierSetC2::is_gc_barrier_node(node) || node->Opcode() == Op_StoreCM; } -void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { +void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const { assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required"); Node *shift = node->unique_out(); Node *addp = shift->unique_out(); @@ -178,11 +178,11 @@ void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* assert(mem->Opcode() == Op_LoadB, "unexpected code shape"); // The load is checking if the card has been written so // replace it with zero to fold the test. - macro->replace_node(mem, macro->intcon(0)); + igvn->replace_node(mem, igvn->intcon(0)); continue; } assert(mem->is_Store(), "store required"); - macro->replace_node(mem, mem->in(MemNode::Memory)); + igvn->replace_node(mem, mem->in(MemNode::Memory)); } } diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp index c0703f79fef..48a0dad73f5 100644 --- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp @@ -44,7 +44,7 @@ class CardTableBarrierSetC2: public ModRefBarrierSetC2 { public: virtual void clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* countx, bool is_array) const; virtual bool is_gc_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; + virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const; virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const; bool use_ReduceInitialCardMarks() const; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index d3baab50e93..99bb33c45fd 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -830,7 +830,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM); Node* thread = phase->transform_later(new ThreadLocalNode()); - Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset)); uint gc_state_idx = Compile::AliasIdxRaw; @@ -882,7 +882,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo call = phase->transform_later(call); // Hook up the whole thing into the graph - phase->igvn().replace_node(ac, call); + phase->replace_node(ac, call); } else { BarrierSetC2::clone_at_expansion(phase, ac); } @@ -908,9 +908,9 @@ void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const } } -void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const { +void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const { if (is_shenandoah_wb_pre_call(n)) { - shenandoah_eliminate_wb_pre(n, ¯o->igvn()); + shenandoah_eliminate_wb_pre(n, igvn); } } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index db494012805..687eec6df32 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -123,7 +123,7 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 { // Support for macro expanded GC barriers virtual void register_potential_barrier_node(Node* node) const; virtual void unregister_potential_barrier_node(Node* node) const; - virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const; + virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const; virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const; virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const; diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp index d0ea716bf15..393f5e7f79e 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp @@ -267,7 +267,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a full_size, phase->top()); phase->transform_later(call); - phase->igvn().replace_node(ac, call); + phase->replace_node(ac, call); } // == Dominating barrier elision == diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index fe2ca4b63a6..a8f87ca2d2a 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "ci/ciInlineKlass.hpp" +#include "gc/shared/barrierSet.hpp" #include "opto/addnode.hpp" #include "opto/castnode.hpp" #include "opto/graphKit.hpp" @@ -783,16 +784,27 @@ void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, Extended // Replace a buffer allocation by a dominating allocation static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) { - // Remove initializing stores + // Remove initializing stores and GC barriers for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) { - AddPNode* addp = res->fast_out(i)->isa_AddP(); - if (addp != NULL) { - for (DUIterator_Fast jmax, j = addp->fast_outs(jmax); j < jmax; j++) { - StoreNode* store = addp->fast_out(j)->isa_Store(); + Node* use = res->fast_out(i); + if (use->is_AddP()) { + for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { + Node* store = use->fast_out(j)->isa_Store(); if (store != NULL) { + igvn->rehash_node_delayed(store); igvn->replace_in_uses(store, store->in(MemNode::Memory)); } } + } else if (use->Opcode() == Op_CastP2X) { + if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) { + // The G1 pre-barrier uses a CastP2X both for the pointer of the object + // we store into, as well as the value we are storing. Skip if this is a + // barrier for storing 'res' into another object. + continue; + } + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->eliminate_gc_barrier(igvn, use); + --i; --imax; } } igvn->replace_node(res, dom); @@ -892,7 +904,7 @@ void InlineTypeNode::remove_redundant_allocations(PhaseIterGVN* igvn, PhaseIdeal Node* out = fast_out(i); if (out->is_InlineType()) { // Unlink and recursively process inline type users - igvn->hash_delete(out); + igvn->rehash_node_delayed(out); int nb = out->replace_edge(this, igvn->C->top()); out->as_InlineType()->remove_redundant_allocations(igvn, phase); --i; imax -= nb; diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 23c34d23e7f..40641ed9c56 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -201,7 +201,7 @@ void PhaseMacroExpand::extract_call_projections(CallNode *call) { void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) { BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2(); - bs->eliminate_gc_barrier(this, p2x); + bs->eliminate_gc_barrier(&_igvn, p2x); } // Search for a memory operation for the specified memory slice. @@ -721,7 +721,8 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr } else if (use->is_InlineType() && use->isa_InlineType()->get_oop() == res) { // ok to eliminate } else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) { - // store to mark work + // Store to mark word of inline type larval buffer + assert(res_type->is_inlinetypeptr(), "Unexpected store to mark word"); } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark if (use->is_Phi()) { if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) { @@ -1052,7 +1053,9 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc, bool inline_ assert(use->isa_InlineType()->get_oop() == res, "unexpected inline type use"); _igvn.rehash_node_delayed(use); use->isa_InlineType()->set_oop(_igvn.zerocon(T_INLINE_TYPE)); - } else if (use->is_Store()) { + } else if (use->Opcode() == Op_StoreX && use->in(MemNode::Address) == res) { + // Store to mark word of inline type larval buffer + assert(inline_alloc, "Unexpected store to mark word"); _igvn.replace_node(use, use->in(MemNode::Memory)); } else { eliminate_gc_barrier(use);