diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 0895418ef848b..80494a0294602 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -467,6 +467,9 @@ class methodHandle; do_intrinsic(_Reference_clear0, java_lang_ref_Reference, clear0_name, void_method_signature, F_RN) \ do_intrinsic(_PhantomReference_clear0, java_lang_ref_PhantomReference, clear0_name, void_method_signature, F_RN) \ \ + do_intrinsic(_Reference_reachabilityFence, java_lang_ref_Reference, reachabilityFence_name, object_void_signature, F_S) \ + do_name(reachabilityFence_name, "reachabilityFence") \ + \ /* support for com.sun.crypto.provider.AES_Crypt and some of its callers */ \ do_class(com_sun_crypto_provider_aescrypt, "com/sun/crypto/provider/AES_Crypt") \ do_intrinsic(_aescrypt_encryptBlock, com_sun_crypto_provider_aescrypt, encryptBlock_name, byteArray_int_byteArray_int_signature, F_R) \ diff --git a/src/hotspot/share/opto/block.cpp b/src/hotspot/share/opto/block.cpp index 7d3d4ec16f4f1..a93e2e43a29a0 100644 --- a/src/hotspot/share/opto/block.cpp +++ b/src/hotspot/share/opto/block.cpp @@ -179,9 +179,11 @@ int Block::is_Empty() const { // Ideal nodes (except BoxLock) are allowable in empty blocks: skip them. Only // Mach and BoxLock nodes turn directly into code via emit(). + // Keep ReachabilityFence for diagnostic purposes. while ((end_idx > 0) && !get_node(end_idx)->is_Mach() && - !get_node(end_idx)->is_BoxLock()) { + !get_node(end_idx)->is_BoxLock() && + !get_node(end_idx)->is_ReachabilityFence()) { end_idx--; } diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index 0a4f231c49b36..88c77c132cc3a 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -76,6 +76,15 @@ develop(bool, StressBailout, false, \ "Perform bailouts randomly at C2 failing() checks") \ \ + product(bool, OptimizeReachabilityFences, true, DIAGNOSTIC, \ + "Optimize reachability fences") \ + \ + product(bool, PreserveReachabilityFencesOnConstants, false, DIAGNOSTIC, \ + "Preserve reachability fences on constants") \ + \ + product(bool, StressReachabilityFences, false, DIAGNOSTIC, \ + "Aggressively insert reachability fences for all oop arguments") \ + \ develop(uint, StressBailoutMean, 100000, \ "The expected number of failing() checks made until " \ "a random bailout.") \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index acc2896462740..9e59f5bf80793 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -775,6 +775,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_longBitsToDouble: case vmIntrinsics::_Reference_get0: case vmIntrinsics::_Reference_refersTo0: + case vmIntrinsics::_Reference_reachabilityFence: case vmIntrinsics::_PhantomReference_refersTo0: case vmIntrinsics::_Reference_clear0: case vmIntrinsics::_PhantomReference_clear0: diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 483cb73110341..d7974c6632783 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -611,6 +611,20 @@ void CallGenerator::do_late_inline_helper() { } Compile* C = Compile::current(); + + uint endoff = call->jvms()->endoff(); + if (C->inlining_incrementally()) { + // No reachability edges should be present when incremental inlining takes place. + // Inlining logic doesn't expect any extra edges past debug info and fails with + // an assert in SafePointNode::grow_stack. + assert(endoff == call->req(), "reachability edges not supported"); + } else { + if (call->req() > endoff) { // reachability edges present + assert(OptimizeReachabilityFences, "required"); + return; // keep the original call node as the holder of reachability info + } + } + // Remove inlined methods from Compiler's lists. if (call->is_macro()) { C->remove_macro_node(call); diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index ad6548a649e88..a694ccc3bd479 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -893,7 +893,7 @@ bool CallNode::may_modify(const TypeOopPtr* t_oop, PhaseValues* phase) { } // Does this call have a direct reference to n other than debug information? -bool CallNode::has_non_debug_use(Node *n) { +bool CallNode::has_non_debug_use(const Node *n) { const TypeTuple * d = tf()->domain(); for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { Node *arg = in(i); @@ -960,11 +960,12 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) { cpn = cn->fast_out(k)->as_Proj(); assert(cpn->is_CatchProj(), "must be a CatchProjNode"); - if (cpn->_con == CatchProjNode::fall_through_index) - projs->fallthrough_catchproj = cpn; - else { - assert(cpn->_con == CatchProjNode::catch_all_index, "must be correct index."); - projs->catchall_catchproj = cpn; + switch (cpn->_con) { + case CatchProjNode::fall_through_index: projs->fallthrough_catchproj = cpn; break; + case CatchProjNode::catch_all_index: projs->catchall_catchproj = cpn; break; + default: { + assert(cpn->_con > 1, "not an exception table projection"); // exception table; rethrow case + } } } } @@ -978,8 +979,12 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj for (DUIterator j = pn->outs(); pn->has_out(j); j++) { Node* e = pn->out(j); if (e->Opcode() == Op_CreateEx && e->in(0)->is_CatchProj() && e->outcnt() > 0) { - assert(projs->exobj == nullptr, "only one"); - projs->exobj = e; + if (e->in(0)->as_CatchProj()->_con == CatchProjNode::catch_all_index) { + assert(projs->exobj == nullptr, "only one"); + projs->exobj = e; + } else { + assert(e->in(0)->as_CatchProj()->_con > 1, "not an exception table projection"); // exception table for rethrow case + } } } break; @@ -1577,6 +1582,25 @@ void SafePointNode::disconnect_from_root(PhaseIterGVN *igvn) { } } +void SafePointNode::remove_non_debug_edges(GrowableArray& non_debug_edges) { + assert(non_debug_edges.is_empty(), "edges not processed"); + while (req() > jvms()->endoff()) { + uint last = req() - 1; + non_debug_edges.push(in(last)); + del_req(last); + } + assert(jvms()->endoff() == req(), "no extra edges past debug info allowed"); +} + +void SafePointNode::restore_non_debug_edges(GrowableArray& non_debug_edges) { + assert(jvms()->endoff() == req(), "no extra edges past debug info allowed"); + while (non_debug_edges.is_nonempty()) { + Node* non_debug_edge = non_debug_edges.pop(); + add_req(non_debug_edge); + } + assert(non_debug_edges.is_empty(), "edges not processed"); +} + //============== SafePointScalarObjectNode ============== SafePointScalarObjectNode::SafePointScalarObjectNode(const TypeOopPtr* tp, Node* alloc, uint first_index, uint depth, uint n_fields) : diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 9029a009989f6..9fc438e0f728a 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -505,6 +505,9 @@ class SafePointNode : public MultiNode { return _has_ea_local_in_scope; } + void remove_non_debug_edges(GrowableArray& non_debug_edges); + void restore_non_debug_edges(GrowableArray& non_debug_edges); + void disconnect_from_root(PhaseIterGVN *igvn); // Standard Node stuff @@ -737,7 +740,7 @@ class CallNode : public SafePointNode { // Returns true if the call may modify n virtual bool may_modify(const TypeOopPtr* t_oop, PhaseValues* phase); // Does this node have a use of n other than in debug information? - bool has_non_debug_use(Node* n); + bool has_non_debug_use(const Node* n); // Returns the unique CheckCastPP of a call // or result projection is there are several CheckCastPP // or returns null if there is no one. diff --git a/src/hotspot/share/opto/classes.cpp b/src/hotspot/share/opto/classes.cpp index b760a179b57b1..1cd6c52393b09 100644 --- a/src/hotspot/share/opto/classes.cpp +++ b/src/hotspot/share/opto/classes.cpp @@ -43,6 +43,7 @@ #include "opto/narrowptrnode.hpp" #include "opto/node.hpp" #include "opto/opaquenode.hpp" +#include "opto/reachability.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" #include "opto/subtypenode.hpp" diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 587d5fad8f29e..b9696b15ae65b 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -541,3 +541,4 @@ macro(MaskAll) macro(AndVMask) macro(OrVMask) macro(XorVMask) +macro(ReachabilityFence) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 6babc13e1b315..e435457752353 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -74,6 +74,7 @@ #include "opto/output.hpp" #include "opto/parse.hpp" #include "opto/phaseX.hpp" +#include "opto/reachability.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" #include "opto/stringopts.hpp" @@ -396,6 +397,9 @@ void Compile::remove_useless_node(Node* dead) { if (dead->is_expensive()) { remove_expensive_node(dead); } + if (dead->is_ReachabilityFence()) { + remove_reachability_fence(dead->as_ReachabilityFence()); + } if (dead->is_OpaqueTemplateAssertionPredicate()) { remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate()); } @@ -459,6 +463,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis // Remove useless Template Assertion Predicate opaque nodes remove_useless_nodes(_template_assertion_predicate_opaques, useful); remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes + remove_useless_nodes(_reachability_fences, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps @@ -663,6 +668,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _parse_predicates(comp_arena(), 8, 0, nullptr), _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr), _expensive_nodes(comp_arena(), 8, 0, nullptr), + _reachability_fences(comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr), _unstable_if_traps(comp_arena(), 8, 0, nullptr), @@ -932,6 +938,7 @@ Compile::Compile(ciEnv* ci_env, _directive(directive), _log(ci_env->log()), _first_failure_details(nullptr), + _reachability_fences(comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr), _congraph(nullptr), @@ -2512,12 +2519,21 @@ void Compile::Optimize() { return; } - if (failing()) return; - C->clear_major_progress(); // ensure that major progress is now clear process_for_post_loop_opts_igvn(igvn); + // Once loop optimizations are over, it is safe to get rid of all reachability fence nodes and + // migrate reachability edges to safepoints. + if (OptimizeReachabilityFences && _reachability_fences.length() > 0) { + TracePhase tp1(_t_idealLoop); + TracePhase tp2(_t_reachability); + PhaseIdealLoop::optimize(igvn, PostLoopOptsEliminateReachabilityFences); + print_method(PHASE_ELIMINATE_REACHABILITY_FENCES, 2); + if (failing()) return; + assert(_reachability_fences.length() == 0 || PreserveReachabilityFencesOnConstants, "no RF nodes allowed"); + } + process_for_merge_stores_igvn(igvn); if (failing()) return; @@ -3973,11 +3989,29 @@ void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_R } } + expand_reachability_fences(sfpt); + // Skip next transformation if compressed oops are not used. if ((UseCompressedOops && !Matcher::gen_narrow_oop_implicit_null_checks()) || (!UseCompressedOops && !UseCompressedClassPointers)) return; + // Go over ReachabilityFence nodes to skip DecodeN nodes for referents. + // The sole purpose of RF node is to keep the referent oop alive and + // decoding the oop for that is not needed. + for (int i = 0; i < C->reachability_fences_count(); i++) { + ReachabilityFenceNode* rf = C->reachability_fence(i); + DecodeNNode* dn = rf->in(1)->isa_DecodeN(); + if (dn != nullptr) { + if (!dn->has_non_debug_uses() || Matcher::narrow_oop_use_complex_address()) { + rf->set_req(1, dn->in(1)); + if (dn->outcnt() == 0) { + dn->disconnect_inputs(this); + } + } + } + } + // Go over safepoints nodes to skip DecodeN/DecodeNKlass nodes for debug edges. // It could be done for an uncommon traps or any safepoints/calls // if the DecodeN/DecodeNKlass node is referenced only in a debug info. @@ -3991,21 +4025,8 @@ void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_R n->as_CallStaticJava()->uncommon_trap_request() != 0); for (int j = start; j < end; j++) { Node* in = n->in(j); - if (in->is_DecodeNarrowPtr()) { - bool safe_to_skip = true; - if (!is_uncommon ) { - // Is it safe to skip? - for (uint i = 0; i < in->outcnt(); i++) { - Node* u = in->raw_out(i); - if (!u->is_SafePoint() || - (u->is_Call() && u->as_Call()->has_non_debug_use(n))) { - safe_to_skip = false; - } - } - } - if (safe_to_skip) { - n->set_req(j, in->in(1)); - } + if (in->is_DecodeNarrowPtr() && (is_uncommon || !in->has_non_debug_uses())) { + n->set_req(j, in->in(1)); if (in->outcnt() == 0) { in->disconnect_inputs(this); } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 66a5497a7ad16..5d27d85565c37 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -80,6 +80,7 @@ class PhaseIterGVN; class PhaseRegAlloc; class PhaseCCP; class PhaseOutput; +class ReachabilityFenceNode; class RootNode; class relocInfo; class StartNode; @@ -107,7 +108,8 @@ enum LoopOptsMode { LoopOptsMaxUnroll, LoopOptsShenandoahExpand, LoopOptsSkipSplitIf, - LoopOptsVerify + LoopOptsVerify, + PostLoopOptsEliminateReachabilityFences }; // The type of all node counts and indexes. @@ -375,6 +377,7 @@ class Compile : public Phase { // of Template Assertion Predicates themselves. GrowableArray _template_assertion_predicate_opaques; GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common + GrowableArray _reachability_fences; // List of reachability fences GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _for_merge_stores_igvn; // List of nodes for IGVN merge stores GrowableArray _unstable_if_traps; // List of ifnodes after IGVN @@ -702,11 +705,13 @@ class Compile : public Phase { int template_assertion_predicate_count() const { return _template_assertion_predicate_opaques.length(); } int expensive_count() const { return _expensive_nodes.length(); } int coarsened_count() const { return _coarsened_locks.length(); } - Node* macro_node(int idx) const { return _macro_nodes.at(idx); } Node* expensive_node(int idx) const { return _expensive_nodes.at(idx); } + ReachabilityFenceNode* reachability_fence(int idx) const { return _reachability_fences.at(idx); } + int reachability_fences_count() const { return _reachability_fences.length(); } + ConnectionGraph* congraph() { return _congraph;} void set_congraph(ConnectionGraph* congraph) { _congraph = congraph;} void add_macro_node(Node * n) { @@ -728,6 +733,14 @@ class Compile : public Phase { _expensive_nodes.remove_if_existing(n); } + void add_reachability_fence(ReachabilityFenceNode* rf) { + _reachability_fences.append(rf); + } + + void remove_reachability_fence(ReachabilityFenceNode* n) { + _reachability_fences.remove_if_existing(n); + } + void add_parse_predicate(ParsePredicateNode* n) { assert(!_parse_predicates.contains(n), "duplicate entry in Parse Predicate list"); _parse_predicates.append(n); @@ -1280,6 +1293,8 @@ class Compile : public Phase { // Definitions of pd methods static void pd_compiler2_init(); + void expand_reachability_fences(Unique_Node_List& safepoints); + // Static parse-time type checking logic for gen_subtype_check: enum SubTypeCheckResult { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test }; SubTypeCheckResult static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip = StressReflectiveCode); diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index a148b167ee301..5a5ef12b7ffd4 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -1223,23 +1223,32 @@ bool ConnectionGraph::reduce_phi_on_safepoints_helper(Node* ophi, Node* cast, No nsr_merge_pointer = _igvn->transform(ConstraintCastNode::make_cast_for_type(cast->in(0), cast->in(1), new_t, ConstraintCastNode::RegularDependency, nullptr)); } + GrowableArray non_debug_edges_worklist; for (uint spi = 0; spi < safepoints.size(); spi++) { SafePointNode* sfpt = safepoints.at(spi)->as_SafePoint(); - JVMState *jvms = sfpt->jvms(); - uint merge_idx = (sfpt->req() - jvms->scloff()); - int debug_start = jvms->debug_start(); + + sfpt->remove_non_debug_edges(non_debug_edges_worklist); + + JVMState* jvms = sfpt->jvms(); + uint merge_idx = (sfpt->req() - jvms->scloff()); + int debug_start = jvms->debug_start(); SafePointScalarMergeNode* smerge = new SafePointScalarMergeNode(merge_t, merge_idx); smerge->init_req(0, _compile->root()); _igvn->register_new_node_with_optimizer(smerge); + assert(sfpt->jvms()->endoff() == sfpt->req(), "no extra edges past debug info allowed"); + // The next two inputs are: // (1) A copy of the original pointer to NSR objects. // (2) A selector, used to decide if we need to rematerialize an object // or use the pointer to a NSR object. - // See more details of these fields in the declaration of SafePointScalarMergeNode + // See more details of these fields in the declaration of SafePointScalarMergeNode. + // It is safe to include them into debug info straight away since create_scalarized_object_description() + // will include all newly added inputs into debug info anyway. sfpt->add_req(nsr_merge_pointer); sfpt->add_req(selector); + sfpt->jvms()->set_endoff(sfpt->req()); for (uint i = 1; i < ophi->req(); i++) { Node* base = ophi->in(i); @@ -1254,13 +1263,14 @@ bool ConnectionGraph::reduce_phi_on_safepoints_helper(Node* ophi, Node* cast, No AllocateNode* alloc = ptn->ideal_node()->as_Allocate(); SafePointScalarObjectNode* sobj = mexp.create_scalarized_object_description(alloc, sfpt); if (sobj == nullptr) { - return false; + return false; // non-recoverable failure; recompile } // Now make a pass over the debug information replacing any references // to the allocated object with "sobj" Node* ccpp = alloc->result_cast(); sfpt->replace_edges_in_range(ccpp, sobj, debug_start, jvms->debug_end(), _igvn); + non_debug_edges_worklist.remove_if_existing(ccpp); // drop scalarized input from non-debug info // Register the scalarized object as a candidate for reallocation smerge->add_req(sobj); @@ -1268,11 +1278,15 @@ bool ConnectionGraph::reduce_phi_on_safepoints_helper(Node* ophi, Node* cast, No // Replaces debug information references to "original_sfpt_parent" in "sfpt" with references to "smerge" sfpt->replace_edges_in_range(original_sfpt_parent, smerge, debug_start, jvms->debug_end(), _igvn); + non_debug_edges_worklist.remove_if_existing(original_sfpt_parent); // drop scalarized input from non-debug info // The call to 'replace_edges_in_range' above might have removed the // reference to ophi that we need at _merge_pointer_idx. The line below make // sure the reference is maintained. sfpt->set_req(smerge->merge_pointer_idx(jvms), nsr_merge_pointer); + + sfpt->restore_non_debug_edges(non_debug_edges_worklist); + _igvn->_worklist.push(sfpt); } @@ -4630,6 +4644,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, op == Op_StrIndexOf || op == Op_StrIndexOfChar || op == Op_SubTypeCheck || op == Op_ReinterpretS2HF || + op == Op_ReachabilityFence || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use))) { n->dump(); use->dump(); diff --git a/src/hotspot/share/opto/gcm.cpp b/src/hotspot/share/opto/gcm.cpp index 4a1553b1e0092..2d6402fd6eb61 100644 --- a/src/hotspot/share/opto/gcm.cpp +++ b/src/hotspot/share/opto/gcm.cpp @@ -152,9 +152,13 @@ bool PhaseCFG::is_CFG(Node* n) { } bool PhaseCFG::is_control_proj_or_safepoint(Node* n) const { - bool result = (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) || (n->is_Proj() && n->as_Proj()->bottom_type() == Type::CONTROL); - assert(!result || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) - || (n->is_Proj() && n->as_Proj()->_con == 0), "If control projection, it must be projection 0"); + bool result = n->is_ReachabilityFence() || + (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) || + (n->is_Proj() && n->as_Proj()->bottom_type() == Type::CONTROL); + assert(!result || + (n->is_ReachabilityFence()) || + (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_SafePoint) || + (n->is_Proj() && n->as_Proj()->_con == 0), "If control projection, it must be projection 0"); return result; } diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index d4776d9d2f0bd..d6243a54c7a16 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -41,6 +41,7 @@ #include "opto/machnode.hpp" #include "opto/opaquenode.hpp" #include "opto/parse.hpp" +#include "opto/reachability.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" #include "opto/subtypenode.hpp" @@ -3455,6 +3456,15 @@ Node* GraphKit::insert_mem_bar_volatile(int opcode, int alias_idx, Node* precede return membar; } +//------------------------------insert_reachability_fence---------------------- +Node* GraphKit::insert_reachability_fence(Node* referent) { + assert(!referent->is_top(), ""); + Node* rf = _gvn.transform(new ReachabilityFenceNode(C, control(), referent)); + set_control(rf); + C->record_for_igvn(rf); + return rf; +} + //------------------------------shared_lock------------------------------------ // Emit locking code. FastLockNode* GraphKit::shared_lock(Node* obj) { diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 806a211d7e254..9d5b6e90b0bed 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -796,6 +796,7 @@ class GraphKit : public Phase { int next_monitor(); Node* insert_mem_bar(int opcode, Node* precedent = nullptr); Node* insert_mem_bar_volatile(int opcode, int alias_idx, Node* precedent = nullptr); + Node* insert_reachability_fence(Node* referent); // Optional 'precedent' is appended as an extra edge, to force ordering. FastLockNode* shared_lock(Node* obj); void shared_unlock(Node* box, Node* obj); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index bd8a550b9ab76..dd005c12933bf 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -567,6 +567,7 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_Reference_get0: return inline_reference_get0(); case vmIntrinsics::_Reference_refersTo0: return inline_reference_refersTo0(false); + case vmIntrinsics::_Reference_reachabilityFence: return inline_reference_reachabilityFence(); case vmIntrinsics::_PhantomReference_refersTo0: return inline_reference_refersTo0(true); case vmIntrinsics::_Reference_clear0: return inline_reference_clear0(false); case vmIntrinsics::_PhantomReference_clear0: return inline_reference_clear0(true); @@ -7072,6 +7073,14 @@ bool LibraryCallKit::inline_reference_clear0(bool is_phantom) { return true; } +//-----------------------inline_reference_reachabilityFence----------------- +// bool java.lang.ref.Reference.reachabilityFence(); +bool LibraryCallKit::inline_reference_reachabilityFence() { + Node* referent = argument(0); + insert_reachability_fence(referent); + return true; +} + Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, DecoratorSet decorators, bool is_static, ciInstanceKlass* fromKls) { diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index fbac1363dae3d..46a431b803561 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -329,6 +329,7 @@ class LibraryCallKit : public GraphKit { bool inline_divmod_methods(vmIntrinsics::ID id); bool inline_reference_get0(); bool inline_reference_refersTo0(bool is_phantom); + bool inline_reference_reachabilityFence(); bool inline_reference_clear0(bool is_phantom); bool inline_Class_cast(); bool inline_aescrypt_Block(vmIntrinsics::ID id); diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index f92833e9e1c0a..81e2995624517 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -61,6 +61,24 @@ Node *IdealLoopTree::is_loop_exit(Node *iff) const { return nullptr; } +//------------------------------unique_loop_exit_or_null---------------------- +// Return the loop-exit projection if it is unique. +IfFalseNode* IdealLoopTree::unique_loop_exit_or_null() { + if (is_loop()) { + if (head()->is_BaseCountedLoop()) { + ProjNode* loop_exit = head()->as_BaseCountedLoop()->loopexit()->proj_out_or_null(0 /* false */); + if (loop_exit != nullptr) { + return loop_exit->as_IfFalse(); + } + } else if (head()->is_OuterStripMinedLoop()) { + return head()->as_OuterStripMinedLoop()->outer_loop_exit(); + } else { + // For now, conservatively report multiple loop exits exist. + } + } + return nullptr; // not found or multiple loop exists exist +} + //============================================================================= diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e8058edb4e5e5..0d7614cd12d76 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4680,6 +4680,9 @@ void IdealLoopTree::dump_head() { if (_has_call) tty->print(" has_call"); if (_has_sfpt) tty->print(" has_sfpt"); if (_rce_candidate) tty->print(" rce"); + if (_reachability_fences != nullptr && _reachability_fences->size() > 0) { + tty->print(" has_rf"); + } if (_safepts != nullptr && _safepts->size() > 0) { tty->print(" sfpts={"); _safepts->dump_simple(); tty->print(" }"); } @@ -4687,6 +4690,9 @@ void IdealLoopTree::dump_head() { tty->print(" req={"); _required_safept->dump_simple(); tty->print(" }"); } if (Verbose) { + if (_reachability_fences != nullptr && _reachability_fences->size() > 0) { + tty->print(" rfs={"); _reachability_fences->dump_simple(); tty->print(" }"); + } tty->print(" body={"); _body.dump_simple(); tty->print(" }"); } if (_head->is_Loop() && _head->as_Loop()->is_strip_mined()) { @@ -4927,12 +4933,10 @@ bool PhaseIdealLoop::process_expensive_nodes() { // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. void PhaseIdealLoop::build_and_optimize() { - assert(!C->post_loop_opts_phase(), "no loop opts allowed"); - bool do_split_ifs = (_mode == LoopOptsDefault); bool skip_loop_opts = (_mode == LoopOptsNone); bool do_max_unroll = (_mode == LoopOptsMaxUnroll); - + bool do_eliminate_reachability_fences = (_mode == PostLoopOptsEliminateReachabilityFences); int old_progress = C->major_progress(); uint orig_worklist_size = _igvn._worklist.size(); @@ -4999,11 +5003,13 @@ void PhaseIdealLoop::build_and_optimize() { BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); // Nothing to do, so get out - bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !do_max_unroll && !_verify_me && - !_verify_only && !bs->is_gc_specific_loop_opts_pass(_mode); + bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !do_max_unroll && + !do_eliminate_reachability_fences && !_verify_me && !_verify_only && + !bs->is_gc_specific_loop_opts_pass(_mode) ; bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); + bool do_optimize_reachability_fences = OptimizeReachabilityFences && (C->reachability_fences_count() > 0); bool strip_mined_loops_expanded = bs->strip_mined_loops_expanded(_mode); - if (stop_early && !do_expensive_nodes) { + if (stop_early && !do_expensive_nodes && !do_optimize_reachability_fences) { return; } @@ -5079,7 +5085,7 @@ void PhaseIdealLoop::build_and_optimize() { // Given early legal placement, try finding counted loops. This placement // is good enough to discover most loop invariants. - if (!_verify_me && !_verify_only && !strip_mined_loops_expanded) { + if (!_verify_me && !_verify_only && !strip_mined_loops_expanded && !do_eliminate_reachability_fences) { _ltree_root->counted_loop( this ); } @@ -5109,8 +5115,12 @@ void PhaseIdealLoop::build_and_optimize() { eliminate_useless_multiversion_if(); if (stop_early) { - assert(do_expensive_nodes, "why are we here?"); - if (process_expensive_nodes()) { + assert(do_expensive_nodes || do_optimize_reachability_fences, "why are we here?"); + if (do_optimize_reachability_fences && optimize_reachability_fences()) { + recompute_dom_depth(); + DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } ); + } + if (do_expensive_nodes && process_expensive_nodes()) { // If we made some progress when processing expensive nodes then // the IGVN may modify the graph in a way that will allow us to // make some more progress: we need to try processing expensive @@ -5138,6 +5148,22 @@ void PhaseIdealLoop::build_and_optimize() { } #endif + if (do_optimize_reachability_fences && optimize_reachability_fences()) { + recompute_dom_depth(); + DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } ); + } + + if (do_eliminate_reachability_fences) { + assert(C->post_loop_opts_phase(), "required"); + if (eliminate_reachability_fences()) { + recompute_dom_depth(); + DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } ); + } + return; + } + + assert(!C->post_loop_opts_phase(), "required"); + if (skip_loop_opts) { C->restore_major_progress(old_progress); return; @@ -6028,6 +6054,11 @@ int PhaseIdealLoop::build_loop_tree_impl(Node* n, int pre_order) { // Record all safepoints in this loop. if (innermost->_safepts == nullptr) innermost->_safepts = new Node_List(); innermost->_safepts->push(n); + } else if (n->is_ReachabilityFence()) { + if (innermost->_reachability_fences == nullptr) { + innermost->_reachability_fences = new Node_List(); + } + innermost->_reachability_fences->push(n); } } } @@ -7230,7 +7261,7 @@ void LoopTreeIterator::next() { assert(!done(), "must not be done."); if (_curnt->_child != nullptr) { _curnt = _curnt->_child; - } else if (_curnt->_next != nullptr) { + } else if (_curnt != _root && _curnt->_next != nullptr) { _curnt = _curnt->_next; } else { while (_curnt != _root && _curnt->_next == nullptr) { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 1101de815953d..fa869c6126cee 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -44,6 +44,7 @@ class PredicateBlock; class PathFrequency; class PhaseIdealLoop; class LoopSelector; +class ReachabilityFenceNode; class UnswitchedLoopSelector; class VectorSet; class VSharedData; @@ -667,6 +668,7 @@ class IdealLoopTree : public ResourceObj { Node_List* _safepts; // List of safepoints in this loop Node_List* _required_safept; // A inner loop cannot delete these safepts; + Node_List* _reachability_fences; // List of reachability fences in this loop bool _allow_optimizations; // Allow loop optimizations IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail ) @@ -679,6 +681,7 @@ class IdealLoopTree : public ResourceObj { _has_range_checks(0), _has_range_checks_computed(0), _safepts(nullptr), _required_safept(nullptr), + _reachability_fences(nullptr), _allow_optimizations(true) { precond(_head != nullptr); @@ -739,6 +742,9 @@ class IdealLoopTree : public ResourceObj { // Check for Node being a loop-breaking test Node *is_loop_exit(Node *iff) const; + // Return unique loop-exit projection or null if there are multiple exits exist. + IfFalseNode* unique_loop_exit_or_null(); + // Remove simplistic dead code from loop body void DCE_loop_body(); @@ -1140,6 +1146,17 @@ class PhaseIdealLoop : public PhaseTransform { lazy_update(old_node, new_node); } + void remove_dead_node(Node* dead) { + assert(dead->outcnt() == 0 && !dead->is_top(), "node must be dead"); + Node* c = get_ctrl(dead); + IdealLoopTree* lpt = get_loop(c); + _loop_or_ctrl.map(dead->_idx, nullptr); // This node is useless + if (!lpt->is_root()) { + lpt->_body.yank(dead); + } + igvn().remove_dead_node(dead); + } + private: // Place 'n' in some loop nest, where 'n' is a CFG node @@ -1463,6 +1480,20 @@ class PhaseIdealLoop : public PhaseTransform { // Implementation of the loop predication to promote checks outside the loop bool loop_predication_impl(IdealLoopTree *loop); + // Reachability Fence (RF) support. + private: + bool is_redundant_rf(ReachabilityFenceNode* rf, bool rf_only); + bool find_redundant_rfs(Unique_Node_List& redundant_rfs); + void insert_rf(Node* ctrl, Node* referent); + void replace_rf(Node* old_node, Node* new_node); + void remove_rf(ReachabilityFenceNode* rf); +#ifdef ASSERT + bool has_redundant_rfs(Unique_Node_List& ignored_rfs, bool rf_only); +#endif // ASSERT + public: + bool optimize_reachability_fences(); + bool eliminate_reachability_fences(); + private: bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero, diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 88cb59c115fc2..a107a8606c03e 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -44,6 +44,7 @@ #include "opto/node.hpp" #include "opto/opaquenode.hpp" #include "opto/phaseX.hpp" +#include "opto/reachability.hpp" #include "opto/rootnode.hpp" #include "opto/runtime.hpp" #include "opto/subnode.hpp" @@ -612,6 +613,8 @@ bool PhaseMacroExpand::can_eliminate_allocation(PhaseIterGVN* igvn, AllocateNode use->as_ArrayCopy()->is_copyofrange_validated()) && use->in(ArrayCopyNode::Dest) == res) { // ok to eliminate + } else if (use->is_ReachabilityFence() && OptimizeReachabilityFences) { + // ok to eliminate } else if (use->is_SafePoint()) { SafePointNode* sfpt = use->as_SafePoint(); if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) { @@ -705,9 +708,14 @@ void PhaseMacroExpand::undo_previous_scalarizations(GrowableArray non_debug_edges_worklist; while (safepoints_done.length() > 0) { SafePointNode* sfpt_done = safepoints_done.pop(); + + sfpt_done->remove_non_debug_edges(non_debug_edges_worklist); + // remove any extra entries we added to the safepoint + assert(sfpt_done->jvms()->endoff() == sfpt_done->req(), "no extra edges past debug info allowed"); uint last = sfpt_done->req() - 1; for (int k = 0; k < nfields; k++) { sfpt_done->del_req(last--); @@ -728,11 +736,16 @@ void PhaseMacroExpand::undo_previous_scalarizations(GrowableArray restore_non_debug_edges(non_debug_edges_worklist); + _igvn._worklist.push(sfpt_done); } } SafePointScalarObjectNode* PhaseMacroExpand::create_scalarized_object_description(AllocateNode *alloc, SafePointNode* sfpt) { + assert(sfpt->jvms()->endoff() == sfpt->req(), "no extra edges past debug info allowed"); + // Fields of scalar objs are referenced only at the end // of regular debuginfo at the last (youngest) JVMS. // Record relative start index. @@ -857,17 +870,22 @@ SafePointScalarObjectNode* PhaseMacroExpand::create_scalarized_object_descriptio } // Do scalar replacement. -bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray & safepoints) { - GrowableArray safepoints_done; +bool PhaseMacroExpand::scalar_replacement(AllocateNode* alloc, GrowableArray& safepoints) { + GrowableArray safepoints_done; + GrowableArray non_debug_edges_worklist; Node* res = alloc->result_cast(); assert(res == nullptr || res->is_CheckCastPP(), "unexpected AllocateNode result"); // Process the safepoint uses while (safepoints.length() > 0) { SafePointNode* sfpt = safepoints.pop(); + + sfpt->remove_non_debug_edges(non_debug_edges_worklist); + SafePointScalarObjectNode* sobj = create_scalarized_object_description(alloc, sfpt); if (sobj == nullptr) { + sfpt->restore_non_debug_edges(non_debug_edges_worklist); undo_previous_scalarizations(safepoints_done, alloc); return false; } @@ -876,6 +894,8 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray jvms(); sfpt->replace_edges_in_range(res, sobj, jvms->debug_start(), jvms->debug_end(), &_igvn); + non_debug_edges_worklist.remove_if_existing(res); // drop scalarized input from non-debug info + sfpt->restore_non_debug_edges(non_debug_edges_worklist); _igvn._worklist.push(sfpt); // keep it for rollback @@ -966,6 +986,8 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { } } _igvn._worklist.push(ac); + } else if (use->is_ReachabilityFence() && OptimizeReachabilityFences) { + use->as_ReachabilityFence()->clear_referent(_igvn); // redundant fence; will be removed during IGVN } else { eliminate_gc_barrier(use); } diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 9187ef1a36150..acd88089b5422 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -3955,7 +3955,7 @@ uint LoadStoreNode::ideal_reg() const { bool LoadStoreNode::result_not_used() const { for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node *x = fast_out(i); - if (x->Opcode() == Op_SCMemProj) { + if (x->Opcode() == Op_SCMemProj || x->is_ReachabilityFence()) { continue; } if (x->bottom_type() == TypeTuple::MEMBAR && diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 497d0d1aeb0bd..5c5b49d98eb19 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -38,6 +38,7 @@ #include "opto/matcher.hpp" #include "opto/node.hpp" #include "opto/opcodes.hpp" +#include "opto/reachability.hpp" #include "opto/regmask.hpp" #include "opto/rootnode.hpp" #include "opto/type.hpp" @@ -503,6 +504,9 @@ Node *Node::clone() const { if (is_expensive()) { C->add_expensive_node(n); } + if (is_ReachabilityFence()) { + C->add_reachability_fence(n->as_ReachabilityFence()); + } if (for_post_loop_opts_igvn()) { // Don't add cloned node to Compile::_for_post_loop_opts_igvn list automatically. // If it is applicable, it will happen anyway when the cloned node is registered with IGVN. @@ -622,6 +626,9 @@ void Node::destruct(PhaseValues* phase) { if (is_expensive()) { compile->remove_expensive_node(this); } + if (is_ReachabilityFence()) { + compile->remove_reachability_fence(as_ReachabilityFence()); + } if (is_OpaqueTemplateAssertionPredicate()) { compile->remove_template_assertion_predicate_opaque(as_OpaqueTemplateAssertionPredicate()); } @@ -2966,6 +2973,25 @@ bool Node::is_data_proj_of_pure_function(const Node* maybe_pure_function) const return Opcode() == Op_Proj && as_Proj()->_con == TypeFunc::Parms && maybe_pure_function->is_CallLeafPure(); } +//--------------------------has_non_debug_uses------------------------------ +// Checks whether the node has any non-debug uses or not. +bool Node::has_non_debug_uses() const { + for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { + Node* u = fast_out(i); + if (u->is_SafePoint()) { + if (u->is_Call() && u->as_Call()->has_non_debug_use(this)) { + return true; + } + // Non-call safepoints have only debug uses. + } else if (u->is_ReachabilityFence()) { + // Reachability fence is treated as debug use. + } else { + return true; // everything else is conservatively treated as non-debug use + } + } + return false; // no non-debug uses found +} + //============================================================================= //------------------------------yank------------------------------------------- // Find and remove diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 9ce9e705eec27..406a76b4411a7 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -166,6 +166,7 @@ class Pipeline; class PopulateIndexNode; class ProjNode; class RangeCheckNode; +class ReachabilityFenceNode; class ReductionNode; class RegMask; class RegionNode; @@ -443,6 +444,9 @@ class Node { // Check whether node has become unreachable bool is_unreachable(PhaseIterGVN &igvn) const; + // Does the node have any immediate non-debug uses? + bool has_non_debug_uses() const; + // Set a required input edge, also updates corresponding output edge void add_req( Node *n ); // Append a NEW required input void add_req( Node *n0, Node *n1 ) { @@ -697,6 +701,7 @@ class Node { DEFINE_CLASS_ID(MemBar, Multi, 3) DEFINE_CLASS_ID(Initialize, MemBar, 0) DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) + DEFINE_CLASS_ID(ReachabilityFence, Multi, 4) DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) @@ -999,6 +1004,7 @@ class Node { DEFINE_CLASS_QUERY(PCTable) DEFINE_CLASS_QUERY(Phi) DEFINE_CLASS_QUERY(Proj) + DEFINE_CLASS_QUERY(ReachabilityFence) DEFINE_CLASS_QUERY(Reduction) DEFINE_CLASS_QUERY(Region) DEFINE_CLASS_QUERY(Root) diff --git a/src/hotspot/share/opto/parse.hpp b/src/hotspot/share/opto/parse.hpp index 19b302a89246a..97c4f27330782 100644 --- a/src/hotspot/share/opto/parse.hpp +++ b/src/hotspot/share/opto/parse.hpp @@ -356,6 +356,7 @@ class Parse : public GraphKit { bool _wrote_stable; // Did we write a @Stable field? bool _wrote_fields; // Did we write any field? Node* _alloc_with_final_or_stable; // An allocation node with final or @Stable field + Node* _stress_rf_hook; // StressReachabilityFences support // Variables which track Java semantics during bytecode parsing: diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 7aa96d2ace3db..34ad525d0832a 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -370,6 +370,15 @@ void Parse::load_interpreter_state(Node* osr_buf) { continue; } set_local(index, check_interpreter_type(l, type, bad_type_exit)); + if (StressReachabilityFences && type->isa_oopptr() != nullptr) { + // Keep all oop locals alive until the method returns as if there are + // reachability fences for them at the end of the method. + Node* loc = local(index); + if (loc->bottom_type() != TypePtr::NULL_PTR) { + assert(loc->bottom_type()->isa_oopptr() != nullptr, "%s", Type::str(loc->bottom_type())); + _stress_rf_hook->add_req(loc); + } + } } for (index = 0; index < sp(); index++) { @@ -378,6 +387,15 @@ void Parse::load_interpreter_state(Node* osr_buf) { if (l->is_top()) continue; // nothing here const Type *type = osr_block->stack_type_at(index); set_stack(index, check_interpreter_type(l, type, bad_type_exit)); + if (StressReachabilityFences && type->isa_oopptr() != nullptr) { + // Keep all oops on stack alive until the method returns as if there are + // reachability fences for them at the end of the method. + Node* stk = stack(index); + if (stk->bottom_type() != TypePtr::NULL_PTR) { + assert(stk->bottom_type()->isa_oopptr() != nullptr, "%s", Type::str(stk->bottom_type())); + _stress_rf_hook->add_req(stk); + } + } } if (bad_type_exit->control()->req() > 1) { @@ -412,6 +430,7 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) _wrote_stable = false; _wrote_fields = false; _alloc_with_final_or_stable = nullptr; + _stress_rf_hook = (StressReachabilityFences ? new Node(1) : nullptr); _block = nullptr; _first_return = true; _replaced_nodes_for_exceptions = false; @@ -643,6 +662,11 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) if (log) log->done("parse nodes='%d' live='%d' memory='%zu'", C->unique(), C->live_nodes(), C->node_arena()->used()); + + if (StressReachabilityFences) { + _stress_rf_hook->destruct(&_gvn); + _stress_rf_hook = nullptr; + } } //---------------------------do_all_blocks------------------------------------- @@ -1208,6 +1232,18 @@ void Parse::do_method_entry() { make_dtrace_method_entry(method()); } + if (StressReachabilityFences) { + // Keep all oop arguments alive until the method returns as if there are + // reachability fences for them at the end of the method. + int max_locals = jvms()->loc_size(); + for (int idx = 0; idx < max_locals; idx++) { + Node* loc = local(idx); + if (loc->bottom_type()->isa_oopptr() != nullptr) { + _stress_rf_hook->add_req(loc); + } + } + } + #ifdef ASSERT // Narrow receiver type when it is too broad for the method being parsed. if (!method()->is_static()) { @@ -2194,6 +2230,15 @@ void Parse::return_current(Node* value) { call_register_finalizer(); } + if (StressReachabilityFences) { + // Insert reachability fences for all oop arguments at the end of the method. + for (uint i = 1; i < _stress_rf_hook->req(); i++) { + Node* referent = _stress_rf_hook->in(i); + assert(referent->bottom_type()->isa_oopptr(), "%s", Type::str(referent->bottom_type())); + insert_reachability_fence(referent); + } + } + // Do not set_parse_bci, so that return goo is credited to the return insn. set_bci(InvocationEntryBci); if (method()->is_synchronized()) { diff --git a/src/hotspot/share/opto/phase.cpp b/src/hotspot/share/opto/phase.cpp index 5603033ce69d8..4811aabedbfe5 100644 --- a/src/hotspot/share/opto/phase.cpp +++ b/src/hotspot/share/opto/phase.cpp @@ -90,6 +90,9 @@ void Phase::print_timers() { tty->print_cr (" Prune Useless: %7.3f s", timers[_t_vector_pru].seconds()); tty->print_cr (" Renumber Live: %7.3f s", timers[_t_renumberLive].seconds()); tty->print_cr (" IdealLoop: %7.3f s", timers[_t_idealLoop].seconds()); + tty->print_cr (" ReachabilityFence: %7.3f s", timers[_t_reachability].seconds()); + tty->print_cr (" Optimize: %7.3f s", timers[_t_reachability_optimize].seconds()); + tty->print_cr (" Eliminate: %7.3f s", timers[_t_reachability_eliminate].seconds()); tty->print_cr (" AutoVectorize: %7.3f s", timers[_t_autoVectorize].seconds()); tty->print_cr (" IdealLoop Verify: %7.3f s", timers[_t_idealLoopVerify].seconds()); tty->print_cr (" Cond Const Prop: %7.3f s", timers[_t_ccp].seconds()); diff --git a/src/hotspot/share/opto/phase.hpp b/src/hotspot/share/opto/phase.hpp index 6700df6ec177e..0b05788499fdc 100644 --- a/src/hotspot/share/opto/phase.hpp +++ b/src/hotspot/share/opto/phase.hpp @@ -85,6 +85,9 @@ class Phase : public StackObj { f( _t_vector_pru, "vector_pru") \ f( _t_renumberLive, "") \ f( _t_idealLoop, "idealLoop") \ + f( _t_reachability, "reachabilityFence") \ + f( _t_reachability_optimize, "reachabilityFence_optimize") \ + f( _t_reachability_eliminate, "reachabilityFence_eliminate") \ f( _t_autoVectorize, "autoVectorize") \ f( _t_idealLoopVerify, "idealLoopVerify") \ f( _t_ccp, "ccp") \ diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index ce5160c5984aa..2c634b719aedb 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -2040,6 +2040,16 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) { return false; } + if (n->is_ReachabilityFence()) { + // ReachabilityFencesNode::Identity() walks CFG to determine whether the node is redundant or not. + // But since there's no accurate dominance information avaialble, it reports a conservative answer. + // So, any change in CFG can potentially expose a ReachabilityFence node as redundant. + // + // Found with: + // java -XX:VerifyIterativeGVN=1000 -Xcomp -XX:+StressReachabilityFences + return false; + } + Node* i = n->Identity(this); // If we cannot find any other Identity, we are happy. if (i == n) { diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index f24938b51c185..65f19f1eae3ed 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -88,6 +88,7 @@ flags(PHASEIDEALLOOP1, "PhaseIdealLoop 1") \ flags(PHASEIDEALLOOP2, "PhaseIdealLoop 2") \ flags(PHASEIDEALLOOP3, "PhaseIdealLoop 3") \ + flags(ELIMINATE_REACHABILITY_FENCES, "Eliminate Reachability Fences") \ flags(AUTO_VECTORIZATION1_BEFORE_APPLY, "AutoVectorization 1, before Apply") \ flags(AUTO_VECTORIZATION3_AFTER_ADJUST_LIMIT, "AutoVectorization 2, after Adjusting Pre-loop Limit") \ flags(AUTO_VECTORIZATION4_AFTER_SPECULATIVE_RUNTIME_CHECKS, "AutoVectorization 3, after Adding Speculative Runtime Checks") \ diff --git a/src/hotspot/share/opto/reachability.cpp b/src/hotspot/share/opto/reachability.cpp new file mode 100644 index 0000000000000..ef516d2c280c9 --- /dev/null +++ b/src/hotspot/share/opto/reachability.cpp @@ -0,0 +1,554 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "opto/c2_MacroAssembler.hpp" +#include "opto/callnode.hpp" +#include "opto/compile.hpp" +#include "opto/loopnode.hpp" +#include "opto/phaseX.hpp" +#include "opto/reachability.hpp" +#include "opto/regalloc.hpp" +#include "opto/runtime.hpp" + +/* + * java.lang.ref.Reference::reachabilityFence support. + * + * Reachability Fence (RF) ensures that the given object (referent) remains strongly reachable + * regardless of any optimizing transformations the virtual machine may perform that might otherwise + * allow the object to become unreachable. + * + * RFs are intended to be used in performance-critical code, so the primary goal for C2 support is + * to reduce their runtime overhead as much as possible. + * + * Reference::reachabilityFence() calls are intrinsified into ReachabilityFence CFG nodes. RF node keeps + * its referent alive, so the referent's location is recorded at every safepoint (in its oop map) which + * interferes with referent's live range. + * + * It is tempting to directly attach referents to interfering safepoints right from the beginning, but it + * doesn't play well with some optimizations C2 does (e.g., during loop-invariant code motion a safepoint + * can become interfering once a load is hoisted). + * + * Instead, reachability representation transitions through multiple phases: + * (0) initial set of RFs is materialized during parsing (as a result of + * Reference.reachabilityFence intrinsification); + * (1) optimization pass during loop opts eliminates redundant RF nodes and + * moves the ones with loop-invariant referents outside (after) loops; + * (2) after loop opts are over, RF nodes are eliminated and their referents are transferred to + * safepoint nodes (appended as edges after debug info); + * (3) during final graph reshaping, referent edges are removed from safepoints and materialized as RF nodes + * attached to their safepoint node (closely following it in CFG graph). + * + * Some implementation considerations. + * + * (a) It looks attractive to get rid of RF nodes early and transfer to safepoint-attached representation, + * but it is not correct until loop opts are done. + * + * Live ranges of values are routinely extended during loop opts. And it can break the invariant that + * all interfering safepoints contain the referent in their oop map. (If an interfering safepoint doesn't + * keep the referent alive, then it becomes possible for the referent to be prematurely GCed.) + * + * After loop opts are over, it becomes possible to reliably enumerate all interfering safe points and + * to ensure that the referent is present in their oop maps. + * + * (b) RF nodes may interfere with Register Allocator (RA). If a safepoint is pruned during macro expansion, + * it can make some RF nodes redundant, but we don't have information about their relations anymore to detect that. + * Redundant RF node unnecessarily extends referent's live range and increases register pressure. + * + * Hence, we eliminate RF nodes and transfer their referents to corresponding safepoints (phase #2). + * When safepoints are pruned, corresponding reachability edges also go away. + * + * (c) Unfortunately, it's not straightforward to stay with safepoint-attached representation till the very end, + * because information about derived oops is attached to safepoints in a similar way. So, for now RFs are + * rematerialized at safepoints before RA (phase #3). + */ + +// RF is redundant for some referent oop when the referent has another user which keeps it alive across the RF. +// In terms of dominance relation it can be formulated as "a referent has a user which is dominated by the redundant RF". +// Until loop opts are over, only RF nodes are considered as usages (controlled by rf_only flag). +static bool is_redundant_rf_helper(Node* ctrl, Node* referent, PhaseIdealLoop* phase, PhaseGVN& gvn, bool rf_only) { + const Type* t = gvn.type(referent); + if (!PreserveReachabilityFencesOnConstants && t->singleton()) { + return true; // no-op fence + } + if (t == TypePtr::NULL_PTR) { + return true; // no-op fence + } + if (referent->is_Proj() && referent->in(0)->is_CallJava()) { + ciMethod* m = referent->in(0)->as_CallJava()->method(); + if (m != nullptr && m->is_boxing_method()) { + return true; + } + } + for (Node* cur = referent; + cur != nullptr; + cur = (cur->is_ConstraintCast() ? cur->in(1) : nullptr)) { + for (DUIterator_Fast imax, i = cur->fast_outs(imax); i < imax; i++) { + Node* use = cur->fast_out(i); + if (rf_only && !use->is_ReachabilityFence()) { + continue; // skip non-RF uses + } + if (use != ctrl) { + if (phase != nullptr) { + Node* use_ctrl = (rf_only ? use : phase->ctrl_or_self(use)); + if (phase->is_dominator(ctrl, use_ctrl)) { + return true; + } + } else { + assert(rf_only, ""); + if (gvn.is_dominator(ctrl, use)) { + return true; + } + } + } + } + } + return false; +} + +Node* ReachabilityFenceNode::Ideal(PhaseGVN* phase, bool can_reshape) { + return (remove_dead_region(phase, can_reshape) ? this : nullptr); +} + +Node* ReachabilityFenceNode::Identity(PhaseGVN* phase) { + if (is_redundant_rf_helper(this, in(1), nullptr, *phase, true /*rf_only*/)) { + return in(0); + } + return this; +} + +// Turn the RF node into a no-op by setting it's referent to null. +// Subsequent IGVN pass removes cleared nodes. +bool ReachabilityFenceNode::clear_referent(PhaseIterGVN& phase) { + if (phase.type(in(1)) == TypePtr::NULL_PTR) { + return false; + } else { + phase.replace_input_of(this, 1, phase.makecon(TypePtr::NULL_PTR)); + return true; + } +} + +#ifndef PRODUCT +static void rf_desc(outputStream* st, const ReachabilityFenceNode* rf, PhaseRegAlloc* ra) { + char buf[50]; + ra->dump_register(rf->in(1), buf, sizeof(buf)); + st->print("reachability fence [%s]", buf); +} + +void ReachabilityFenceNode::format(PhaseRegAlloc* ra, outputStream* st) const { + rf_desc(st, this, ra); +} + +void ReachabilityFenceNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra) const { + ResourceMark rm; + stringStream ss; + rf_desc(&ss, this, ra); + const char* desc = masm->code_string(ss.freeze()); + masm->block_comment(desc); +} +#endif + +// Detect safepoint nodes which are important for reachability tracking purposes. +static bool is_significant_sfpt(Node* n) { + if (n->is_SafePoint()) { + SafePointNode* sfpt = n->as_SafePoint(); + if (sfpt->jvms() == nullptr) { + return false; // not a real safepoint + } else if (sfpt->is_CallStaticJava() && sfpt->as_CallStaticJava()->is_uncommon_trap()) { + return false; // uncommon traps are exit points + } + return true; + } + return false; +} + +void PhaseIdealLoop::insert_rf(Node* ctrl, Node* referent) { + IdealLoopTree* lpt = get_loop(ctrl); + Node* ctrl_end = ctrl->unique_ctrl_out(); + + Node* new_rf = new ReachabilityFenceNode(C, ctrl, referent); + + register_control(new_rf, lpt, ctrl); + set_idom(new_rf, ctrl, dom_depth(ctrl) + 1); + if (lpt->_reachability_fences == nullptr) { + lpt->_reachability_fences = new Node_List(); + } + lpt->_reachability_fences->push(new_rf); + + igvn().rehash_node_delayed(ctrl_end); + ctrl_end->replace_edge(ctrl, new_rf); + + if (idom(ctrl_end) == ctrl) { + set_idom(ctrl_end, new_rf, dom_depth(new_rf) + 1); + } else { + assert(ctrl_end->is_Region(), ""); + } +} + +void PhaseIdealLoop::replace_rf(Node* old_node, Node* new_node) { + assert(old_node->is_ReachabilityFence() || + (old_node->is_Proj() && old_node->in(0)->is_ReachabilityFence()), + "%s", NodeClassNames[old_node->Opcode()]); + + IdealLoopTree* lpt = get_loop(old_node); + if (!lpt->is_root()) { + lpt->_body.yank(old_node); + } + assert(lpt->_reachability_fences != nullptr, "missing"); + assert(lpt->_reachability_fences->contains(old_node), "missing"); + lpt->_reachability_fences->yank(old_node); + lazy_replace(old_node, new_node); +} + +void PhaseIdealLoop::remove_rf(ReachabilityFenceNode* rf) { + Node* referent = rf->in(1); + if (igvn().type(referent) != TypePtr::NULL_PTR) { + igvn().replace_input_of(rf, 1, makecon(TypePtr::NULL_PTR)); + if (referent->outcnt() == 0) { + remove_dead_node(referent); + } + } + Node* rf_ctrl_in = rf->in(0); + replace_rf(rf, rf_ctrl_in); +} + +bool PhaseIdealLoop::is_redundant_rf(ReachabilityFenceNode* rf, bool rf_only) { + Node* referent = rf->in(1); + return is_redundant_rf_helper(rf, referent, this, igvn(), rf_only); +} + +// Updates the unique list of redundant RFs. +// Returns true if new instances of redundant fences are found. +bool PhaseIdealLoop::find_redundant_rfs(Unique_Node_List& redundant_rfs) { + bool found = false; + for (int i = 0; i < C->reachability_fences_count(); i++) { + ReachabilityFenceNode* rf = C->reachability_fence(i); + Node* referent = rf->in(1); + assert(rf->outcnt() > 0, "dead node"); + if (!redundant_rfs.member(rf) && is_redundant_rf(rf, true /*rf_only*/)) { + redundant_rfs.push(rf); + found = true; + } + } + return found; +} + +#ifdef ASSERT +static void dump_rfs_on(outputStream* st, PhaseIdealLoop* phase, Unique_Node_List& redundant_rfs, bool rf_only) { + for (int i = 0; i < phase->C->reachability_fences_count(); i++) { + Node* rf = phase->C->reachability_fence(i); + Node* referent = rf->in(1); + bool detected = redundant_rfs.member(rf); + bool redundant = is_redundant_rf_helper(rf, referent, phase, phase->igvn(), rf_only); + + st->print(" %3d: %s%s ", i, (redundant ? "R" : " "), (detected ? "D" : " ")); + rf->dump("", false, st); + st->cr(); + + st->print(" "); + referent->dump("", false, st); + st->cr(); + if (redundant != detected) { + for (Node* cur = referent; + cur != nullptr; + cur = (cur->is_ConstraintCast() ? cur->in(1) : nullptr)) { + bool first = true; + for (DUIterator_Fast imax, i = cur->fast_outs(imax); i < imax; i++) { + Node* use = cur->fast_out(i); + if (rf_only && !use->is_ReachabilityFence()) { + continue; // skip non-RF uses + } + if (use != rf) { + Node* use_ctrl = (rf_only ? use : phase->ctrl_or_self(use)); + if (phase->is_dominator(rf, use_ctrl)) { + if (first) { + st->print("=====REF "); cur->dump("", false, st); st->cr(); + first = false; + } + st->print(" D "); use_ctrl->dump("", false, st); st->cr(); + if (use != use_ctrl) { + st->print(" "); use->dump("", false, st); st->cr(); + } + } + } + } + } + } + } +} + +bool PhaseIdealLoop::has_redundant_rfs(Unique_Node_List& ignored_rfs, bool rf_only) { + for (int i = 0; i < C->reachability_fences_count(); i++) { + ReachabilityFenceNode* rf = C->reachability_fence(i); + Node* referent = rf->in(1); + assert(rf->outcnt() > 0, "dead node"); + if (ignored_rfs.member(rf)) { + continue; // skip + } else if (is_redundant_rf(rf, rf_only)) { + dump_rfs_on(tty, this, ignored_rfs, rf_only); + return true; + } + } + return false; +} +#endif // ASSERT + + +//====================================================================== +//---------------------------- Phase 1 --------------------------------- +// Optimization pass over reachability fences during loop opts. +// Eliminate redundant RFs and move RFs with loop-invariant referent out of the loop. +bool PhaseIdealLoop::optimize_reachability_fences() { + Compile::TracePhase tp(_t_reachability_optimize); + + assert(OptimizeReachabilityFences, "required"); + + Unique_Node_List redundant_rfs; + find_redundant_rfs(redundant_rfs); + + Node_List worklist; + for (int i = 0; i < C->reachability_fences_count(); i++) { + Node* rf = C->reachability_fence(i); + if (!redundant_rfs.member(rf)) { + // Move RFs out of counted loops when possible. + IdealLoopTree* lpt = get_loop(rf); + Node* referent = rf->in(1); + Node* loop_exit = lpt->unique_loop_exit_or_null(); + if (lpt->is_invariant(referent) && loop_exit != nullptr) { + // Switch to the outermost loop. + for (IdealLoopTree* outer_loop = lpt->_parent; + outer_loop->is_invariant(referent) && outer_loop->unique_loop_exit_or_null() != nullptr; + outer_loop = outer_loop->_parent) { + assert(is_member(outer_loop, rf), ""); + loop_exit = outer_loop->unique_loop_exit_or_null(); + } + assert(loop_exit != nullptr, ""); + worklist.push(referent); + worklist.push(loop_exit); + redundant_rfs.push(rf); + } + } + } + + // Populate RFs outside counted loops. + while (worklist.size() > 0) { + Node* ctrl_out = worklist.pop(); + Node* referent = worklist.pop(); + insert_rf(ctrl_out, referent); + } + + // Redundancy is determined by dominance relation. + // Sometimes it becomes evident that an RF is redundant once it is moved out of the loop. + // Also, newly introduced RF can make some existing RFs redundant. + find_redundant_rfs(redundant_rfs); + + // Eliminate redundant RFs. + bool progress = (redundant_rfs.size() > 0); + while (redundant_rfs.size() > 0) { + remove_rf(redundant_rfs.pop()->as_ReachabilityFence()); + } + + assert(redundant_rfs.size() == 0, ""); + assert(!has_redundant_rfs(redundant_rfs, true /*rf_only*/), ""); + + return progress; +} + +//====================================================================== +//---------------------------- Phase 2 --------------------------------- + +// Linearly traverse CFG upwards starting at n until first merge point. +// All encountered safepoints are recorded in safepoints list. +static void linear_traversal(Node* n, Node_Stack& worklist, VectorSet& visited, Node_List& safepoints) { + for (Node* ctrl = n; ctrl != nullptr; ctrl = ctrl->in(0)) { + assert(ctrl->is_CFG(), ""); + if (visited.test_set(ctrl->_idx)) { + return; + } else { + if (ctrl->is_Region()) { + worklist.push(ctrl, 1); + return; // stop at merge points + } else if (is_significant_sfpt(ctrl)) { + safepoints.push(ctrl); + } + } + } +} + +// Enumerate all safepoints which are reachable from the RF to its referent through CFG. +// Start at RF node and traverse CFG upwards until referent's control node is reached. +static void enumerate_interfering_sfpts(Node* rf, PhaseIdealLoop* phase, Node_List& safepoints) { + Node* referent = rf->in(1); + Node* referent_ctrl = phase->get_ctrl(referent); + assert(phase->is_dominator(referent_ctrl, rf), "sanity"); + + VectorSet visited; + visited.set(referent_ctrl->_idx); // end point + + Node_Stack stack(0); + linear_traversal(rf, stack, visited, safepoints); // start point + while (stack.is_nonempty()) { + Node* cur = stack.node(); + uint idx = stack.index(); + + assert(cur != nullptr, ""); + assert(cur->is_Region(), "%s", NodeClassNames[cur->Opcode()]); + assert(phase->is_dominator(referent_ctrl, cur), ""); + assert(idx > 0 && idx <= cur->req(), "%d %d", idx, cur->req()); + + if (idx < cur->req()) { + stack.set_index(idx + 1); + linear_traversal(cur->in(idx), stack, visited, safepoints); + } else { + stack.pop(); + } + } +} + +// Start offset for reachability info on a safepoint node. +static uint rf_base_offset(SafePointNode* sfpt) { + return sfpt->jvms()->oopoff(); +} + +// Phase 2: migrate reachability info to safepoints. +// All RFs are replaced with edges from corresponding referents to interfering safepoints. +// Interfering safepoints are safepoint nodes which are reachable from the RF to its referent through CFG. +bool PhaseIdealLoop::eliminate_reachability_fences() { + Compile::TracePhase tp(_t_reachability_eliminate); + + assert(OptimizeReachabilityFences, "required"); + assert(C->post_loop_opts_phase(), "required"); + DEBUG_ONLY( int no_of_constant_rfs = 0; ) + + ResourceMark rm; + Unique_Node_List redundant_rfs; + Node_List worklist; + for (int i = 0; i < C->reachability_fences_count(); i++) { + ReachabilityFenceNode* rf = C->reachability_fence(i); + assert(!is_redundant_rf(rf, true /*rf_only*/), "missed"); + if (PreserveReachabilityFencesOnConstants) { + const Type* referent_t = igvn().type(rf->in(1)); + assert(referent_t != TypePtr::NULL_PTR, "redundant rf"); + bool is_constant_rf = referent_t->singleton(); + if (is_constant_rf) { + DEBUG_ONLY( no_of_constant_rfs += 1; ) + continue; // don't eliminate constant rfs + } + } + if (!is_redundant_rf(rf, false /*rf_only*/)) { + Node_List safepoints; + enumerate_interfering_sfpts(rf, this, safepoints); + + Node* referent = rf->in(1); + while (safepoints.size() > 0) { + SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); + assert(is_dominator(get_ctrl(referent), sfpt), ""); + assert(sfpt->req() == rf_base_offset(sfpt), "no extra edges allowed"); + if (sfpt->find_edge(referent) == -1) { + worklist.push(sfpt); + worklist.push(referent); + } + } + } + redundant_rfs.push(rf); + } + + while (worklist.size() > 0) { + Node* referent = worklist.pop(); + Node* sfpt = worklist.pop(); + sfpt->add_req(referent); + igvn()._worklist.push(sfpt); + } + + // Eliminate redundant RFs. + bool progress = (redundant_rfs.size() > 0); + while (redundant_rfs.size() > 0) { + remove_rf(redundant_rfs.pop()->as_ReachabilityFence()); + } + + assert(C->reachability_fences_count() == no_of_constant_rfs, ""); + return progress; +} + +//====================================================================== +//---------------------------- Phase 3 --------------------------------- + +// Find a point in CFG right after safepoint node to insert reachability fence. +static Node* sfpt_ctrl_out(SafePointNode* sfpt) { + if (sfpt->is_Call()) { + CallProjections callprojs; + sfpt->as_Call()->extract_projections(&callprojs, false /*separate_io_proj*/, false /*do_asserts*/); + if (callprojs.fallthrough_catchproj != nullptr) { + return callprojs.fallthrough_catchproj; + } else if (callprojs.catchall_catchproj != nullptr) { + return callprojs.catchall_catchproj; // rethrow stub // TODO: safe to ignore? + } else if (callprojs.fallthrough_proj != nullptr) { + return callprojs.fallthrough_proj; // no exceptions thrown + } else { + ShouldNotReachHere(); + } + } else { + return sfpt; + } +} + +// Phase 3: expand reachability fences from safepoint info. +// Turn extra safepoint edges into reachability fences immediately following the safepoint. +void Compile::expand_reachability_fences(Unique_Node_List& safepoints) { + for (uint i = 0; i < safepoints.size(); i++) { + SafePointNode* sfpt = safepoints.at(i)->as_SafePoint(); + + uint rf_offset = rf_base_offset(sfpt); + if (sfpt->jvms() != nullptr && sfpt->req() > rf_offset) { + assert(is_significant_sfpt(sfpt), ""); + Node* ctrl_out = sfpt_ctrl_out(sfpt); + Node* ctrl_end = ctrl_out->unique_ctrl_out(); + + Node* extra_edge = nullptr; + if (sfpt->is_Call()) { + address entry = sfpt->as_Call()->entry_point(); + if (entry == OptoRuntime::new_array_Java() || + entry == OptoRuntime::new_array_nozero_Java()) { + // valid_length_test_input is appended during macro expansion at the very end + int last_idx = sfpt->req() - 1; + extra_edge = sfpt->in(last_idx); + sfpt->del_req(last_idx); + } + } + + while (sfpt->req() > rf_offset) { + int idx = sfpt->req() - 1; + Node* referent = sfpt->in(idx); + sfpt->del_req(idx); + + Node* new_rf = new ReachabilityFenceNode(C, ctrl_out, referent); + ctrl_end->replace_edge(ctrl_out, new_rf); + ctrl_end = new_rf; + } + + if (extra_edge != nullptr) { + sfpt->add_req(extra_edge); // Add valid_length_test_input edge back + } + } + } +} diff --git a/src/hotspot/share/opto/reachability.hpp b/src/hotspot/share/opto/reachability.hpp new file mode 100644 index 0000000000000..e25458371da31 --- /dev/null +++ b/src/hotspot/share/opto/reachability.hpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#ifndef SHARE_OPTO_REACHABILITY_HPP +#define SHARE_OPTO_REACHABILITY_HPP + +#include "opto/multnode.hpp" +#include "opto/node.hpp" +#include "opto/opcodes.hpp" +#include "opto/type.hpp" + +//------------------------ReachabilityFenceNode-------------------------- +// Represents a Reachability Fence (RF) in the code. +// +// RF ensures that the given object (referent) remains strongly reachable regardless of +// any optimizing transformations the virtual machine may perform that might otherwise +// allow the object to become unreachable. + +// java.lang.ref.Reference::reachabilityFence calls are intrinsified into ReachabilityFence nodes. +// +// More details in reachability.cpp. +class ReachabilityFenceNode : public Node { +public: + ReachabilityFenceNode(Compile* C, Node* ctrl, Node* referent) + : Node(1) { + assert(referent->bottom_type()->isa_oopptr() || + referent->bottom_type()->isa_narrowoop() != nullptr || + referent->bottom_type() == TypePtr::NULL_PTR, + "%s", Type::str(referent->bottom_type())); + init_class_id(Class_ReachabilityFence); + init_req(TypeFunc::Control, ctrl); + add_req(referent); + C->add_reachability_fence(this); + } + virtual int Opcode() const; + virtual bool is_CFG() const { return true; } + virtual uint hash() const { return NO_HASH; } // CFG nodes do not hash + virtual bool depends_only_on_test() const { return false; }; + virtual uint ideal_reg() const { return 0; } // not matched in the AD file + virtual const Type* bottom_type() const { return Type::CONTROL; } + virtual const RegMask& in_RegMask(uint idx) const { + // Fake input register mask for the referent: accepts all registers and all stack slots. + // This avoids redundant register moves around reachability fences. + return RegMask::ALL; + } + virtual const RegMask& out_RegMask() const { + return RegMask::EMPTY; + } + + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual Node* Identity(PhaseGVN* phase); + + bool clear_referent(PhaseIterGVN& phase); + +#ifndef PRODUCT + virtual void format(PhaseRegAlloc* ra, outputStream* st) const; + virtual void emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra) const; +#endif +}; + +#endif // SHARE_OPTO_REACHABILITY_HPP diff --git a/src/java.base/share/classes/java/lang/ref/Reference.java b/src/java.base/share/classes/java/lang/ref/Reference.java index 43d9f414b3c28..9eb956cf07b5f 100644 --- a/src/java.base/share/classes/java/lang/ref/Reference.java +++ b/src/java.base/share/classes/java/lang/ref/Reference.java @@ -650,12 +650,10 @@ protected Object clone() throws CloneNotSupportedException { * {@code null}, this method has no effect. * @since 9 */ - @ForceInline + @IntrinsicCandidate public static void reachabilityFence(Object ref) { - // Does nothing. This method is annotated with @ForceInline to eliminate - // most of the overhead that using @DontInline would cause with the - // HotSpot JVM, when this fence is used in a wide variety of situations. - // HotSpot JVM retains the ref and does not GC it before a call to - // this method, because the JIT-compilers do not have GC-only safepoints. + // Does nothing. HotSpot JVM retains the ref and does not GC it before a call to this method. + // NB! Some of the optimizations JIT-compilers perform may break that invariant, + // so the method is intrinsified when the invariant doesn't hold. } } diff --git a/test/hotspot/jtreg/compiler/c2/TestReachabilityFence.java b/test/hotspot/jtreg/compiler/c2/TestReachabilityFence.java new file mode 100644 index 0000000000000..af7719b7f76e8 --- /dev/null +++ b/test/hotspot/jtreg/compiler/c2/TestReachabilityFence.java @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package compiler.c2; + +import java.lang.ref.Cleaner; +import java.lang.ref.Reference; +import java.util.concurrent.CountDownLatch; + +import jdk.internal.misc.Unsafe; + +/* + * @test + * @bug 8290892 + * @summary Tests to ensure that reachabilityFence() correctly keeps objects from being collected prematurely. + * @modules java.base/jdk.internal.misc + * @run main/othervm -Xbatch compiler.c2.TestReachabilityFence + */ +public class TestReachabilityFence { + private static final int SIZE = 100; + + static final boolean[] STATUS = new boolean[2]; + + interface MyBuffer { + byte get(int offset); + } + static class MyBufferOnHeap implements MyBuffer { + + private static int current = 0; + private final static byte[][] payload = new byte[10][]; + + private final int id; + + public MyBufferOnHeap() { + // Get a unique id, allocate memory and save the address in the payload array + id = current++; + payload[id] = new byte[SIZE]; + + // Initialize buffer + for (int i = 0; i < SIZE; ++i) { + put(i, (byte) 42); + } + + // Register a cleaner to free the memory when the buffer is garbage collected + int lid = id; // Capture current value + Cleaner.create().register(this, () -> { free(lid); }); + + System.out.println("Created new buffer of size = " + SIZE + " with id = " + id); + } + + private static void free(int id) { + System.out.println("Freeing buffer with id = " + id); + for (int i = 0; i < SIZE; ++i) { + payload[id][i] = (byte)0; + } + payload[id] = null; + + synchronized (STATUS) { + STATUS[0] = true; + STATUS.notifyAll(); + } + } + + public void put(int offset, byte b) { + payload[id][offset] = b; + } + + public byte get(int offset) { + try { + return payload[id][offset]; + } finally { + Reference.reachabilityFence(this); + } + } + } + + static class MyBufferOffHeap implements MyBuffer { + private static Unsafe UNSAFE = Unsafe.getUnsafe(); + + private static int current = 0; + private static long payload[] = new long[10]; + + private final int id; + + public MyBufferOffHeap() { + // Get a unique id, allocate memory and save the address in the payload array + id = current++; + payload[id] = UNSAFE.allocateMemory(SIZE); + + // Initialize buffer + for (int i = 0; i < SIZE; ++i) { + put(i, (byte) 42); + } + + // Register a cleaner to free the memory when the buffer is garbage collected + int lid = id; // Capture current value + Cleaner.create().register(this, () -> { free(lid); }); + + System.out.println("Created new buffer of size = " + SIZE + " with id = " + id); + } + + private static void free(int id) { + System.out.println("Freeing buffer with id = " + id); + for (int i = 0; i < SIZE; ++i) { + UNSAFE.putByte(payload[id] + i, (byte)0); + } + // UNSAFE.freeMemory(payload[id]); // don't deallocate backing memory to avoid crashes + payload[id] = 0; + + synchronized (STATUS) { + STATUS[1] = true; + STATUS.notifyAll(); + } + } + + public void put(int offset, byte b) { + UNSAFE.putByte(payload[id] + offset, b); + } + + public byte get(int offset) { + try { + return UNSAFE.getByte(payload[id] + offset); + } finally { + Reference.reachabilityFence(this); + } + } + } + + static MyBuffer bufferOff = new MyBufferOffHeap(); + static MyBuffer bufferOn = new MyBufferOnHeap(); + + static long counter1 = 0; + static long counter2 = 0; + + // Off-heap version. + static long testOffHeap(int limit) { + for (long j = 0; j < limit; j++) { + MyBuffer myBuffer = bufferOff; // local + if (myBuffer == null) { + return j; + } + for (int i = 0; i < SIZE; i++) { + // The access is split into base address load (payload[id]), offset computation, and data load. + // While offset is loop-variant, payload[id] is not and can be hoisted. + // If bufferOff and payload[id] loads are hoisted outside outermost loop, it eliminates all usages of + // myBuffer oop inside the loop and bufferOff can be GCed at the safepoint on outermost loop back branch. + byte b = myBuffer.get(i); // inlined + if (b != 42) { + String msg = "Unexpected value = " + b + ". Buffer was garbage collected before reachabilityFence was reached!"; + throw new AssertionError(msg); + } + } + counter1 = j; + // Keep the buffer live while we read from it + Reference.reachabilityFence(myBuffer); + } // safepoint on loop backedge does NOT contain myBuffer local as part of its JVM state + return limit; + } + + // On-heap version. + static long testOnHeap(int limit) { + for (long j = 0; j < limit; j++) { + MyBuffer myBuffer = bufferOn; + if (myBuffer == null) { + return j; + } + for (int i = 0; i < SIZE; i++) { + byte b = myBuffer.get(i); + if (b != 42) { + String msg = "Unexpected value = " + b + ". Buffer was garbage collected before reachabilityFence was reached!"; + throw new AssertionError(msg); + } + } + counter2 = j; + // Keep the buffer live while we read from it + Reference.reachabilityFence(myBuffer); + } + return limit; + } + + public static void main(String[] args) throws Throwable { + // Warmup to trigger compilation + for (int i = 0; i < 10_000; i++) { + testOffHeap(10); + testOnHeap(10); + } + + CountDownLatch latch = new CountDownLatch(3); + final Throwable[] result = new Throwable[2]; + + Thread compThread1 = new Thread(() -> { + latch.countDown(); // synchronize with main thread + try { + System.out.printf("Computation thread #1 has started\n"); + long cnt = testOffHeap(100_000_000); + System.out.printf("#1 Finished after %d iterations\n", cnt); + } catch (Throwable e) { + System.out.printf("#1 Finished with an exception %s\n", e); + result[0] = e; + } + }); + + Thread compThread2 = new Thread(() -> { + latch.countDown(); // synchronize with main thread + try { + System.out.printf("Computation thread #2 has started\n"); + long cnt = testOnHeap(100_000_000); + System.out.printf("#2 Finished after %d iterations\n", cnt); + } catch (Throwable e) { + System.out.printf("#2 Finished with an exception %s\n", e); + result[1] = e; + } + }); + + compThread1.start(); + compThread2.start(); + + latch.countDown(); // synchronize with comp thread + + Thread.sleep(100); // let compThread proceed + + // Clear reference to 'buffer' and make sure it's garbage collected + System.out.printf("Buffer set to null. Waiting for garbage collection. (counter1 = %d; counter2 = %d)\n", + counter1, counter2); + bufferOn = null; + bufferOff = null; + + System.gc(); + + synchronized (STATUS) { + do { + if (STATUS[0] && STATUS[1] ) { + break; + } else { + System.out.printf("Waiting for cleanup... (counter1 = %d; counter2 = %d)\n", counter1, counter2); + System.gc(); + STATUS.wait(100); + } + } while (true); + } + + compThread1.join(); + compThread2.join(); + if (result[0] != null) { + System.out.println("TEST FAILED"); + throw result[0]; + } + if (result[1] != null) { + System.out.println("TEST FAILED"); + throw result[0]; + } + + System.out.println("TEST PASSED"); + } +} diff --git a/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceFlags.java b/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceFlags.java new file mode 100644 index 0000000000000..502db9e520c2b --- /dev/null +++ b/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceFlags.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package compiler.c2; + +/* + * @test + * @bug 8290892 + * @summary Test diagnostic modes for Reference.reachabilityFence support + * + * @requires vm.compiler2.enabled + * + * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:+UnlockDiagnosticVMOptions + * -XX:+StressReachabilityFences -XX:-OptimizeReachabilityFences -XX:-PreserveReachabilityFencesOnConstants + * compiler.c2.TestReachabilityFenceFlags + * + * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:+UnlockDiagnosticVMOptions + * -XX:+StressReachabilityFences -XX:-OptimizeReachabilityFences -XX:+PreserveReachabilityFencesOnConstants + * compiler.c2.TestReachabilityFenceFlags + * + * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:+UnlockDiagnosticVMOptions + * -XX:+StressReachabilityFences -XX:+OptimizeReachabilityFences -XX:-PreserveReachabilityFencesOnConstants + * compiler.c2.TestReachabilityFenceFlags + * + * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:+UnlockDiagnosticVMOptions + * -XX:+StressReachabilityFences -XX:+OptimizeReachabilityFences -XX:+PreserveReachabilityFencesOnConstants + * compiler.c2.TestReachabilityFenceFlags + */ +public class TestReachabilityFenceFlags { + public static void main(String[] args) throws Throwable { + } +} diff --git a/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceOnConstant.java b/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceOnConstant.java new file mode 100644 index 0000000000000..8b4b1eead225f --- /dev/null +++ b/test/hotspot/jtreg/compiler/c2/TestReachabilityFenceOnConstant.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package compiler.c2; + +import jdk.internal.misc.Unsafe; +import jdk.internal.vm.annotation.Stable; + +import java.lang.ref.Cleaner; +import java.lang.ref.Reference; + +/* + * @test + * @bug 8290892 + * @summary Tests to ensure that reachabilityFence() correctly keeps objects from being collected prematurely. + * @modules java.base/jdk.internal.misc + * java.base/jdk.internal.vm.annotation + * @run main/bootclasspath/othervm -Xbatch -XX:-TieredCompilation -XX:CompileCommand=quiet + * -XX:CompileCommand=compileonly,*::test + * -XX:+UnlockDiagnosticVMOptions -XX:+PreserveReachabilityFencesOnConstants + * compiler.c2.TestReachabilityFenceOnConstant + */ +public class TestReachabilityFenceOnConstant { + static final Unsafe U = Unsafe.getUnsafe(); + + static final long BUFFER_SIZE = 1024; + static @Stable MyBuffer BUFFER = new MyBuffer(); + + static volatile boolean isCleaned = false; + + static class MyBuffer { + final @Stable long address; + final @Stable long limit; + + MyBuffer() { + final long adr = U.allocateMemory(BUFFER_SIZE); + U.setMemory(adr, BUFFER_SIZE, (byte)0); + address = adr; + limit = BUFFER_SIZE; + System.out.printf("Allocated memory (%d bytes): 0x%016x\n", BUFFER_SIZE, adr); + Cleaner.create().register(this, () -> { + System.out.printf("Freed memory (%d bytes): 0x%016x\n", BUFFER_SIZE, adr); + U.setMemory(adr, BUFFER_SIZE, (byte)-1); // clear + U.freeMemory(adr); + isCleaned = true; + }); + } + + byte getByte(long offset) { + return U.getByte(null, address + offset); + } + } + + static int test() { + int acc = 0; + MyBuffer buf = BUFFER; + try { + for (long i = 0; i < buf.limit; i++) { + acc += buf.getByte(i); + } + } finally { + Reference.reachabilityFence(buf); + } + return acc; + } + + static void runTest() { + for (int i = 0; i < 20_000; i++) { + if (test() != 0) { + throw new AssertionError("observed stale buffer: TestConstantOop::isCleaned=" + isCleaned); + } + } + } + + public static void main(String[] args) throws NoSuchFieldException, IllegalAccessException { + runTest(); // run test() and constant fold accesses to BUFFER (and it's state) during JIT-compilation + + BUFFER = null; // remove last strong root + + // Ensure the instance is GCed. + while (!isCleaned) { + try { + System.gc(); + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + try { + runTest(); // repeat to ensure stale BUFFER contents is not accessed + } catch (NullPointerException e) { + // expected; ignore + } + System.out.println("TEST PASSED"); + } +} diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/CompilePhase.java b/test/hotspot/jtreg/compiler/lib/ir_framework/CompilePhase.java index 06b2afa8a67d0..d1ae10077560c 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/CompilePhase.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/CompilePhase.java @@ -97,6 +97,7 @@ public enum CompilePhase { PHASEIDEALLOOP1( "PhaseIdealLoop 1"), PHASEIDEALLOOP2( "PhaseIdealLoop 2"), PHASEIDEALLOOP3( "PhaseIdealLoop 3"), + ELIMINATE_REACHABILITY_FENCES( "Eliminate Reachability Fences"), AUTO_VECTORIZATION1_BEFORE_APPLY( "AutoVectorization 1, before Apply"), AUTO_VECTORIZATION3_AFTER_ADJUST_LIMIT( "AutoVectorization 2, after Adjusting Pre-loop Limit"), AUTO_VECTORIZATION4_AFTER_SPECULATIVE_RUNTIME_CHECKS("AutoVectorization 3, after Adding Speculative Runtime Checks"), diff --git a/test/hotspot/jtreg/compiler/loopopts/UseCountedLoopSafepointsTest.java b/test/hotspot/jtreg/compiler/loopopts/UseCountedLoopSafepointsTest.java index 2a9cedfc87ee6..c64bb7515a61c 100644 --- a/test/hotspot/jtreg/compiler/loopopts/UseCountedLoopSafepointsTest.java +++ b/test/hotspot/jtreg/compiler/loopopts/UseCountedLoopSafepointsTest.java @@ -73,6 +73,8 @@ private static void check(boolean enabled) { // parse output in seach of SafePoint and CountedLoopEnd nodes List safePoints = new ArrayList<>(); List loopEnds = new ArrayList<>(); + List reachabilityFences = new ArrayList<>(); + List projections = new ArrayList<>(); for (String line : oa.getOutput().split("\\n")) { int separatorIndex = line.indexOf(" ==="); if (separatorIndex > -1) { @@ -81,17 +83,28 @@ private static void check(boolean enabled) { safePoints.add(new Node("SafePoint", line)); } else if (header.endsWith("CountedLoopEnd")) { loopEnds.add(new Node("CountedLoopEnd", line)); + } else if (header.endsWith("ReachabilityFence")) { + reachabilityFences.add(new Node("ReachabilityFence", line)); + } else if (header.endsWith("Proj")) { + projections.add(new Node("Proj", line)); } } } // now, find CountedLoopEnd -> SafePoint edge boolean found = false; - for (Node loopEnd : loopEnds) { - found |= loopEnd.to.stream() - .filter(id -> nodeListHasElementWithId(safePoints, id)) - .findAny() - .isPresent(); - } + + // CountedLoopEnd -> SafePoint + found |= loopEnds.stream() + .flatMap(n -> n.to.stream()).flatMap(id -> safePoints.stream().filter(node -> node.id == id)) + .findAny().isPresent(); + + // CountedLoopEnd -> Proj -> ReachabilityFence -> SafePoint + found |= loopEnds.stream() + .flatMap(n -> n.to.stream()).flatMap(id -> projections.stream().filter(node -> node.id == id)) + .flatMap(n -> n.to.stream()).flatMap(id -> reachabilityFences.stream().filter(node -> node.id == id)) + .flatMap(n -> n.to.stream()).flatMap(id -> safePoints.stream().filter(node -> node.id == id)) + .findAny().isPresent(); + Asserts.assertEQ(enabled, found, "Safepoint " + (found ? "" : "not ") + "found"); }