diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index 99041df6e3b..11a0719ea8b 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -180,6 +180,7 @@ jobs: - name: Configure run: > bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=linux-x64 ${{ matrix.flags }} --with-version-opt=${GITHUB_ACTOR}-${GITHUB_SHA} @@ -545,6 +546,7 @@ jobs: - name: Configure run: > bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=linux-${{ matrix.gnu-arch }}-hotspot ${{ matrix.flags }} ${{ env.cross_flags }} @@ -644,6 +646,7 @@ jobs: - name: Configure run: > bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=linux-x86 --with-target-bits=32 ${{ matrix.flags }} @@ -920,6 +923,7 @@ jobs: $env:Path = $env:Path -split ";" -match "C:\\Windows|PowerShell|cygwin" -join ";" ; $env:BOOT_JDK = cygpath "$HOME/bootjdk/$env:BOOT_JDK_VERSION" ; & bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=windows-aarch64 --with-msvc-toolset-version=14.29 --openjdk-target=aarch64-unknown-cygwin @@ -1033,6 +1037,7 @@ jobs: $env:JT_HOME = cygpath "$HOME/jtreg" ; $env:GTEST = cygpath "$env:GITHUB_WORKSPACE/gtest" ; & bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=windows-x64 --with-msvc-toolset-version=14.28 ${{ matrix.flags }} @@ -1340,6 +1345,7 @@ jobs: - name: Configure run: > bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=macos-x64 ${{ matrix.flags }} --with-version-opt=${GITHUB_ACTOR}-${GITHUB_SHA} @@ -1442,6 +1448,7 @@ jobs: - name: Configure run: > bash configure + --with-jvm-features=-shenandoahgc,-zgc --with-conf-name=macos-aarch64 --openjdk-target=aarch64-apple-darwin ${{ matrix.flags }} diff --git a/src/hotspot/share/gc/g1/g1EvacFailure.cpp b/src/hotspot/share/gc/g1/g1EvacFailure.cpp index 07a4603ade0..d90f2505702 100644 --- a/src/hotspot/share/gc/g1/g1EvacFailure.cpp +++ b/src/hotspot/share/gc/g1/g1EvacFailure.cpp @@ -116,6 +116,9 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure { // The object failed to move. zap_dead_objects(_last_forwarded_object_end, obj_addr); + + PreservedMarks::init_forwarded_mark(obj); + // We consider all objects that we find self-forwarded to be // live. What we'll do is that we'll update the prev marking // info so that they are all under PTAMS and explicitly marked. @@ -138,7 +141,6 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure { size_t obj_size = obj->size(); _marked_bytes += (obj_size * HeapWordSize); - PreservedMarks::init_forwarded_mark(obj); // While we were processing RSet buffers during the collection, // we actually didn't scan any cards on the collection set, @@ -158,6 +160,14 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure { _last_forwarded_object_end = obj_end; _hr->cross_threshold(obj_addr, obj_end); } +#ifdef _LP64 + else if (obj->is_forwarded()) { + // Restore klass so that we can safely iterate. + // TODO: This could probably be built more efficiently into the iterator. + Klass* klass = obj->forwardee()->klass(); + obj->set_mark(markWord::prototype().set_klass(klass)); + } +#endif } // Fill the memory area from start to end with filler objects, and update the BOT diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 5143685446b..52d756ed21f 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -218,7 +218,6 @@ void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) { oop from_obj = task.to_source_array(); assert(_g1h->is_in_reserved(from_obj), "must be in heap."); - assert(from_obj->is_objArray(), "must be obj array"); assert(from_obj->is_forwarded(), "must be forwarded"); oop to_obj = from_obj->forwardee(); @@ -248,7 +247,6 @@ MAYBE_INLINE_EVACUATION void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr, oop from_obj, oop to_obj) { - assert(from_obj->is_objArray(), "precondition"); assert(from_obj->is_forwarded(), "precondition"); assert(from_obj->forwardee() == to_obj, "precondition"); assert(from_obj != to_obj, "should not be scanning self-forwarded objects"); @@ -428,9 +426,17 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio assert(region_attr.is_in_cset(), "Unexpected region attr type: %s", region_attr.get_type_str()); + if (old_mark.is_marked()) { + // Already forwarded by somebody else, return forwardee. + return old->forwardee(old_mark); + } // Get the klass once. We'll need it again later, and this avoids // re-decoding when it's compressed. +#ifdef _LP64 + Klass* klass = old_mark.safe_klass(); +#else Klass* klass = old->klass(); +#endif const size_t word_sz = old->size_given_klass(klass); uint age = 0; diff --git a/src/hotspot/share/gc/parallel/psPromotionLAB.cpp b/src/hotspot/share/gc/parallel/psPromotionLAB.cpp index 53fac0e98a5..0b3b6fc04a8 100644 --- a/src/hotspot/share/gc/parallel/psPromotionLAB.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionLAB.cpp @@ -83,7 +83,7 @@ void PSPromotionLAB::flush() { // so they can always fill with an array. HeapWord* tlab_end = end() + filler_header_size; typeArrayOop filler_oop = (typeArrayOop) cast_to_oop(top()); - filler_oop->set_mark(markWord::prototype()); + filler_oop->set_mark(Universe::intArrayKlassObj()->prototype_header()); filler_oop->set_klass(Universe::intArrayKlassObj()); const size_t array_length = pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 123e17e4d48..c9ea1256db1 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -314,7 +314,6 @@ void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) { assert(PSChunkLargeArrays, "invariant"); oop old = task.to_source_array(); - assert(old->is_objArray(), "invariant"); assert(old->is_forwarded(), "invariant"); TASKQUEUE_STATS_ONLY(++_array_chunks_processed); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index eab6f891d65..59899374742 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -160,7 +160,12 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, oop new_obj = NULL; bool new_obj_is_tenured = false; - size_t new_obj_size = o->size(); +#ifdef _LP64 + Klass* klass = test_mark.safe_klass(); +#else + Klass* klass = o->klass(); +#endif + size_t new_obj_size = o->size_given_klass(klass); // Find the objects age, MT safe. uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ? diff --git a/src/hotspot/share/gc/serial/markSweep.inline.hpp b/src/hotspot/share/gc/serial/markSweep.inline.hpp index 4d05f81b29b..be9eb6fc166 100644 --- a/src/hotspot/share/gc/serial/markSweep.inline.hpp +++ b/src/hotspot/share/gc/serial/markSweep.inline.hpp @@ -41,7 +41,7 @@ inline void MarkSweep::mark_object(oop obj) { // some marks may contain information we need to preserve so we store them away // and overwrite the mark. We'll restore it at the end of markSweep. markWord mark = obj->mark(); - obj->set_mark(markWord::prototype().set_marked()); + obj->set_mark(obj->klass()->prototype_header().set_marked()); if (obj->mark_must_be_preserved(mark)) { preserve_mark(obj, mark); diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp index 96b3eb94681..ca2f697f93a 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.cpp +++ b/src/hotspot/share/gc/shared/collectedHeap.cpp @@ -208,10 +208,6 @@ bool CollectedHeap::is_oop(oop object) const { return false; } - if (is_in(object->klass_or_null())) { - return false; - } - return true; } diff --git a/src/hotspot/share/gc/shared/preservedMarks.inline.hpp b/src/hotspot/share/gc/shared/preservedMarks.inline.hpp index 44bb26f87e6..71f8883e747 100644 --- a/src/hotspot/share/gc/shared/preservedMarks.inline.hpp +++ b/src/hotspot/share/gc/shared/preservedMarks.inline.hpp @@ -48,7 +48,20 @@ inline void PreservedMarks::push_if_necessary(oop obj, markWord m) { } inline void PreservedMarks::init_forwarded_mark(oop obj) { - obj->init_mark(); + assert(obj->is_forwarded(), "only forwarded here"); +#ifdef _LP64 + oop forwardee = obj->forwardee(); + markWord header = forwardee->mark(); + if (header.has_displaced_mark_helper()) { + header = header.displaced_mark_helper(); + } + assert(UseCompressedClassPointers, "assume +UseCompressedClassPointers"); + narrowKlass nklass = header.narrow_klass(); + assert(nklass == obj->narrow_klass_legacy(), "narrow klass must match: header: " PTR_FORMAT ", nklass: " PTR_FORMAT, forwardee->mark().value(), uintptr_t(nklass)); + obj->set_mark(markWord::prototype().set_narrow_klass(nklass)); +#else + obj->set_mark(markWord::prototype()); +#endif } inline PreservedMarks::PreservedMarks() diff --git a/src/hotspot/share/gc/shared/slidingForwarding.inline.hpp b/src/hotspot/share/gc/shared/slidingForwarding.inline.hpp index aed241ebf51..c848d592d6f 100644 --- a/src/hotspot/share/gc/shared/slidingForwarding.inline.hpp +++ b/src/hotspot/share/gc/shared/slidingForwarding.inline.hpp @@ -87,6 +87,9 @@ HeapWord* SlidingForwarding::decode_forwarding(HeapWord* original, uintptr_t enc void SlidingForwarding::forward_to(oop original, oop target) { #ifdef _LP64 markWord header = original->mark(); + if (header.has_displaced_mark_helper()) { + header = header.displaced_mark_helper(); + } uintptr_t encoded = encode_forwarding(cast_from_oop(original), cast_from_oop(target)); assert((encoded & markWord::klass_mask_in_place) == 0, "encoded forwardee must not overlap with Klass*: " PTR_FORMAT, encoded); header = markWord((header.value() & markWord::klass_mask_in_place) | encoded); diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index 4156559df05..d87013abcdd 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -126,7 +126,7 @@ class InstanceRefKlass: public InstanceKlass { static void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains); template - static void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; + void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; public: // Update non-static oop maps so 'referent', 'nextPending' and diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index 4b729e6dd13..616c115e6b1 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -185,7 +185,7 @@ void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) { T* discovered_addr = (T*) java_lang_ref_Reference::discovered_addr_raw(obj); log_develop_trace(gc, ref)("InstanceRefKlass %s for obj " PTR_FORMAT, s, p2i(obj)); - if (java_lang_ref_Reference::is_phantom(obj)) { + if (reference_type() == REF_PHANTOM) { log_develop_trace(gc, ref)(" referent_addr/* " PTR_FORMAT " / " PTR_FORMAT, p2i(referent_addr), p2i((oop)HeapAccess::oop_load(referent_addr))); } else { diff --git a/src/hotspot/share/oops/markWord.cpp b/src/hotspot/share/oops/markWord.cpp index 931712c8d5c..4536c4c5327 100644 --- a/src/hotspot/share/oops/markWord.cpp +++ b/src/hotspot/share/oops/markWord.cpp @@ -94,24 +94,3 @@ void markWord::print_on(outputStream* st, bool print_monitor_info) const { st->print(" age=%d)", age()); } } - -#ifdef _LP64 -narrowKlass markWord::narrow_klass() const { - return narrowKlass(value() >> klass_shift); -} - -Klass* markWord::klass() const { - return CompressedKlassPointers::decode(narrow_klass()); -} - -markWord markWord::set_narrow_klass(const narrowKlass nklass) const { - return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) nklass << klass_shift)); -} - -markWord markWord::set_klass(const Klass* klass) const { - assert(UseCompressedClassPointers, "expect compressed klass pointers"); - // TODO: Don't cast to non-const, change CKP::encode() to accept const Klass* instead. - narrowKlass nklass = CompressedKlassPointers::encode(const_cast(klass)); - return set_narrow_klass(nklass); -} -#endif diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index db5166b7b7a..5ab87ce97d0 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -248,10 +248,12 @@ class markWord { } #ifdef _LP64 - narrowKlass narrow_klass() const; - Klass* klass() const; - markWord set_klass(const Klass* klass) const; - markWord set_narrow_klass(const narrowKlass klass) const; + inline Klass* klass() const; + inline Klass* klass_or_null() const; + inline Klass* safe_klass() const; + inline markWord set_klass(const Klass* klass) const; + inline narrowKlass narrow_klass() const; + inline markWord set_narrow_klass(const narrowKlass klass) const; #endif // Prototype mark for initialization diff --git a/src/hotspot/share/oops/markWord.inline.hpp b/src/hotspot/share/oops/markWord.inline.hpp new file mode 100644 index 00000000000..af185f6d05a --- /dev/null +++ b/src/hotspot/share/oops/markWord.inline.hpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_MARKWORD_INLINE_HPP +#define SHARE_OOPS_MARKWORD_INLINE_HPP + +#include "oops/compressedOops.inline.hpp" +#include "oops/markWord.hpp" +#include "runtime/safepoint.hpp" + +#ifdef _LP64 +narrowKlass markWord::narrow_klass() const { + return narrowKlass(value() >> klass_shift); +} + +Klass* markWord::klass() const { + return CompressedKlassPointers::decode_not_null(narrow_klass()); +} + +Klass* markWord::klass_or_null() const { + return CompressedKlassPointers::decode(narrow_klass()); +} + +markWord markWord::set_narrow_klass(const narrowKlass nklass) const { + return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) nklass << klass_shift)); +} + +Klass* markWord::safe_klass() const { + assert(SafepointSynchronize::is_at_safepoint(), "only call at safepoint"); + markWord m = *this; + if (m.has_displaced_mark_helper()) { + m = m.displaced_mark_helper(); + } + return CompressedKlassPointers::decode_not_null(m.narrow_klass()); +} + +markWord markWord::set_klass(const Klass* klass) const { + assert(UseCompressedClassPointers, "expect compressed klass pointers"); + // TODO: Don't cast to non-const, change CKP::encode() to accept const Klass* instead. + narrowKlass nklass = CompressedKlassPointers::encode(const_cast(klass)); + return set_narrow_klass(nklass); +} +#endif + +#endif // SHARE_OOPS_MARKWORD_INLINE_HPP diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index da75ec35ecb..324bdb38fe7 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -156,7 +156,6 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK } int ObjArrayKlass::oop_size(oop obj) const { - assert(obj->is_objArray(), "must be object array"); return objArrayOop(obj)->object_size(); } diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 443a3eac09f..bd2c57b6bd4 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -70,7 +70,6 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded( template void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - assert (obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); if (Devirtualizer::do_metadata(closure)) { diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 54485fd1957..16cf56d97ba 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -77,6 +77,7 @@ class oopDesc { inline Klass* klass_or_null() const; inline Klass* klass_or_null_acquire() const; + narrowKlass narrow_klass_legacy() const { return _metadata._compressed_klass; } void set_narrow_klass(narrowKlass nk) NOT_CDS_JAVA_HEAP_RETURN; inline void set_klass(Klass* k); static inline void release_set_klass(HeapWord* mem, Klass* k); diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index dd591f3a7af..2a32ef729fa 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -32,10 +32,12 @@ #include "oops/arrayKlass.hpp" #include "oops/arrayOop.hpp" #include "oops/compressedOops.inline.hpp" -#include "oops/markWord.hpp" +#include "oops/markWord.inline.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/synchronizer.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/macros.hpp" @@ -80,39 +82,63 @@ markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark, atomic_memo } void oopDesc::init_mark() { - markWord header = markWord::prototype(); #ifdef _LP64 + markWord header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); + assert(_metadata._compressed_klass == header.narrow_klass(), "klass must match: " PTR_FORMAT, header.value()); assert(UseCompressedClassPointers, "expect compressed klass pointers"); - narrowKlass nklass = _metadata._compressed_klass; - assert(nklass != 0, "expect klass"); - header = header.set_narrow_klass(nklass); + header = markWord((header.value() & markWord::klass_mask_in_place) | markWord::prototype().value()); +#else + markWord header = markWord::prototype(); #endif set_mark(header); } Klass* oopDesc::klass() const { - if (UseCompressedClassPointers) { - return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass); - } else { - return _metadata._klass; +#ifdef _LP64 + assert(UseCompressedClassPointers, "only with compressed class pointers"); + markWord header = mark(); + if (!header.is_neutral()) { + header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); } + assert(_metadata._compressed_klass == header.narrow_klass(), "narrow klass must be equal, header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + Klass* klass = header.klass(); + assert(klass == CompressedKlassPointers::decode_not_null(_metadata._compressed_klass), "klass must match: header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + return klass; +#else + return _metadata._klass; +#endif } Klass* oopDesc::klass_or_null() const { - if (UseCompressedClassPointers) { - return CompressedKlassPointers::decode(_metadata._compressed_klass); - } else { - return _metadata._klass; +#ifdef _LP64 + assert(UseCompressedClassPointers, "only with compressed class pointers"); + markWord header = mark(); + if (!header.is_neutral()) { + header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); } + assert(_metadata._compressed_klass == header.narrow_klass(), "narrow klass must be equal, header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + Klass* klass = header.klass_or_null(); + assert(klass == CompressedKlassPointers::decode(_metadata._compressed_klass), "klass must match: header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + return klass; +#else + return _metadata._klass; +#endif } Klass* oopDesc::klass_or_null_acquire() const { - if (UseCompressedClassPointers) { - narrowKlass nklass = Atomic::load_acquire(&_metadata._compressed_klass); - return CompressedKlassPointers::decode(nklass); - } else { - return Atomic::load_acquire(&_metadata._klass); +#ifdef _LP64 + assert(UseCompressedClassPointers, "only with compressed class pointers"); + markWord header = mark_acquire(); + if (!header.is_neutral()) { + header = ObjectSynchronizer::stable_mark(cast_to_oop(this)); } + assert(_metadata._compressed_klass == header.narrow_klass(), "narrow klass must be equal, header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + Klass* klass = header.klass_or_null(); + assert(klass == CompressedKlassPointers::decode(_metadata._compressed_klass), "klass must match: header: " INTPTR_FORMAT ", nklass: " INTPTR_FORMAT, header.value(), intptr_t(_metadata._compressed_klass)); + return klass; +#else + return Atomic::load_acquire(&_metadata._klass); +#endif } void oopDesc::set_klass(Klass* k) { @@ -288,7 +314,13 @@ void oopDesc::forward_to(oop p) { void oopDesc::forward_to_self() { verify_forwardee(this); - markWord m = mark().set_self_forwarded(); + markWord m = mark(); + // If mark is displaced, we need to preserve the Klass* from real header. + assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint"); + if (m.has_displaced_mark_helper()) { + m = m.displaced_mark_helper(); + } + m = m.set_self_forwarded(); assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable"); set_mark(m); } @@ -307,7 +339,13 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) { verify_forwardee(this); - markWord m = compare.set_self_forwarded(); + markWord m = compare; + // If mark is displaced, we need to preserve the Klass* from real header. + assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint"); + if (m.has_displaced_mark_helper()) { + m = m.displaced_mark_helper(); + } + m = m.set_self_forwarded(); assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable"); markWord old_mark = cas_set_mark(m, compare, order); if (old_mark == compare) { @@ -386,7 +424,6 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl) { template void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { - assert(k == klass(), "wrong klass"); OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k); } diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index d15afe8537d..3eabe9ff251 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -228,7 +228,6 @@ Klass* TypeArrayKlass::array_klass_or_null() { } int TypeArrayKlass::oop_size(oop obj) const { - assert(obj->is_typeArray(),"must be a type array"); typeArrayOop t = typeArrayOop(obj); return t->object_size(this); } diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index 098f9e73993..31d87e3b85a 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -35,7 +35,6 @@ class OopIterateClosure; inline void TypeArrayKlass::oop_oop_iterate_impl(oop obj, OopIterateClosure* closure) { - assert(obj->is_typeArray(),"must be a type array"); // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } diff --git a/src/hotspot/share/prims/jvmtiTagMap.cpp b/src/hotspot/share/prims/jvmtiTagMap.cpp index 5f9de94fbcf..e5790b81c31 100644 --- a/src/hotspot/share/prims/jvmtiTagMap.cpp +++ b/src/hotspot/share/prims/jvmtiTagMap.cpp @@ -1438,7 +1438,7 @@ inline void ObjectMarker::mark(oop o) { } // mark the object - o->set_mark(markWord::prototype().set_marked()); + o->set_mark(o->klass()->prototype_header().set_marked()); } // return true if object is marked diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index 6fbbd666d08..ed96062996e 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -740,6 +740,29 @@ static markWord read_stable_mark(oop obj) { } } +markWord ObjectSynchronizer::stable_mark(const oop obj) { + markWord mark = read_stable_mark(obj); + if (!mark.is_neutral() && !mark.is_marked()) { + if (mark.has_monitor()) { + ObjectMonitor* monitor = mark.monitor(); + mark = monitor->header(); + assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); + } else if (SafepointSynchronize::is_at_safepoint() || Thread::current()->is_lock_owned((address) mark.locker())) { + // This is a stack lock owned by the calling thread so fetch the + // displaced markWord from the BasicLock on the stack. + assert(mark.has_displaced_mark_helper(), "must be displaced header here"); + mark = mark.displaced_mark_helper(); + assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); + } else { + ObjectMonitor* monitor = inflate(Thread::current(), obj, inflate_cause_vm_internal); + mark = monitor->header(); + assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); + assert(!mark.is_marked(), "no forwarded objects here"); + } + } + return mark; +} + // hashCode() generation : // // Possibilities: diff --git a/src/hotspot/share/runtime/synchronizer.hpp b/src/hotspot/share/runtime/synchronizer.hpp index 89cf4f38de7..6865c8c76ef 100644 --- a/src/hotspot/share/runtime/synchronizer.hpp +++ b/src/hotspot/share/runtime/synchronizer.hpp @@ -126,6 +126,8 @@ class ObjectSynchronizer : AllStatic { static intptr_t identity_hash_value_for(Handle obj); static intptr_t FastHashCode(Thread* current, oop obj); + static markWord stable_mark(const oop obj); + // java.lang.Thread support static bool current_thread_holds_lock(JavaThread* current, Handle h_obj);