diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml
index c73def5bae4..9a15430653c 100644
--- a/.github/workflows/submit.yml
+++ b/.github/workflows/submit.yml
@@ -178,7 +178,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install gcc-10=10.2.0-5ubuntu1~20.04 g++-10=10.2.0-5ubuntu1~20.04 libxrandr-dev libxtst-dev libcups2-dev libasound2-dev
+          sudo apt-get install gcc-10=10.3.0-1ubuntu1~20.04 g++-10=10.3.0-1ubuntu1~20.04 libxrandr-dev libxtst-dev libcups2-dev libasound2-dev
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10
 
       - name: Configure
@@ -493,12 +493,12 @@ jobs:
 
       - name: Install native host dependencies
         run: |
-          sudo apt-get install gcc-10=10.2.0-5ubuntu1~20.04 g++-10=10.2.0-5ubuntu1~20.04 libxrandr-dev libxtst-dev libcups2-dev libasound2-dev
+          sudo apt-get install gcc-10=10.3.0-1ubuntu1~20.04 g++-10=10.3.0-1ubuntu1~20.04 libxrandr-dev libxtst-dev libcups2-dev libasound2-dev
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10
         if: matrix.debian-arch == ''
 
       - name: Install cross-compilation host dependencies
-        run: sudo apt-get install gcc-10-${{ matrix.gnu-arch }}-linux-gnu${{ matrix.gnu-flavor}}=10.2.0-5ubuntu1~20.04cross1 g++-10-${{ matrix.gnu-arch }}-linux-gnu${{ matrix.gnu-flavor}}=10.2.0-5ubuntu1~20.04cross1
+        run: sudo apt-get install gcc-10-${{ matrix.gnu-arch }}-linux-gnu${{ matrix.gnu-flavor}}=10.3.0-1ubuntu1~20.04cross1 g++-10-${{ matrix.gnu-arch }}-linux-gnu${{ matrix.gnu-flavor}}=10.3.0-1ubuntu1~20.04cross1
         if: matrix.debian-arch != ''
 
       - name: Cache sysroot
@@ -916,7 +916,7 @@ jobs:
         run: >
           Start-Process -FilePath 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vs_installer.exe' -Wait -NoNewWindow -ArgumentList
           'modify --installPath "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise" --quiet
-          --add Microsoft.VisualStudio.Component.VC.14.28.arm64'
+          --add Microsoft.VisualStudio.Component.VC.14.29.arm64'
 
       - name: Configure
         run: >
@@ -925,7 +925,7 @@ jobs:
           $env:BOOT_JDK = cygpath "$HOME/bootjdk/$env:BOOT_JDK_VERSION" ;
           & bash configure
           --with-conf-name=windows-aarch64
-          --with-msvc-toolset-version=14.28
+          --with-msvc-toolset-version=14.29
           --openjdk-target=aarch64-unknown-cygwin
           ${{ matrix.flags }}
           --with-version-opt="$env:GITHUB_ACTOR-$env:GITHUB_SHA"
diff --git a/.gitignore b/.gitignore
index cf21c8919cd..6787b232535 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,5 @@ NashornProfile.txt
 **/JTreport/**
 **/JTwork/**
 /src/utils/LogCompilation/target/
+/.project/
+/.settings/
diff --git a/bin/idea.sh b/bin/idea.sh
index 49c6ee45e3b..6359a77b6db 100644
--- a/bin/idea.sh
+++ b/bin/idea.sh
@@ -25,7 +25,7 @@
 # Shell script for generating an IDEA project from a given list of modules
 
 usage() {
-      echo "usage: $0 [-h|--help] [-v|--verbose] [-o|--output <path>] [modules]+"
+      echo "usage: $0 [-h|--help] [-v|--verbose] [-o|--output <path>] [-c|--conf <conf_name>] [modules]+"
       exit 1
 }
 
@@ -37,6 +37,7 @@ cd $TOP;
 
 IDEA_OUTPUT=$TOP/.idea
 VERBOSE="false"
+CONF_ARG=
 while [ $# -gt 0 ]
 do
   case $1 in
@@ -52,6 +53,10 @@ do
       IDEA_OUTPUT=$2/.idea
       shift
       ;;
+    -c | --conf )
+      CONF_ARG="CONF_NAME=$2"
+      shift
+      ;;
 
     -*)  # bad option
       usage
@@ -64,6 +69,9 @@ do
   shift
 done
 
+if [ -e $IDEA_OUTPUT ] ; then
+    rm -r $IDEA_OUTPUT
+fi
 mkdir -p $IDEA_OUTPUT || exit 1
 cd $IDEA_OUTPUT; IDEA_OUTPUT=`pwd`
 
@@ -91,7 +99,7 @@ if [ "$VERBOSE" = "true" ] ; then
   echo "idea template dir: $IDEA_TEMPLATE"
 fi
 
-cd $TOP ; make -f "$IDEA_MAKE/idea.gmk" -I $MAKE_DIR/.. idea MAKEOVERRIDES= OUT=$IDEA_OUTPUT/env.cfg MODULES="$*" || exit 1
+cd $TOP ; make -f "$IDEA_MAKE/idea.gmk" -I $MAKE_DIR/.. idea MAKEOVERRIDES= OUT=$IDEA_OUTPUT/env.cfg MODULES="$*" $CONF_ARG || exit 1
 cd $SCRIPT_DIR
 
 . $IDEA_OUTPUT/env.cfg
@@ -148,14 +156,14 @@ add_replacement "###MODULE_NAMES###" "$MODULE_NAMES"
 add_replacement "###VCS_TYPE###" "$VCS_TYPE"
 SPEC_DIR=`dirname $SPEC`
 if [ "x$CYGPATH" != "x" ]; then
-    add_replacement "###BUILD_DIR###" "`cygpath -am $SPEC_DIR`"
-    add_replacement "###IMAGES_DIR###" "`cygpath -am $SPEC_DIR`/images/jdk"
-    add_replacement "###ROOT_DIR###" "`cygpath -am $TOPLEVEL_DIR`"
-    add_replacement "###IDEA_DIR###" "`cygpath -am $IDEA_OUTPUT`"
+    add_replacement "###BUILD_DIR###" "`$CYGPATH -am $SPEC_DIR`"
+    add_replacement "###IMAGES_DIR###" "`$CYGPATH -am $SPEC_DIR`/images/jdk"
+    add_replacement "###ROOT_DIR###" "`$CYGPATH -am $TOPLEVEL_DIR`"
+    add_replacement "###IDEA_DIR###" "`$CYGPATH -am $IDEA_OUTPUT`"
     if [ "x$JT_HOME" = "x" ]; then
       add_replacement "###JTREG_HOME###" ""
     else
-      add_replacement "###JTREG_HOME###" "`cygpath -am $JT_HOME`"
+      add_replacement "###JTREG_HOME###" "`$CYGPATH -am $JT_HOME`"
     fi
 elif [ "x$WSL_DISTRO_NAME" != "x" ]; then
     add_replacement "###BUILD_DIR###" "`wslpath -am $SPEC_DIR`"
@@ -180,7 +188,7 @@ SOURCE_POSTFIX="\" isTestSource=\"false\" />"
 
 for root in $MODULE_ROOTS; do
     if [ "x$CYGPATH" != "x" ]; then
-      root=`cygpath -am $root`
+      root=`$CYGPATH -am $root`
     elif [ "x$WSL_DISTRO_NAME" != "x" ]; then
       root=`wslpath -am $root`
     fi
@@ -219,26 +227,33 @@ fi
 CP=$ANT_HOME/lib/ant.jar
 rm -rf $CLASSES; mkdir $CLASSES
 
-if [ "x$CYGPATH" != "x" ] ; then ## CYGPATH may be set in env.cfg
-  JAVAC_SOURCE_FILE=`cygpath -am $IDEA_OUTPUT/src/idea/IdeaLoggerWrapper.java`
-  JAVAC_SOURCE_PATH=`cygpath -am $IDEA_OUTPUT/src`
-  JAVAC_CLASSES=`cygpath -am $CLASSES`
-  JAVAC_CP=`cygpath -am $CP`
+# If we have a Windows boot JDK, we need a .exe suffix
+if [ -e "$BOOT_JDK/bin/java.exe" ] ; then
+  JAVAC=javac.exe
+else 
   JAVAC=javac
-elif [ "x$WSL_DISTRO_NAME" != "x" ]; then
+fi
+
+# If we are on WSL, the boot JDK might be either Windows or Linux,
+# and we need to use realpath instead of CYGPATH to make javac work on both.
+# We need to handle this case first since CYGPATH might be set on WSL.
+if [ "x$WSL_DISTRO_NAME" != "x" ]; then
   JAVAC_SOURCE_FILE=`realpath --relative-to=./ $IDEA_OUTPUT/src/idea/IdeaLoggerWrapper.java`
   JAVAC_SOURCE_PATH=`realpath --relative-to=./ $IDEA_OUTPUT/src`
   JAVAC_CLASSES=`realpath --relative-to=./ $CLASSES`
   ANT_TEMP=`mktemp -d -p ./`
   cp $ANT_HOME/lib/ant.jar $ANT_TEMP/ant.jar
   JAVAC_CP=$ANT_TEMP/ant.jar
-  JAVAC=javac.exe
+elif [ "x$CYGPATH" != "x" ] ; then ## CYGPATH may be set in env.cfg
+  JAVAC_SOURCE_FILE=`$CYGPATH -am $IDEA_OUTPUT/src/idea/IdeaLoggerWrapper.java`
+  JAVAC_SOURCE_PATH=`$CYGPATH -am $IDEA_OUTPUT/src`
+  JAVAC_CLASSES=`$CYGPATH -am $CLASSES`
+  JAVAC_CP=`$CYGPATH -am $CP`
 else
   JAVAC_SOURCE_FILE=$IDEA_OUTPUT/src/idea/IdeaLoggerWrapper.java
   JAVAC_SOURCE_PATH=$IDEA_OUTPUT/src
   JAVAC_CLASSES=$CLASSES
   JAVAC_CP=$CP
-  JAVAC=javac
 fi
 
 $BOOT_JDK/bin/$JAVAC -d $JAVAC_CLASSES -sourcepath $JAVAC_SOURCE_PATH -cp $JAVAC_CP $JAVAC_SOURCE_FILE
diff --git a/doc/building.html b/doc/building.html
index 71c3710c9ae..8b69e11d1b1 100644
--- a/doc/building.html
+++ b/doc/building.html
@@ -506,7 +506,7 @@ <h4 id="advanced-make-control-variables">Advanced Make Control Variables</h4>
 </ul>
 <h2 id="running-tests">Running Tests</h2>
 <p>Most of the JDK tests are using the <a href="http://openjdk.java.net/jtreg">JTReg</a> test framework. Make sure that your configuration knows where to find your installation of JTReg. If this is not picked up automatically, use the <code>--with-jtreg=&lt;path to jtreg home&gt;</code> option to point to the JTReg framework. Note that this option should point to the JTReg home, i.e. the top directory, containing <code>lib/jtreg.jar</code> etc.</p>
-<p>The <a href="https://wiki.openjdk.java.net/display/Adoption">Adoption Group</a> provides recent builds of jtreg <a href="https://ci.adoptopenjdk.net/view/Dependencies/job/jtreg/lastSuccessfulBuild/artifact">here</a>. Download the latest <code>.tar.gz</code> file, unpack it, and point <code>--with-jtreg</code> to the <code>jtreg</code> directory that you just unpacked.</p>
+<p>The <a href="https://wiki.openjdk.java.net/display/Adoption">Adoption Group</a> provides recent builds of jtreg <a href="https://ci.adoptopenjdk.net/view/Dependencies/job/dependency_pipeline/lastSuccessfulBuild/artifact/jtreg/">here</a>. Download the latest <code>.tar.gz</code> file, unpack it, and point <code>--with-jtreg</code> to the <code>jtreg</code> directory that you just unpacked.</p>
 <p>Building of Hotspot Gtest suite requires the source code of Google Test framework. The top directory, which contains both <code>googletest</code> and <code>googlemock</code> directories, should be specified via <code>--with-gtest</code>. The supported version of Google Test is 1.8.1, whose source code can be obtained:</p>
 <ul>
 <li>by downloading and unpacking the source bundle from <a href="https://github.com/google/googletest/releases/tag/release-1.8.1">here</a></li>
diff --git a/doc/building.md b/doc/building.md
index 2f9a0026e28..835a7d99413 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -848,7 +848,7 @@ containing `lib/jtreg.jar` etc.
 
 The [Adoption Group](https://wiki.openjdk.java.net/display/Adoption) provides
 recent builds of jtreg [here](
-https://ci.adoptopenjdk.net/view/Dependencies/job/jtreg/lastSuccessfulBuild/artifact).
+https://ci.adoptopenjdk.net/view/Dependencies/job/dependency_pipeline/lastSuccessfulBuild/artifact/jtreg/).
 Download the latest `.tar.gz` file, unpack it, and point `--with-jtreg` to the
 `jtreg` directory that you just unpacked.
 
diff --git a/make/CreateJmods.gmk b/make/CreateJmods.gmk
index 01ce2cf48ea..6edc69397a5 100644
--- a/make/CreateJmods.gmk
+++ b/make/CreateJmods.gmk
@@ -213,12 +213,12 @@ endif
 
 ifeq ($(call isTargetOs, windows), true)
   ifeq ($(SHIP_DEBUG_SYMBOLS), )
-    JMOD_FLAGS += --exclude '**{_the.*,_*.marker,*.diz,*.pdb,*.map}'
+    JMOD_FLAGS += --exclude '**{_the.*,_*.marker*,*.diz,*.pdb,*.map}'
   else
-    JMOD_FLAGS += --exclude '**{_the.*,_*.marker,*.diz,*.map}'
+    JMOD_FLAGS += --exclude '**{_the.*,_*.marker*,*.diz,*.map}'
   endif
 else
-  JMOD_FLAGS += --exclude '**{_the.*,_*.marker,*.diz,*.debuginfo,*.dSYM/**,*.dSYM}'
+  JMOD_FLAGS += --exclude '**{_the.*,_*.marker*,*.diz,*.debuginfo,*.dSYM/**,*.dSYM}'
 endif
 
 # Create jmods in the support dir and then move them into place to keep the
diff --git a/make/Main.gmk b/make/Main.gmk
index e34766f0611..cad3ecb3203 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -857,6 +857,9 @@ else
   # virtual target.
   jdk.jdwp.agent-libs: jdk.jdwp.agent-gensrc
 
+  # jdk.jfr-gendata uses TOOL_JFR_GEN from buildtools-hotspot
+  jdk.jfr-gendata: buildtools-hotspot
+
   # The swing beans need to have java base properly generated to avoid errors
   # in javadoc. The X11 wrappers need the java.base include files to have been
   # copied and processed.
diff --git a/make/autoconf/build-aux/config.guess b/make/autoconf/build-aux/config.guess
index 3e10c6e9110..d589529f35a 100644
--- a/make/autoconf/build-aux/config.guess
+++ b/make/autoconf/build-aux/config.guess
@@ -102,6 +102,15 @@ if [ "x$OUT" = x ]; then
   fi
 fi
 
+# Test and fix LoongArch64.
+if [ "x$OUT" = x ]; then
+  if [ `uname -s` = Linux ]; then
+    if [ `uname -m` = loongarch64 ]; then
+      OUT=loongarch64-unknown-linux-gnu
+    fi
+  fi
+fi
+
 # Test and fix cpu on macos-aarch64, uname -p reports arm, buildsys expects aarch64
 echo $OUT | grep arm-apple-darwin > /dev/null 2> /dev/null
 if test $? != 0; then
diff --git a/make/autoconf/platform.m4 b/make/autoconf/platform.m4
index 2dd13d0d5e2..205d64f566d 100644
--- a/make/autoconf/platform.m4
+++ b/make/autoconf/platform.m4
@@ -72,6 +72,12 @@ AC_DEFUN([PLATFORM_EXTRACT_VARS_FROM_CPU],
       VAR_CPU_BITS=64
       VAR_CPU_ENDIAN=little
       ;;
+    loongarch64)
+      VAR_CPU=loongarch64
+      VAR_CPU_ARCH=loongarch
+      VAR_CPU_BITS=64
+      VAR_CPU_ENDIAN=little
+      ;;
     m68k)
       VAR_CPU=m68k
       VAR_CPU_ARCH=m68k
diff --git a/make/autoconf/toolchain.m4 b/make/autoconf/toolchain.m4
index 78895888095..69540e1608e 100644
--- a/make/autoconf/toolchain.m4
+++ b/make/autoconf/toolchain.m4
@@ -221,6 +221,12 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETERMINE_TOOLCHAIN_TYPE],
   AC_ARG_WITH(toolchain-type, [AS_HELP_STRING([--with-toolchain-type],
       [the toolchain type (or family) to use, use '--help' to show possible values @<:@platform dependent@:>@])])
 
+  # Linux x86_64 needs higher binutils after 8265783
+  # (this really is a dependency on as version, but we take ld as a check for a general binutils version)
+  if test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
+    TOOLCHAIN_MINIMUM_LD_VERSION_gcc="2.25"
+  fi
+
   # Use indirect variable referencing
   toolchain_var_name=VALID_TOOLCHAINS_$OPENJDK_BUILD_OS
   VALID_TOOLCHAINS=${!toolchain_var_name}
@@ -677,9 +683,10 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETECT_TOOLCHAIN_CORE],
   TOOLCHAIN_PREPARE_FOR_LD_VERSION_COMPARISONS
 
   if test "x$TOOLCHAIN_MINIMUM_LD_VERSION" != x; then
+    AC_MSG_NOTICE([comparing linker version to minimum version $TOOLCHAIN_MINIMUM_LD_VERSION])
     TOOLCHAIN_CHECK_LINKER_VERSION(VERSION: $TOOLCHAIN_MINIMUM_LD_VERSION,
         IF_OLDER_THAN: [
-          AC_MSG_WARN([You are using a linker older than $TOOLCHAIN_MINIMUM_LD_VERSION. This is not a supported configuration.])
+          AC_MSG_ERROR([You are using a linker older than $TOOLCHAIN_MINIMUM_LD_VERSION. This is not a supported configuration.])
         ]
     )
   fi
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 65c17f6bd7e..7b1e1f2c648 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -1164,7 +1164,7 @@ var getJibProfilesDependencies = function (input, common) {
             // build_number: "b07",
             // file: "bundles/jcov-3_0.zip",
             organization: common.organization,
-            revision: "3.0-7-jdk-asm+1.0",
+            revision: "3.0-9-jdk-asm+1.0",
             ext: "zip",
             environment_name: "JCOV_HOME",
         },
diff --git a/make/data/symbols/java.base-H.sym.txt b/make/data/symbols/java.base-H.sym.txt
index 753fd5505bc..585d05063b3 100644
--- a/make/data/symbols/java.base-H.sym.txt
+++ b/make/data/symbols/java.base-H.sym.txt
@@ -114,8 +114,8 @@ header extends java/lang/Object implements java/lang/annotation/Annotation flags
 class name java/lang/System
 -method name setSecurityManager descriptor (Ljava/lang/SecurityManager;)V
 -method name getSecurityManager descriptor ()Ljava/lang/SecurityManager;
-method name setSecurityManager descriptor (Ljava/lang/SecurityManager;)V flags 9 deprecated true runtimeAnnotations @Ljava/lang/Deprecated;(forRemoval=Ztrue,since="17")
 method name getSecurityManager descriptor ()Ljava/lang/SecurityManager; flags 9 deprecated true runtimeAnnotations @Ljava/lang/Deprecated;(forRemoval=Ztrue,since="17")
+method name setSecurityManager descriptor (Ljava/lang/SecurityManager;)V flags 9 deprecated true runtimeAnnotations @Ljava/lang/Deprecated;(forRemoval=Ztrue,since="17")@Ljdk/internal/reflect/CallerSensitive;
 
 class name java/lang/Thread
 -method name checkAccess descriptor ()V
@@ -209,7 +209,8 @@ header extends java/lang/RuntimeException flags 21
 class name java/lang/runtime/SwitchBootstraps
 header extends java/lang/Object flags 21 classAnnotations @Ljdk/internal/javac/PreviewFeature;(feature=eLjdk/internal/javac/PreviewFeature$Feature;SWITCH_PATTERN_MATCHING;)
 innerclass innerClass java/lang/invoke/MethodHandles$Lookup outerClass java/lang/invoke/MethodHandles innerClassName Lookup flags 19
-method name typeSwitch descriptor (Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite; thrownTypes java/lang/Throwable flags 89
+method name typeSwitch descriptor (Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite; flags 89
+method name enumSwitch descriptor (Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite; flags 89
 
 class name java/net/DatagramSocket
 -method name setDatagramSocketImplFactory descriptor (Ljava/net/DatagramSocketImplFactory;)V
@@ -454,8 +455,7 @@ innerclass innerClass java/io/ObjectInputStream$GetField outerClass java/io/Obje
 innerclass innerClass java/io/ObjectOutputStream$PutField outerClass java/io/ObjectOutputStream innerClassName PutField flags 409
 
 class name java/util/SplittableRandom
-header extends jdk/internal/util/random/RandomSupport$AbstractSplittableGenerator flags 31 runtimeAnnotations @Ljdk/internal/util/random/RandomSupport$RandomGeneratorProperties;(name="SplittableRandom",i=I64,j=I0,k=I0,equidistribution=I1)
-innerclass innerClass jdk/internal/util/random/RandomSupport$AbstractSplittableGenerator outerClass jdk/internal/util/random/RandomSupport innerClassName AbstractSplittableGenerator flags 409
+header extends java/lang/Object implements java/util/random/RandomGenerator,java/util/random/RandomGenerator$SplittableGenerator flags 31 runtimeAnnotations @Ljdk/internal/util/random/RandomSupport$RandomGeneratorProperties;(name="SplittableRandom",i=I64,j=I0,k=I0,equidistribution=I1)
 innerclass innerClass java/util/random/RandomGenerator$SplittableGenerator outerClass java/util/random/RandomGenerator innerClassName SplittableGenerator flags 609
 -method name nextInt descriptor (I)I
 -method name nextInt descriptor (II)I
diff --git a/make/data/symbols/jdk.incubator.foreign-H.sym.txt b/make/data/symbols/jdk.incubator.foreign-H.sym.txt
index c33a11c6812..376ba519f57 100644
--- a/make/data/symbols/jdk.incubator.foreign-H.sym.txt
+++ b/make/data/symbols/jdk.incubator.foreign-H.sym.txt
@@ -39,6 +39,7 @@ innerclass innerClass jdk/incubator/foreign/CLinker$VaList$Builder outerClass jd
 -method name toCString descriptor (Ljava/lang/String;Ljava/nio/charset/Charset;Ljdk/incubator/foreign/NativeScope;)Ljdk/incubator/foreign/MemorySegment;
 -method name toJavaStringRestricted descriptor (Ljdk/incubator/foreign/MemoryAddress;)Ljava/lang/String;
 -method name toJavaStringRestricted descriptor (Ljdk/incubator/foreign/MemoryAddress;Ljava/nio/charset/Charset;)Ljava/lang/String;
+-method name toJavaString descriptor (Ljdk/incubator/foreign/MemorySegment;Ljava/nio/charset/Charset;)Ljava/lang/String;
 -method name allocateMemoryRestricted descriptor (J)Ljdk/incubator/foreign/MemoryAddress;
 -method name freeMemoryRestricted descriptor (Ljdk/incubator/foreign/MemoryAddress;)V
 method name getInstance descriptor ()Ljdk/incubator/foreign/CLinker; flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
@@ -47,10 +48,7 @@ method name downcallHandle descriptor (Ljava/lang/invoke/MethodType;Ljdk/incubat
 method name upcallStub descriptor (Ljava/lang/invoke/MethodHandle;Ljdk/incubator/foreign/FunctionDescriptor;Ljdk/incubator/foreign/ResourceScope;)Ljdk/incubator/foreign/MemoryAddress; flags 401
 method name toCString descriptor (Ljava/lang/String;Ljdk/incubator/foreign/SegmentAllocator;)Ljdk/incubator/foreign/MemorySegment; flags 9
 method name toCString descriptor (Ljava/lang/String;Ljdk/incubator/foreign/ResourceScope;)Ljdk/incubator/foreign/MemorySegment; flags 9
-method name toCString descriptor (Ljava/lang/String;Ljava/nio/charset/Charset;Ljdk/incubator/foreign/SegmentAllocator;)Ljdk/incubator/foreign/MemorySegment; flags 9
-method name toCString descriptor (Ljava/lang/String;Ljava/nio/charset/Charset;Ljdk/incubator/foreign/ResourceScope;)Ljdk/incubator/foreign/MemorySegment; flags 9
 method name toJavaString descriptor (Ljdk/incubator/foreign/MemoryAddress;)Ljava/lang/String; flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
-method name toJavaString descriptor (Ljdk/incubator/foreign/MemoryAddress;Ljava/nio/charset/Charset;)Ljava/lang/String; flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
 method name allocateMemory descriptor (J)Ljdk/incubator/foreign/MemoryAddress; flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
 method name freeMemory descriptor (Ljdk/incubator/foreign/MemoryAddress;)V flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
 method name systemLookup descriptor ()Ljdk/incubator/foreign/SymbolLookup; flags 9 runtimeAnnotations @Ljdk/internal/reflect/CallerSensitive;
diff --git a/make/data/symbols/jdk.javadoc-H.sym.txt b/make/data/symbols/jdk.javadoc-H.sym.txt
index 4681a5674de..b2ab2561f5c 100644
--- a/make/data/symbols/jdk.javadoc-H.sym.txt
+++ b/make/data/symbols/jdk.javadoc-H.sym.txt
@@ -29,9 +29,9 @@
 class name jdk/javadoc/doclet/Reporter
 header extends java/lang/Object flags 601
 innerclass innerClass javax/tools/Diagnostic$Kind outerClass javax/tools/Diagnostic innerClassName Kind flags 4019
-method name print descriptor (Ljavax/tools/Diagnostic$Kind;Ljavax/tools/FileObject;IIILjava/lang/String;)V flags 401
 method name getStandardWriter descriptor ()Ljava/io/PrintWriter; flags 1
 method name getDiagnosticWriter descriptor ()Ljava/io/PrintWriter; flags 1
+method name print descriptor (Ljavax/tools/Diagnostic$Kind;Ljavax/tools/FileObject;IIILjava/lang/String;)V flags 1
 
 class name jdk/javadoc/doclet/StandardDoclet
 header extends java/lang/Object implements jdk/javadoc/doclet/Doclet flags 21
diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk
index e378191f074..5d597b9ad11 100644
--- a/make/hotspot/lib/JvmFeatures.gmk
+++ b/make/hotspot/lib/JvmFeatures.gmk
@@ -116,8 +116,12 @@ endif
 ifneq ($(call check-jvm-feature, cds), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_CDS=0
   JVM_EXCLUDE_FILES += \
+      cdsProtectionDomain.cpp \
       classLoaderDataShared.cpp \
       classLoaderExt.cpp \
+      dumpTimeSharedClassInfo.cpp \
+      lambdaProxyClassDictionary.cpp \
+      runTimeSharedClassInfo.cpp \
       systemDictionaryShared.cpp
   JVM_EXCLUDE_PATTERNS += cds/
 endif
@@ -183,7 +187,6 @@ ifeq ($(call check-jvm-feature, opt-size), true)
       assembler.cpp \
       barrierSet.cpp \
       basicLock.cpp \
-      biasedLocking.cpp \
       bytecode.cpp \
       bytecodeInterpreter.cpp \
       c1_Compilation.cpp \
diff --git a/make/hotspot/lib/JvmOverrideFiles.gmk b/make/hotspot/lib/JvmOverrideFiles.gmk
index a9f8a0e54ed..d53694fb2df 100644
--- a/make/hotspot/lib/JvmOverrideFiles.gmk
+++ b/make/hotspot/lib/JvmOverrideFiles.gmk
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,7 @@ ifeq ($(TOOLCHAIN_TYPE), gcc)
   BUILD_LIBJVM_assembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_cardTableBarrierSetAssembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_interp_masm_x86.cpp_CXXFLAGS := -Wno-uninitialized
+  BUILD_LIBJVM_ad_$(HOTSPOT_TARGET_CPU_ARCH).cpp_CXXFLAGS := -Wno-nonnull
   ifeq ($(DEBUG_LEVEL), release)
     # Need extra inlining to collapse shared marking code into the hot marking loop
     BUILD_LIBJVM_shenandoahMark.cpp_CXXFLAGS := --param inline-unit-growth=1000
diff --git a/make/ide/idea/jdk/template/misc.xml b/make/ide/idea/jdk/template/misc.xml
index 669c382327a..48600125348 100644
--- a/make/ide/idea/jdk/template/misc.xml
+++ b/make/ide/idea/jdk/template/misc.xml
@@ -12,7 +12,7 @@
       <target file="file://###ROOT_DIR###/make/ide/idea/jdk/build.xml" name="images" />
     </ant>
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_16" assert-keyword="true" jdk-15="true">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_X" assert-keyword="true" project-jdk-type="JavaSDK">
     <output url="file://###BUILD_DIR###/idea" />
   </component>
 </project>
diff --git a/make/ide/idea/jdk/template/module.iml b/make/ide/idea/jdk/template/module.iml
deleted file mode 100644
index 446d055f240..00000000000
--- a/make/ide/idea/jdk/template/module.iml
+++ /dev/null
@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://###MODULE_DIR###/###MODULE_CONTENT###">
-      ###SOURCE_DIRS###
-    </content>
-    <orderEntry type="sourceFolder" forTests="false" />
-    ###DEPENDENCIES###
-    <orderEntry type="inheritedJdk" />
-  </component>
-</module>
-
diff --git a/make/ide/idea/jdk/template/test.iml b/make/ide/idea/jdk/template/test.iml
deleted file mode 100644
index f684c675afe..00000000000
--- a/make/ide/idea/jdk/template/test.iml
+++ /dev/null
@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://###MODULE_DIR###/test/jdk"></content>
-    <orderEntry type="sourceFolder" forTests="true" />
-    ###TEST_MODULE_DEPENDENCIES###
-    <orderEntry type="inheritedJdk" />
-  </component>
-</module>
diff --git a/make/langtools/src/classes/build/tools/symbolgenerator/CreateSymbols.java b/make/langtools/src/classes/build/tools/symbolgenerator/CreateSymbols.java
index 74a817e990b..71ff01f94dd 100644
--- a/make/langtools/src/classes/build/tools/symbolgenerator/CreateSymbols.java
+++ b/make/langtools/src/classes/build/tools/symbolgenerator/CreateSymbols.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -229,6 +229,7 @@ public void createSymbols(String ctDescriptionFileExtra, String ctDescriptionFil
                                                                     : null,
                                      Paths.get(ctDescriptionFile));
 
+        stripNonExistentAnnotations(data);
         splitHeaders(data.classes);
 
         Map<String, Map<Character, String>> package2Version2Module = new HashMap<>();
@@ -301,6 +302,50 @@ public void createSymbols(String ctDescriptionFileExtra, String ctDescriptionFil
         }
     }
 
+    private static final String PREVIEW_FEATURE_ANNOTATION_OLD =
+            "Ljdk/internal/PreviewFeature;";
+    private static final String PREVIEW_FEATURE_ANNOTATION_NEW =
+            "Ljdk/internal/javac/PreviewFeature;";
+    private static final String PREVIEW_FEATURE_ANNOTATION_INTERNAL =
+            "Ljdk/internal/PreviewFeature+Annotation;";
+    private static final String VALUE_BASED_ANNOTATION =
+            "Ljdk/internal/ValueBased;";
+    private static final String VALUE_BASED_ANNOTATION_INTERNAL =
+            "Ljdk/internal/ValueBased+Annotation;";
+    public static final Set<String> HARDCODED_ANNOTATIONS = new HashSet<>(
+            List.of("Ljdk/Profile+Annotation;",
+                    "Lsun/Proprietary+Annotation;",
+                    PREVIEW_FEATURE_ANNOTATION_OLD,
+                    PREVIEW_FEATURE_ANNOTATION_NEW,
+                    VALUE_BASED_ANNOTATION));
+
+    private void stripNonExistentAnnotations(LoadDescriptions data) {
+        Set<String> allClasses = data.classes.name2Class.keySet();
+        data.modules.values().forEach(mod -> {
+            stripNonExistentAnnotations(allClasses, mod.header);
+        });
+        data.classes.classes.forEach(clazz -> {
+            stripNonExistentAnnotations(allClasses, clazz.header);
+            stripNonExistentAnnotations(allClasses, clazz.fields);
+            stripNonExistentAnnotations(allClasses, clazz.methods);
+        });
+    }
+
+    private void stripNonExistentAnnotations(Set<String> allClasses, Iterable<? extends FeatureDescription> descs) {
+        descs.forEach(d -> stripNonExistentAnnotations(allClasses, d));
+    }
+
+    private void stripNonExistentAnnotations(Set<String> allClasses, FeatureDescription d) {
+        stripNonExistentAnnotations(allClasses, d.classAnnotations);
+        stripNonExistentAnnotations(allClasses, d.runtimeAnnotations);
+    }
+
+    private void stripNonExistentAnnotations(Set<String> allClasses, List<AnnotationDescription> annotations) {
+        if (annotations != null)
+            annotations.removeIf(ann -> !HARDCODED_ANNOTATIONS.contains(ann.annotationType) &&
+                                        !allClasses.contains(ann.annotationType.substring(1, ann.annotationType.length() - 1)));
+    }
+
     private ZipEntry createZipEntry(String name, long timestamp) {
         ZipEntry ze = new ZipEntry(name);
 
@@ -1140,17 +1185,16 @@ private Annotation createAnnotation(List<CPInfo> constantPool, AnnotationDescrip
             values.put("reflective", essentialAPI != null && !essentialAPI);
         }
 
+        if (VALUE_BASED_ANNOTATION.equals(annotationType)) {
+            //the non-public ValueBased annotation will not be available in ct.sym,
+            //replace with purely synthetic javac-internal annotation:
+            annotationType = VALUE_BASED_ANNOTATION_INTERNAL;
+        }
+
         return new Annotation(null,
                               addString(constantPool, annotationType),
                               createElementPairs(constantPool, values));
     }
-    //where:
-        private static final String PREVIEW_FEATURE_ANNOTATION_OLD =
-                "Ljdk/internal/PreviewFeature;";
-        private static final String PREVIEW_FEATURE_ANNOTATION_NEW =
-                "Ljdk/internal/javac/PreviewFeature;";
-        private static final String PREVIEW_FEATURE_ANNOTATION_INTERNAL =
-                "Ljdk/internal/PreviewFeature+Annotation;";
 
     private element_value_pair[] createElementPairs(List<CPInfo> constantPool, Map<String, Object> annotationAttributes) {
         element_value_pair[] pairs = new element_value_pair[annotationAttributes.size()];
diff --git a/make/modules/jdk.javadoc/Gendata.gmk b/make/modules/jdk.javadoc/Gendata.gmk
index 78432cd6a68..c648df6e032 100644
--- a/make/modules/jdk.javadoc/Gendata.gmk
+++ b/make/modules/jdk.javadoc/Gendata.gmk
@@ -61,36 +61,54 @@ $(eval $(call SetupJavaCompilation, COMPILE_CREATE_SYMBOLS, \
         $(COMPILECREATESYMBOLS_ADD_EXPORTS), \
 ))
 
-$(SUPPORT_OUTPUTDIR)/javadoc-symbols/symbols: \
+GENERATE_SYMBOLS_FROM_JDK_VERSION := 11
+JDK_JAVADOC_DIR := $(JDK_OUTPUTDIR)/modules/jdk.javadoc
+ELEMENT_LISTS_PKG := jdk/javadoc/internal/doclets/toolkit/resources/releases
+ELEMENT_LISTS_DIR := $(JDK_JAVADOC_DIR)/$(ELEMENT_LISTS_PKG)
+
+$(JDK_JAVADOC_DIR)/_element_lists.marker: \
     $(COMPILE_CREATE_SYMBOLS) \
     $(wildcard $(TOPDIR)/make/data/symbols/*) \
     $(MODULE_INFOS)
-	$(RM) -r $(@D)
-	$(MKDIR) -p $(@D)
-	$(ECHO) Creating javadoc element list
-	$(JAVA_SMALL) $(INTERIM_LANGTOOLS_ARGS) \
-	    $(COMPILECREATESYMBOLS_ADD_EXPORTS) \
-	    -classpath $(BUILDTOOLS_OUTPUTDIR)/create_symbols_javadoc \
-	    build.tools.symbolgenerator.CreateSymbols \
-	    build-javadoc-data \
-	    $(CT_DATA_DESCRIPTION) \
-	    $(JDK_OUTPUTDIR)/modules/jdk.javadoc/jdk/javadoc/internal/doclets/toolkit/resources/releases \
-	    11
-	$(JAVA_SMALL) $(INTERIM_LANGTOOLS_ARGS) \
-	    $(COMPILECREATESYMBOLS_ADD_EXPORTS) \
-	    -classpath $(BUILDTOOLS_OUTPUTDIR)/create_symbols_javadoc \
-	    build.tools.symbolgenerator.JavadocElementList \
-	    $(JDK_OUTPUTDIR)/modules/jdk.javadoc/jdk/javadoc/internal/doclets/toolkit/resources/releases/element-list-$(JDK_SOURCE_TARGET_VERSION).txt \
-	    $(JAVADOC_MODULESOURCEPATH) \
-	    $(JAVADOC_MODULES)
+	$(call MakeTargetDir)
+	$(call LogInfo, Creating javadoc element lists)
+	$(RM) -r $(ELEMENT_LISTS_DIR)
+        # Generate element-list files for JDK 11 to current-1
+	$(call ExecuteWithLog, $@_historic, \
+	    $(JAVA_SMALL) $(INTERIM_LANGTOOLS_ARGS) \
+	        $(COMPILECREATESYMBOLS_ADD_EXPORTS) \
+	        -classpath $(BUILDTOOLS_OUTPUTDIR)/create_symbols_javadoc \
+	        build.tools.symbolgenerator.CreateSymbols \
+	        build-javadoc-data \
+	        $(CT_DATA_DESCRIPTION) \
+	        $(ELEMENT_LISTS_DIR) \
+	        $(GENERATE_SYMBOLS_FROM_JDK_VERSION) \
+	)
+        # Generate element-list file for the current JDK version
+	$(call ExecuteWithLog, $@_current, \
+	    $(JAVA_SMALL) $(INTERIM_LANGTOOLS_ARGS) \
+	        $(COMPILECREATESYMBOLS_ADD_EXPORTS) \
+	        -classpath $(BUILDTOOLS_OUTPUTDIR)/create_symbols_javadoc \
+	        build.tools.symbolgenerator.JavadocElementList \
+	        $(ELEMENT_LISTS_DIR)/element-list-$(JDK_SOURCE_TARGET_VERSION).txt \
+	        $(JAVADOC_MODULESOURCEPATH) \
+	        $(JAVADOC_MODULES) \
+	)
 	$(TOUCH) $@
 
-# Copy ct.sym to the modules libs dir
-$(eval $(call SetupCopyFiles, COPY_TO_LIBS, \
-    FILES := $(SUPPORT_OUTPUTDIR)/javadoc-symbols/*.txt, \
-    DEST := $(JDK_OUTPUTDIR)/modules/jdk.javadoc/jdk/javadoc/internal/doclets/toolkit/resources/releases, \
-))
+################################################################################
+# Copy element-lists to interim langtools
 
-TARGETS += $(SUPPORT_OUTPUTDIR)/javadoc-symbols/symbols
+INTERIM_JDK_JAVADOC_DIR := $(BUILDTOOLS_OUTPUTDIR)/interim_langtools_modules/jdk.javadoc.interim
+INTERIM_ELEMENT_LISTS_DIR := $(INTERIM_JDK_JAVADOC_DIR)/$(ELEMENT_LISTS_PKG)
+
+$(INTERIM_JDK_JAVADOC_DIR)/_element_lists.marker: $(JDK_JAVADOC_DIR)/_element_lists.marker
+	$(call MakeDir, $(INTERIM_ELEMENT_LISTS_DIR))
+	$(RM) -r $(INTERIM_ELEMENT_LISTS_DIR)/*
+	$(CP) -R $(ELEMENT_LISTS_DIR)/* $(INTERIM_ELEMENT_LISTS_DIR)/
+	$(TOUCH) $@
 
 ################################################################################
+
+TARGETS += $(JDK_JAVADOC_DIR)/_element_lists.marker \
+    $(INTERIM_JDK_JAVADOC_DIR)/_element_lists.marker
diff --git a/make/test/JtregNativeJdk.gmk b/make/test/JtregNativeJdk.gmk
index 3342710bcd2..8ed5cbd2a58 100644
--- a/make/test/JtregNativeJdk.gmk
+++ b/make/test/JtregNativeJdk.gmk
@@ -53,6 +53,8 @@ BUILD_JDK_JTREG_EXECUTABLES_CFLAGS_exeJliLaunchTest := \
     -I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/native/libjli \
     -I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/native/libjli
 
+BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libAsyncStackWalk := $(LIBCXX)
+
 # Platform specific setup
 ifeq ($(call isTargetOs, windows), true)
   BUILD_JDK_JTREG_EXCLUDE += libDirectIO.c libInheritedChannel.c exelauncher.c
@@ -63,6 +65,7 @@ ifeq ($(call isTargetOs, windows), true)
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeJliLaunchTest := $(WIN_LIB_JLI)
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeCallerAccessTest := jvm.lib
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exerevokeall := advapi32.lib
+  BUILD_JDK_JTREG_LIBRARIES_CFLAGS_libAsyncStackWalk := /EHsc
 else
   BUILD_JDK_JTREG_LIBRARIES_LIBS_libstringPlatformChars := -ljava
   BUILD_JDK_JTREG_LIBRARIES_LIBS_libDirectIO := -ljava
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index 47053f15c9a..404b4612e01 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1295,7 +1295,7 @@ public:
   };
 };
 
- bool is_CAS(int opcode, bool maybe_volatile);
+  bool is_CAS(int opcode, bool maybe_volatile);
 
   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 
@@ -2416,6 +2416,14 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
   } else { // NEON
     // Special cases
     switch (opcode) {
+    case Op_VectorMaskCmp:
+    // We don't have VectorReinterpret with bit_size less than 64 support for
+    // now, even for byte type. To be refined with fully VectorCast support.
+    case Op_VectorReinterpret:
+      if (vlen < 2 || bit_size < 64) {
+        return false;
+      }
+      break;
     case Op_MulAddVS2VI:
       if (bit_size < 128) {
         return false;
@@ -2429,6 +2437,23 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
         return false;
       }
       break;
+    // Some types of VectorCast are not implemented for now.
+    case Op_VectorCastI2X:
+      if (bt == T_BYTE) {
+        return false;
+      }
+      break;
+    case Op_VectorCastS2X:
+      if (vlen < 4 || bit_size < 64) {
+        return false;
+      }
+      break;
+    case Op_VectorCastF2X:
+    case Op_VectorCastD2X:
+      if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
+        return false;
+      }
+      break;
     default:
       break;
     }
@@ -2454,10 +2479,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Is this branch offset short enough that a short branch can be used?
 //
 // NOTE: If the platform does not provide any short branch variants, then
@@ -2529,7 +2550,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd,
   return NULL;
 }
 
-bool Matcher::is_generic_reg2reg_move(MachNode* m) {
+bool Matcher::is_reg2reg_move(MachNode* m) {
   ShouldNotReachHere();  // generic vector operands not supported
   return false;
 }
@@ -2570,6 +2591,39 @@ bool Matcher::is_spillable_arg(int reg)
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  // JDK-8183543: When taking the number of available registers as int
+  // register pressure threshold, the jtreg test:
+  // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
+  // failed due to C2 compilation failure with
+  // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
+  //
+  // A derived pointer is live at CallNode and then is flagged by RA
+  // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
+  // derived pointers and lastly fail to spill after reaching maximum
+  // number of iterations. Lowering the default pressure threshold to
+  // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
+  // a high register pressure area of the code so that split_DEF can
+  // generate DefinitionSpillCopy for the derived pointer.
+  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
+  if (!PreserveFramePointer) {
+    // When PreserveFramePointer is off, frame pointer is allocatable,
+    // but different from other SOC registers, it is excluded from
+    // fatproj's mask because its save type is No-Save. Decrease 1 to
+    // ensure high pressure at fatproj when PreserveFramePointer is off.
+    // See check_pressure_at_fatproj().
+    default_int_pressure_threshold--;
+  }
+  return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
+  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }
@@ -3794,10 +3848,6 @@ encode %{
       __ br(Assembler::NE, cont);
     }
 
-    if (UseBiasedLocking && !UseOptoBiasInlining) {
-      __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
-    }
-
     // Check for existing monitor
     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 
@@ -3805,7 +3855,6 @@ encode %{
     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 
     if (EnableValhalla) {
-      assert(!UseBiasedLocking, "Not compatible with biased-locking");
       // Mask inline_type bit such that we go to the slow path if object is an inline type
       __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
     }
@@ -3874,10 +3923,6 @@ encode %{
 
     assert_different_registers(oop, box, tmp, disp_hdr);
 
-    if (UseBiasedLocking && !UseOptoBiasInlining) {
-      __ biased_locking_exit(oop, tmp, cont);
-    }
-
     // Find the lock address and load the displaced header from the stack.
     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 
@@ -5490,6 +5535,15 @@ operand vRegD_V31()
   interface(REG_INTER);
 %}
 
+operand pReg()
+%{
+  constraint(ALLOC_IN_RC(pr_reg));
+  match(RegVectMask);
+  op_cost(0);
+  format %{ %}
+  interface(REG_INTER);
+%}
+
 operand pRegGov()
 %{
   constraint(ALLOC_IN_RC(gov_pr));
@@ -8914,11 +8968,6 @@ instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFl
   ins_pipe(pipe_serial);
 %}
 
-
-// storeLConditional is used by PhaseMacroExpand::expand_lock_node
-// when attempting to rebias a lock towards the current thread.  We
-// must use the acquire form of cmpxchg in order to guarantee acquire
-// semantics in this case.
 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 %{
   match(Set cr (StoreLConditional mem (Binary oldval newval)));
@@ -16712,11 +16761,11 @@ instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
 %}
 
 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
-                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
-                              iRegINoSp tmp3, rFlagsReg cr)
+                             iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
+                             iRegINoSp tmp3, rFlagsReg cr)
 %{
   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
-  predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
+  predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
 
@@ -16735,7 +16784,7 @@ instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
                               iRegINoSp tmp3, rFlagsReg cr)
 %{
   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
-  predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
+  predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
 
@@ -16743,8 +16792,8 @@ instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
 
   ins_encode %{
     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
-                           $result$$Register, $tmp1$$Register, $tmp2$$Register,
-                           $tmp3$$Register);
+                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
+                            $tmp3$$Register);
   %}
   ins_pipe(pipe_class_memory);
 %}
diff --git a/src/hotspot/cpu/aarch64/aarch64_neon.ad b/src/hotspot/cpu/aarch64/aarch64_neon.ad
index 1beac317c57..b70893ee1f4 100644
--- a/src/hotspot/cpu/aarch64/aarch64_neon.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_neon.ad
@@ -1850,7 +1850,7 @@ instruct vcmpD(vecD dst, vecD src1, vecD src2, immI cond)
   format %{ "vcmpD  $dst, $src1, $src2\t# vector compare " %}
   ins_cost(INSN_COST);
   ins_encode %{
-    BasicType bt = vector_element_basic_type(this);
+    BasicType bt = Matcher::vector_element_basic_type(this);
     assert(type2aelembytes(bt) != 8, "not supported");
     __ neon_compare(as_FloatRegister($dst$$reg), bt, as_FloatRegister($src1$$reg),
                     as_FloatRegister($src2$$reg), (int)$cond$$constant, /*isQ*/ false);
@@ -1865,7 +1865,7 @@ instruct vcmpX(vecX dst, vecX src1, vecX src2, immI cond)
   format %{ "vcmpX  $dst, $src1, $src2\t# vector compare " %}
   ins_cost(INSN_COST);
   ins_encode %{
-    BasicType bt = vector_element_basic_type(this);
+    BasicType bt = Matcher::vector_element_basic_type(this);
     __ neon_compare(as_FloatRegister($dst$$reg), bt, as_FloatRegister($src1$$reg),
                     as_FloatRegister($src2$$reg), (int)$cond$$constant, /*isQ*/ true);
   %}
@@ -5296,3 +5296,172 @@ instruct vpopcount2I(vecD dst, vecD src) %{
   %}
   ins_pipe(pipe_class_default);
 %}
+
+// vector mask reductions
+
+instruct vmask_truecount8B(iRegINoSp dst, vecD src, vecD tmp) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskTrueCount src));
+  effect(TEMP tmp);
+  ins_cost(2 * INSN_COST);
+  format %{ "addv $tmp, $src\n\t"
+            "umov $dst, $tmp, B, 0\t# vector (8B)" %}
+  ins_encode %{
+    // Input "src" is a vector of boolean represented as bytes with
+    // 0x00/0x01 as element values.
+    __ addv(as_FloatRegister($tmp$$reg), __ T8B, as_FloatRegister($src$$reg));
+    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_truecount16B(iRegINoSp dst, vecX src, vecX tmp) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskTrueCount src));
+  effect(TEMP tmp);
+  ins_cost(2 * INSN_COST);
+  format %{ "addv $tmp, $src\n\t"
+            "umov $dst, $tmp, B, 0\t# vector (16B)" %}
+  ins_encode %{
+    // Input "src" is a vector of boolean represented as bytes with
+    // 0x00/0x01 as element values.
+    __ addv(as_FloatRegister($tmp$$reg), __ T16B, as_FloatRegister($src$$reg));
+    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_firsttrue_LT8B(iRegINoSp dst, vecD src, rFlagsReg cr) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN &&
+            n->in(1)->bottom_type()->is_vect()->length() < 8);
+  match(Set dst (VectorMaskFirstTrue src));
+  effect(KILL cr);
+  ins_cost(7 * INSN_COST);
+  format %{ "vmask_firsttrue $dst, $src\t# vector (4I/4S/2I)" %}
+  ins_encode %{
+    // Returns the index of the first active lane of the
+    // vector mask, or VLENGTH if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+    //
+    // Computed by reversing the bits and counting the leading
+    // zero bytes.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ rbit($dst$$Register, $dst$$Register);
+    __ clz($dst$$Register, $dst$$Register);
+    __ lsrw($dst$$Register, $dst$$Register, 3);
+    __ movw(rscratch1, Matcher::vector_length(this, $src));
+    __ cmpw($dst$$Register, rscratch1);
+    __ cselw($dst$$Register, rscratch1, $dst$$Register, Assembler::GE);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_firsttrue8B(iRegINoSp dst, vecD src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN &&
+            n->in(1)->bottom_type()->is_vect()->length() == 8);
+  match(Set dst (VectorMaskFirstTrue src));
+  ins_cost(4 * INSN_COST);
+  format %{ "vmask_firsttrue $dst, $src\t# vector (8B)" %}
+  ins_encode %{
+    // Returns the index of the first active lane of the
+    // vector mask, or VLENGTH if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+    //
+    // Computed by reversing the bits and counting the leading
+    // zero bytes.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ rbit($dst$$Register, $dst$$Register);
+    __ clz($dst$$Register, $dst$$Register);
+    __ lsrw($dst$$Register, $dst$$Register, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_firsttrue16B(iRegINoSp dst, vecX src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskFirstTrue src));
+  ins_cost(6 * INSN_COST);
+  format %{ "vmask_firsttrue $dst, $src\t# vector (16B)" %}
+  ins_encode %{
+    // Returns the index of the first active lane of the
+    // vector mask, or 16 (VLENGTH) if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+
+    Label FIRST_TRUE_INDEX;
+
+    // Try to compute the result from lower 64 bits.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, zr);
+    __ cbnz($dst$$Register, FIRST_TRUE_INDEX);
+
+    // Compute the result from the higher 64 bits.
+    __ fmovhid($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 8);
+
+    // Reverse the bits and count the leading zero bytes.
+    __ bind(FIRST_TRUE_INDEX);
+    __ rbit($dst$$Register, $dst$$Register);
+    __ clz($dst$$Register, $dst$$Register);
+    __ addw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_lasttrue8B(iRegINoSp dst, vecD src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskLastTrue src));
+  ins_cost(4 * INSN_COST);
+  format %{ "vmask_lasttrue $dst, $src\t# vector (8B)" %}
+  ins_encode %{
+    // Returns the index of the last active lane of the
+    // vector mask, or -1 if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+    //
+    // Computed by counting the leading zero bytes and
+    // substracting it by 7 (VLENGTH - 1).
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ clz($dst$$Register, $dst$$Register);
+    __ movw(rscratch1, 7);
+    __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_lasttrue16B(iRegINoSp dst, vecX src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskLastTrue src));
+  ins_cost(5 * INSN_COST);
+  format %{ "vmask_lasttrue $dst, $src\t# vector (16B)" %}
+  ins_encode %{
+    // Returns the index of the last active lane of the
+    // vector mask, or -1 if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+
+    Label LAST_TRUE_INDEX;
+
+    // Try to compute the result from higher 64 bits.
+    __ fmovhid($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 16 - 1);
+    __ cbnz($dst$$Register, LAST_TRUE_INDEX);
+
+    // Compute the result from the lower 64 bits.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 8 - 1);
+
+    // Count the leading zero bytes and substract it by 15 (VLENGTH - 1).
+    __ bind(LAST_TRUE_INDEX);
+    __ clz($dst$$Register, $dst$$Register);
+    __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
diff --git a/src/hotspot/cpu/aarch64/aarch64_neon_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_neon_ad.m4
index 306cd1b56ce..076eec1c973 100644
--- a/src/hotspot/cpu/aarch64/aarch64_neon_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_neon_ad.m4
@@ -2243,3 +2243,151 @@ instruct vpopcount$1$2`'(vec$5 dst, vec$5 src) %{
 dnl       $1 $2 $3  $4 $5
 VPOPCOUNT(4, I, 16, 8, X)
 VPOPCOUNT(2, I, 8,  4, D)
+dnl
+dnl VMASK_TRUECOUNT($1,     $2 )
+dnl VMASK_TRUECOUNT(suffix, reg)
+define(`VMASK_TRUECOUNT', `
+instruct vmask_truecount$1(iRegINoSp dst, $2 src, $2 tmp) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskTrueCount src));
+  effect(TEMP tmp);
+  ins_cost(2 * INSN_COST);
+  format %{ "addv $tmp, $src\n\t"
+            "umov $dst, $tmp, B, 0\t# vector ($1)" %}
+  ins_encode %{
+    // Input "src" is a vector of boolean represented as bytes with
+    // 0x00/0x01 as element values.
+    __ addv(as_FloatRegister($tmp$$reg), __ T$1, as_FloatRegister($src$$reg));
+    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
+  %}
+  ins_pipe(pipe_slow);
+%}')dnl
+dnl
+dnl
+define(`ARGLIST',
+`ifelse($1, `_LT8B', `iRegINoSp dst, vecD src, rFlagsReg cr', `iRegINoSp dst, vecD src')')
+dnl
+dnl VMASK_FIRSTTRUE_D($1,     $2,   $3,   $4  )
+dnl VMASK_FIRSTTRUE_D(suffix, cond, cost, size)
+define(`VMASK_FIRSTTRUE_D', `
+instruct vmask_firsttrue$1(ARGLIST($1)) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN &&
+            n->in(1)->bottom_type()->is_vect()->length() $2 8);
+  match(Set dst (VectorMaskFirstTrue src));dnl
+ifelse($1, `_LT8B', `
+  effect(KILL cr);')
+  ins_cost($3 * INSN_COST);
+  format %{ "vmask_firsttrue $dst, $src\t# vector ($4)" %}
+  ins_encode %{
+    // Returns the index of the first active lane of the
+    // vector mask, or VLENGTH if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+    //
+    // Computed by reversing the bits and counting the leading
+    // zero bytes.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ rbit($dst$$Register, $dst$$Register);
+    __ clz($dst$$Register, $dst$$Register);
+    __ lsrw($dst$$Register, $dst$$Register, 3);dnl
+ifelse(`$1', `_LT8B', `
+    __ movw(rscratch1, vector_length(this, $src));
+    __ cmpw($dst$$Register, rscratch1);
+    __ cselw($dst$$Register, rscratch1, $dst$$Register, Assembler::GE);')
+  %}
+  ins_pipe(pipe_slow);
+%}')dnl
+dnl
+undefine(ARGLIST)dnl
+dnl
+// vector mask reductions
+VMASK_TRUECOUNT(8B,  vecD)
+VMASK_TRUECOUNT(16B, vecX)
+VMASK_FIRSTTRUE_D(_LT8B, <,  7, 4I/4S/2I)
+VMASK_FIRSTTRUE_D(8B,    ==, 4, 8B)
+
+instruct vmask_firsttrue16B(iRegINoSp dst, vecX src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskFirstTrue src));
+  ins_cost(6 * INSN_COST);
+  format %{ "vmask_firsttrue $dst, $src\t# vector (16B)" %}
+  ins_encode %{
+    // Returns the index of the first active lane of the
+    // vector mask, or 16 (VLENGTH) if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+
+    Label FIRST_TRUE_INDEX;
+
+    // Try to compute the result from lower 64 bits.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, zr);
+    __ cbnz($dst$$Register, FIRST_TRUE_INDEX);
+
+    // Compute the result from the higher 64 bits.
+    __ fmovhid($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 8);
+
+    // Reverse the bits and count the leading zero bytes.
+    __ bind(FIRST_TRUE_INDEX);
+    __ rbit($dst$$Register, $dst$$Register);
+    __ clz($dst$$Register, $dst$$Register);
+    __ addw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_lasttrue8B(iRegINoSp dst, vecD src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskLastTrue src));
+  ins_cost(4 * INSN_COST);
+  format %{ "vmask_lasttrue $dst, $src\t# vector (8B)" %}
+  ins_encode %{
+    // Returns the index of the last active lane of the
+    // vector mask, or -1 if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+    //
+    // Computed by counting the leading zero bytes and
+    // substracting it by 7 (VLENGTH - 1).
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ clz($dst$$Register, $dst$$Register);
+    __ movw(rscratch1, 7);
+    __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_lasttrue16B(iRegINoSp dst, vecX src) %{
+  predicate(n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
+  match(Set dst (VectorMaskLastTrue src));
+  ins_cost(5 * INSN_COST);
+  format %{ "vmask_lasttrue $dst, $src\t# vector (16B)" %}
+  ins_encode %{
+    // Returns the index of the last active lane of the
+    // vector mask, or -1 if no lane is active.
+    //
+    // Input "src" is a vector of boolean represented as
+    // bytes with 0x00/0x01 as element values.
+
+    Label LAST_TRUE_INDEX;
+
+    // Try to compute the result from higher 64 bits.
+    __ fmovhid($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 16 - 1);
+    __ cbnz($dst$$Register, LAST_TRUE_INDEX);
+
+    // Compute the result from the lower 64 bits.
+    __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
+    __ movw(rscratch1, 8 - 1);
+
+    // Count the leading zero bytes and substract it by 15 (VLENGTH - 1).
+    __ bind(LAST_TRUE_INDEX);
+    __ clz($dst$$Register, $dst$$Register);
+    __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
+  %}
+  ins_pipe(pipe_slow);
+%}
diff --git a/src/hotspot/cpu/aarch64/aarch64_sve.ad b/src/hotspot/cpu/aarch64/aarch64_sve.ad
index bcd38acdcb7..843cf6cb919 100644
--- a/src/hotspot/cpu/aarch64/aarch64_sve.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_sve.ad
@@ -87,18 +87,6 @@ source_hpp %{
 %}
 
 source %{
-  static inline BasicType vector_element_basic_type(const MachNode* n) {
-    const TypeVect* vt = n->bottom_type()->is_vect();
-    return vt->element_basic_type();
-  }
-
-  static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) {
-    int def_idx = use->operand_index(opnd);
-    Node* def = use->in(def_idx);
-    const TypeVect* vt = def->bottom_type()->is_vect();
-    return vt->element_basic_type();
-  }
-
   static Assembler::SIMD_RegVariant elemBytes_to_regVariant(int esize) {
     switch(esize) {
       case 1:
@@ -203,6 +191,9 @@ source %{
       case Op_VectorReinterpret:
       case Op_VectorStoreMask:
       case Op_VectorTest:
+      case Op_VectorMaskTrueCount:
+      case Op_VectorMaskLastTrue:
+      case Op_VectorMaskFirstTrue:
         return false;
       default:
         return true;
@@ -228,7 +219,7 @@ instruct loadV(vReg dst, vmemA mem) %{
   ins_encode %{
     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
     loadStoreA_predicate(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
-                         vector_element_basic_type(this), $mem->opcode(),
+                         Matcher::vector_element_basic_type(this), $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
   ins_pipe(pipe_slow);
@@ -242,7 +233,7 @@ instruct storeV(vReg src, vmemA mem) %{
   ins_encode %{
     FloatRegister src_reg = as_FloatRegister($src$$reg);
     loadStoreA_predicate(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
-                         vector_element_basic_type(this, $src), $mem->opcode(),
+                         Matcher::vector_element_basic_type(this, $src), $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
   %}
   ins_pipe(pipe_slow);
@@ -547,7 +538,7 @@ instruct vmin(vReg dst_src1, vReg src2) %{
   ins_cost(SVE_COST);
   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
   ins_encode %{
-    BasicType bt = vector_element_basic_type(this);
+    BasicType bt = Matcher::vector_element_basic_type(this);
     Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
     if (is_floating_point_type(bt)) {
       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
@@ -567,7 +558,7 @@ instruct vmax(vReg dst_src1, vReg src2) %{
   ins_cost(SVE_COST);
   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
   ins_encode %{
-    BasicType bt = vector_element_basic_type(this);
+    BasicType bt = Matcher::vector_element_basic_type(this);
     Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
     if (is_floating_point_type(bt)) {
       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
@@ -1775,3 +1766,42 @@ instruct vmaskcast(vReg dst) %{
   ins_pipe(pipe_class_empty);
 %}
 
+// Intrisics for String.indexOf(char)
+
+
+instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
+                                  iRegI_R0 result, vReg ztmp1, vReg ztmp2,
+                                  pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
+%{
+  match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
+  predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
+  effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
+
+  format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
+
+  ins_encode %{
+    __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
+                               as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
+                               as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
+  %}
+  ins_pipe(pipe_class_memory);
+%}
+
+instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
+                                  iRegI_R0 result, vReg ztmp1, vReg ztmp2,
+                                  pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
+%{
+  match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
+  predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
+  effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
+
+  format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
+
+  ins_encode %{
+    __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
+                               as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
+                               as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
+  %}
+  ins_pipe(pipe_class_memory);
+%}
+
diff --git a/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
index 28a8a64a6f6..bc166185570 100644
--- a/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
@@ -74,18 +74,6 @@ source_hpp %{
 %}
 
 source %{
-  static inline BasicType vector_element_basic_type(const MachNode* n) {
-    const TypeVect* vt = n->bottom_type()->is_vect();
-    return vt->element_basic_type();
-  }
-
-  static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) {
-    int def_idx = use->operand_index(opnd);
-    Node* def = use->in(def_idx);
-    const TypeVect* vt = def->bottom_type()->is_vect();
-    return vt->element_basic_type();
-  }
-
   static Assembler::SIMD_RegVariant elemBytes_to_regVariant(int esize) {
     switch(esize) {
       case 1:
@@ -190,6 +178,9 @@ source %{
       case Op_VectorReinterpret:
       case Op_VectorStoreMask:
       case Op_VectorTest:
+      case Op_VectorMaskTrueCount:
+      case Op_VectorMaskLastTrue:
+      case Op_VectorMaskFirstTrue:
         return false;
       default:
         return true;
@@ -919,3 +910,29 @@ instruct vmaskcast(vReg dst) %{
   ins_pipe(pipe_class_empty);
 %}
 
+// Intrisics for String.indexOf(char)
+
+dnl
+define(`STRING_INDEXOF_CHAR', `
+instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
+                                  iRegI_R0 result, vReg ztmp1, vReg ztmp2,
+                                  pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
+%{
+  match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
+  predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
+  effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
+
+  format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
+
+  ins_encode %{
+    __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
+                               as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
+                               as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
+  %}
+  ins_pipe(pipe_class_memory);
+%}')dnl
+dnl                 $1 $2      $3
+STRING_INDEXOF_CHAR(L, Latin1, true)
+STRING_INDEXOF_CHAR(U, UTF16,  false)
+dnl
+
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
index f9974680d2e..79a040dc0f7 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
@@ -134,7 +134,16 @@ void Assembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset)
 
 #undef __
 
-#define starti Instruction_aarch64 do_not_use(this); set_current(&do_not_use)
+#define starti Instruction_aarch64 current_insn(this);
+
+#define f current_insn.f
+#define sf current_insn.sf
+#define rf current_insn.rf
+#define srf current_insn.srf
+#define zrf current_insn.zrf
+#define prf current_insn.prf
+#define pgrf current_insn.pgrf
+#define fixed current_insn.fixed
 
   void Assembler::adr(Register Rd, address adr) {
     intptr_t offset = adr - pc();
@@ -156,6 +165,53 @@ void Assembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset)
     rf(Rd, 0);
   }
 
+// An "all-purpose" add/subtract immediate, per ARM documentation:
+// A "programmer-friendly" assembler may accept a negative immediate
+// between -(2^24 -1) and -1 inclusive, causing it to convert a
+// requested ADD operation to a SUB, or vice versa, and then encode
+// the absolute value of the immediate as for uimm24.
+void Assembler::add_sub_immediate(Instruction_aarch64 &current_insn,
+                                  Register Rd, Register Rn, unsigned uimm, int op,
+                                  int negated_op) {
+  bool sets_flags = op & 1;   // this op sets flags
+  union {
+    unsigned u;
+    int imm;
+  };
+  u = uimm;
+  bool shift = false;
+  bool neg = imm < 0;
+  if (neg) {
+    imm = -imm;
+    op = negated_op;
+  }
+  assert(Rd != sp || imm % 16 == 0, "misaligned stack");
+  if (imm >= (1 << 11)
+      && ((imm >> 12) << 12 == imm)) {
+    imm >>= 12;
+    shift = true;
+  }
+  f(op, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10);
+
+  // add/subtract immediate ops with the S bit set treat r31 as zr;
+  // with S unset they use sp.
+  if (sets_flags)
+    zrf(Rd, 0);
+  else
+    srf(Rd, 0);
+
+  srf(Rn, 5);
+}
+
+#undef f
+#undef sf
+#undef rf
+#undef srf
+#undef zrf
+#undef prf
+#undef pgrf
+#undef fixed
+
 #undef starti
 
 Address::Address(address target, relocInfo::relocType rtype) : _mode(literal){
@@ -260,43 +316,6 @@ void Assembler::wrap_label(Label &L, prfop op, prefetch_insn insn) {
   }
 }
 
-// An "all-purpose" add/subtract immediate, per ARM documentation:
-// A "programmer-friendly" assembler may accept a negative immediate
-// between -(2^24 -1) and -1 inclusive, causing it to convert a
-// requested ADD operation to a SUB, or vice versa, and then encode
-// the absolute value of the immediate as for uimm24.
-void Assembler::add_sub_immediate(Register Rd, Register Rn, unsigned uimm, int op,
-                                  int negated_op) {
-  bool sets_flags = op & 1;   // this op sets flags
-  union {
-    unsigned u;
-    int imm;
-  };
-  u = uimm;
-  bool shift = false;
-  bool neg = imm < 0;
-  if (neg) {
-    imm = -imm;
-    op = negated_op;
-  }
-  assert(Rd != sp || imm % 16 == 0, "misaligned stack");
-  if (imm >= (1 << 11)
-      && ((imm >> 12) << 12 == imm)) {
-    imm >>= 12;
-    shift = true;
-  }
-  f(op, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10);
-
-  // add/subtract immediate ops with the S bit set treat r31 as zr;
-  // with S unset they use sp.
-  if (sets_flags)
-    zrf(Rd, 0);
-  else
-    srf(Rd, 0);
-
-  srf(Rn, 5);
-}
-
 bool Assembler::operand_valid_for_add_sub_immediate(int64_t imm) {
   bool shift = false;
   uint64_t uimm = (uint64_t)uabs((jlong)imm);
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 78e8fc370ad..1405543b3d3 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -247,12 +247,12 @@ class Instruction_aarch64 {
     int nbits = msb - lsb + 1;
     guarantee(val < (1ULL << nbits), "Field too big for insn");
     assert_cond(msb >= lsb);
-    unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
     val <<= lsb;
-    mask <<= lsb;
     insn |= val;
-    assert_cond((bits & mask) == 0);
 #ifdef ASSERT
+    unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
+    mask <<= lsb;
+    assert_cond((bits & mask) == 0);
     bits |= mask;
 #endif
   }
@@ -313,7 +313,7 @@ class Instruction_aarch64 {
   }
 };
 
-#define starti Instruction_aarch64 do_not_use(this); set_current(&do_not_use)
+#define starti Instruction_aarch64 current_insn(this);
 
 class PrePost {
   int _offset;
@@ -694,46 +694,14 @@ class Assembler : public AbstractAssembler {
 
   static address locate_next_instruction(address inst);
 
-  Instruction_aarch64* current;
-
-  void set_current(Instruction_aarch64* i) { current = i; }
-
-  void f(unsigned val, int msb, int lsb) {
-    current->f(val, msb, lsb);
-  }
-  void f(unsigned val, int msb) {
-    current->f(val, msb, msb);
-  }
-  void sf(int64_t val, int msb, int lsb) {
-    current->sf(val, msb, lsb);
-  }
-  void rf(Register reg, int lsb) {
-    current->rf(reg, lsb);
-  }
-  void srf(Register reg, int lsb) {
-    current->srf(reg, lsb);
-  }
-  void zrf(Register reg, int lsb) {
-    current->zrf(reg, lsb);
-  }
-  void rf(FloatRegister reg, int lsb) {
-    current->rf(reg, lsb);
-  }
-  void prf(PRegister reg, int lsb) {
-    current->prf(reg, lsb);
-  }
-  void pgrf(PRegister reg, int lsb) {
-    current->pgrf(reg, lsb);
-  }
-  void fixed(unsigned value, unsigned mask) {
-    current->fixed(value, mask);
-  }
-
-  void emit() {
-    emit_long(current->get_insn());
-    assert_cond(current->get_bits() == 0xffffffff);
-    current = NULL;
-  }
+#define f current_insn.f
+#define sf current_insn.sf
+#define rf current_insn.rf
+#define srf current_insn.srf
+#define zrf current_insn.zrf
+#define prf current_insn.prf
+#define pgrf current_insn.pgrf
+#define fixed current_insn.fixed
 
   typedef void (Assembler::* uncond_branch_insn)(address dest);
   typedef void (Assembler::* compare_and_branch_insn)(Register Rt, address dest);
@@ -764,8 +732,8 @@ class Assembler : public AbstractAssembler {
 
 #undef INSN
 
-  void add_sub_immediate(Register Rd, Register Rn, unsigned uimm, int op,
-                         int negated_op);
+  void add_sub_immediate(Instruction_aarch64 &current_insn, Register Rd, Register Rn,
+                         unsigned uimm, int op, int negated_op);
 
   // Add/subtract (immediate)
 #define INSN(NAME, decode, negated)                                     \
@@ -777,7 +745,7 @@ class Assembler : public AbstractAssembler {
                                                                         \
   void NAME(Register Rd, Register Rn, unsigned imm) {                   \
     starti;                                                             \
-    add_sub_immediate(Rd, Rn, imm, decode, negated);                    \
+    add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);      \
   }
 
   INSN(addsw, 0b001, 0b011);
@@ -790,7 +758,7 @@ class Assembler : public AbstractAssembler {
 #define INSN(NAME, decode, negated)                     \
   void NAME(Register Rd, Register Rn, unsigned imm) {   \
     starti;                                             \
-    add_sub_immediate(Rd, Rn, imm, decode, negated);    \
+    add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);     \
   }
 
   INSN(addw, 0b000, 0b010);
@@ -1092,7 +1060,7 @@ class Assembler : public AbstractAssembler {
   }
 
   void sys(int op1, int CRn, int CRm, int op2,
-           Register rt = (Register)0b11111) {
+           Register rt = as_Register(0b11111)) {
     system(0b01, op1, CRn, CRm, op2, rt);
   }
 
@@ -1361,7 +1329,7 @@ class Assembler : public AbstractAssembler {
     starti;                                                             \
     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
       sf(offset, 23, 5);                                                \
-    rf((Register)Rt, 0);                                                \
+    rf(as_Register(Rt), 0);                                             \
   }
 
   INSN(ldrs, 0b00, 1);
@@ -1375,7 +1343,7 @@ class Assembler : public AbstractAssembler {
     starti;                                                             \
     f(size, 31, 30), f(0b111100, 29, 24), f(opc, 23, 22), f(0, 21);     \
     f(0, 20, 12), f(0b01, 11, 10);                                      \
-    rf(Rn, 5), rf((Register)Rt, 0);                                     \
+    rf(Rn, 5), rf(as_Register(Rt), 0);                                  \
   }
 
   INSN(ldrs, 0b10, 0b01);
@@ -1408,9 +1376,9 @@ class Assembler : public AbstractAssembler {
     f(opc, 31, 30), f(p1, 29, 27), f(V, 26), f(L, 22);
     zrf(Rt2, 10), zrf(Rt1, 0);
     if (no_allocate) {
-      adr.encode_nontemporal_pair(current);
+      adr.encode_nontemporal_pair(&current_insn);
     } else {
-      adr.encode_pair(current);
+      adr.encode_pair(&current_insn);
     }
   }
 
@@ -1436,7 +1404,8 @@ class Assembler : public AbstractAssembler {
 
 #define INSN(NAME, size, p1, V, L, no_allocate)                         \
   void NAME(FloatRegister Rt1, FloatRegister Rt2, Address adr) {        \
-    ld_st1(size, p1, V, L, (Register)Rt1, (Register)Rt2, adr, no_allocate); \
+    ld_st1(size, p1, V, L,                                              \
+           as_Register(Rt1), as_Register(Rt2), adr, no_allocate);       \
    }
 
   INSN(stps, 0b00, 0b101, 1, 0, false);
@@ -1471,7 +1440,7 @@ class Assembler : public AbstractAssembler {
 
     f(size, 31, 30);
     f(op, 23, 22); // str
-    adr.encode(current);
+    adr.encode(&current_insn);
   }
 
 #define INSN(NAME, size, op)                            \
@@ -1499,7 +1468,7 @@ class Assembler : public AbstractAssembler {
 
 #define INSN(NAME, size, op)                                    \
   void NAME(const Address &adr, prfop pfop = PLDL1KEEP) {       \
-    ld_st2((Register)pfop, adr, size, op);                      \
+    ld_st2(as_Register(pfop), adr, size, op);                   \
   }
 
   INSN(prfm, 0b11, 0b10); // FIXME: PRFM should not be used with
@@ -1510,7 +1479,7 @@ class Assembler : public AbstractAssembler {
 
 #define INSN(NAME, size, op)                            \
   void NAME(FloatRegister Rt, const Address &adr) {     \
-    ld_st2((Register)Rt, adr, size, op, 1);             \
+    ld_st2(as_Register(Rt), adr, size, op, 1);          \
   }
 
   INSN(strd, 0b11, 0b00);
@@ -1547,7 +1516,7 @@ class Assembler : public AbstractAssembler {
 
   enum shift_kind { LSL, LSR, ASR, ROR };
 
-  void op_shifted_reg(unsigned decode,
+  void op_shifted_reg(Instruction_aarch64 &current_insn, unsigned decode,
                       enum shift_kind kind, unsigned shift,
                       unsigned size, unsigned op) {
     f(size, 31);
@@ -1558,14 +1527,14 @@ class Assembler : public AbstractAssembler {
   }
 
   // Logical (shifted register)
-#define INSN(NAME, size, op, N)                                 \
-  void NAME(Register Rd, Register Rn, Register Rm,              \
-            enum shift_kind kind = LSL, unsigned shift = 0) {   \
-    starti;                                                     \
-    guarantee(size == 1 || shift < 32, "incorrect shift");      \
-    f(N, 21);                                                   \
-    zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                        \
-    op_shifted_reg(0b01010, kind, shift, size, op);             \
+#define INSN(NAME, size, op, N)                                         \
+  void NAME(Register Rd, Register Rn, Register Rm,                      \
+            enum shift_kind kind = LSL, unsigned shift = 0) {           \
+    starti;                                                             \
+    guarantee(size == 1 || shift < 32, "incorrect shift");              \
+    f(N, 21);                                                           \
+    zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
+    op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
   }
 
   INSN(andr, 1, 0b00, 0);
@@ -1585,7 +1554,7 @@ class Assembler : public AbstractAssembler {
     starti;                                                             \
     f(N, 21);                                                           \
     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
-    op_shifted_reg(0b01010, kind, shift, size, op);                     \
+    op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
   }                                                                     \
                                                                         \
   /* These instructions have no immediate form. Provide an overload so  \
@@ -1632,7 +1601,7 @@ void mvnw(Register Rd, Register Rm,
     assert_cond(kind != ROR);                           \
     guarantee(size == 1 || shift < 32, "incorrect shift");\
     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
-    op_shifted_reg(0b01011, kind, shift, size, op);     \
+    op_shifted_reg(current_insn, 0b01011, kind, shift, size, op);      \
   }
 
   INSN(add, 1, 0b000);
@@ -1653,10 +1622,10 @@ void mvnw(Register Rd, Register Rm,
            ext::operation option, int amount = 0) {                     \
     starti;                                                             \
     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
-    add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
+    add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
   }
 
-  void add_sub_extended_reg(unsigned op, unsigned decode,
+  void add_sub_extended_reg(Instruction_aarch64 &current_insn, unsigned op, unsigned decode,
     Register Rd, Register Rn, Register Rm,
     unsigned opt, ext::operation option, unsigned imm) {
     guarantee(imm <= 4, "shift amount must be <= 4");
@@ -1676,7 +1645,7 @@ void mvnw(Register Rd, Register Rm,
            ext::operation option, int amount = 0) {                     \
     starti;                                                             \
     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
-    add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
+    add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
   }
 
   INSN(addsw, 0b001);
@@ -1777,7 +1746,7 @@ void mvnw(Register Rd, Register Rm,
   }
 
 #define INSN(NAME, op, op2)                                             \
-  void NAME(Register Rd, Register Rn, Register Rm, Condition cond) { \
+  void NAME(Register Rd, Register Rn, Register Rm, Condition cond) {    \
     conditional_select(op, op2, Rd, Rn, Rm, cond);                      \
   }
 
@@ -1793,7 +1762,7 @@ void mvnw(Register Rd, Register Rm,
 #undef INSN
 
   // Data processing
-  void data_processing(unsigned op29, unsigned opcode,
+  void data_processing(Instruction_aarch64 &current_insn, unsigned op29, unsigned opcode,
                        Register Rd, Register Rn) {
     f(op29, 31, 29), f(0b11010110, 28, 21);
     f(opcode, 15, 10);
@@ -1801,11 +1770,11 @@ void mvnw(Register Rd, Register Rm,
   }
 
   // (1 source)
-#define INSN(NAME, op29, opcode2, opcode)       \
-  void NAME(Register Rd, Register Rn) {         \
-    starti;                                     \
-    f(opcode2, 20, 16);                         \
-    data_processing(op29, opcode, Rd, Rn);      \
+#define INSN(NAME, op29, opcode2, opcode)                       \
+  void NAME(Register Rd, Register Rn) {                         \
+    starti;                                                     \
+    f(opcode2, 20, 16);                                         \
+    data_processing(current_insn, op29, opcode, Rd, Rn);        \
   }
 
   INSN(rbitw,  0b010, 0b00000, 0b00000);
@@ -1824,11 +1793,11 @@ void mvnw(Register Rd, Register Rm,
 #undef INSN
 
   // (2 sources)
-#define INSN(NAME, op29, opcode)                        \
-  void NAME(Register Rd, Register Rn, Register Rm) {    \
-    starti;                                             \
-    rf(Rm, 16);                                         \
-    data_processing(op29, opcode, Rd, Rn);              \
+#define INSN(NAME, op29, opcode)                                \
+  void NAME(Register Rd, Register Rn, Register Rm) {            \
+    starti;                                                     \
+    rf(Rm, 16);                                                 \
+    data_processing(current_insn, op29, opcode, Rd, Rn);        \
   }
 
   INSN(udivw, 0b000, 0b000010);
@@ -1873,9 +1842,9 @@ void mvnw(Register Rd, Register Rm,
 
 #undef INSN
 
-#define INSN(NAME, op54, op31, o0)                      \
-  void NAME(Register Rd, Register Rn, Register Rm) {    \
-    data_processing(op54, op31, o0, Rd, Rn, Rm, (Register)31);  \
+#define INSN(NAME, op54, op31, o0)                                      \
+  void NAME(Register Rd, Register Rn, Register Rm) {                    \
+    data_processing(op54, op31, o0, Rd, Rn, Rm, as_Register(31));       \
   }
 
   INSN(smulh, 0b100, 0b010, 0);
@@ -2054,7 +2023,7 @@ void mvnw(Register Rd, Register Rm,
 
 #define INSN(NAME, op31, type, rmode, opcode)                           \
   void NAME(Register Rd, FloatRegister Vn) {                            \
-    float_int_convert(op31, type, rmode, opcode, Rd, (Register)Vn);     \
+    float_int_convert(op31, type, rmode, opcode, Rd, as_Register(Vn));  \
   }
 
   INSN(fcvtzsw, 0b000, 0b00, 0b11, 0b000);
@@ -2065,13 +2034,13 @@ void mvnw(Register Rd, Register Rm,
   INSN(fmovs, 0b000, 0b00, 0b00, 0b110);
   INSN(fmovd, 0b100, 0b01, 0b00, 0b110);
 
-  // INSN(fmovhid, 0b100, 0b10, 0b01, 0b110);
+  INSN(fmovhid, 0b100, 0b10, 0b01, 0b110);
 
 #undef INSN
 
 #define INSN(NAME, op31, type, rmode, opcode)                           \
   void NAME(FloatRegister Vd, Register Rn) {                            \
-    float_int_convert(op31, type, rmode, opcode, (Register)Vd, Rn);     \
+    float_int_convert(op31, type, rmode, opcode, as_Register(Vd), Rn);  \
   }
 
   INSN(fmovs, 0b000, 0b00, 0b00, 0b111);
@@ -2126,7 +2095,7 @@ void mvnw(Register Rd, Register Rm,
   // Floating-point compare
   void float_compare(unsigned op31, unsigned type,
                      unsigned op, unsigned op2,
-                     FloatRegister Vn, FloatRegister Vm = (FloatRegister)0) {
+                     FloatRegister Vn, FloatRegister Vm = as_FloatRegister(0)) {
     starti;
     f(op31, 31, 29);
     f(0b11110, 28, 24);
@@ -2256,10 +2225,10 @@ void mvnw(Register Rd, Register Rm,
   static short SIMD_Size_in_bytes[];
 
 public:
-#define INSN(NAME, op)                                            \
-  void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {   \
-    ld_st2((Register)Rt, adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
-  }                                                                      \
+#define INSN(NAME, op)                                                  \
+  void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {  \
+    ld_st2(as_Register(Rt), adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
+  }
 
   INSN(ldr, 1);
   INSN(str, 0);
@@ -3245,6 +3214,70 @@ void mvnw(Register Rd, Register Rm,
     f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
   }
 
+// Integer comparisons (SVE)
+#define INSN(NAME, cond)                                                                          \
+  void NAME(PRegister Pd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm)  { \
+    starti;                                                                                       \
+    assert(T != Q, "invalid size");                                                               \
+    f(0b00100100, 31, 24), f(T, 23, 22), f(0, 21), rf(Zm, 16), f((cond >> 1) & 7, 15, 13);        \
+    pgrf(Pg, 10), rf(Zn, 5), f(cond & 1, 4), prf(Pd, 0);                                          \
+  }
+
+  INSN(sve_cmpeq, 0b1010);  // Compare signed equal to vector
+  INSN(sve_cmpne, 0b1011);  // Compare not equal to vector
+  INSN(sve_cmpge, 0b1000);  // Compare signed greater than or equal to vector
+  INSN(sve_cmpgt, 0b1001);  // Compare signed greater than vector
+#undef INSN
+
+// Predicate counted loop (SVE) (32-bit variants are not included)
+#define INSN(NAME, decode)                                                \
+  void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) {  \
+    starti;                                                               \
+    assert(T != Q, "invalid register variant");                           \
+    f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21),                        \
+    zrf(Rm, 16), f(0, 15, 13), f(1, 12), f(decode >> 1, 11, 10),          \
+    zrf(Rn, 5), f(decode & 1, 4), prf(Pd, 0);                             \
+  }
+
+  INSN(sve_whilelt, 0b010);  // While incrementing signed scalar less than scalar
+  INSN(sve_whilele, 0b011);  // While incrementing signed scalar less than or equal to scalar
+  INSN(sve_whilelo, 0b110);  // While incrementing unsigned scalar lower than scalar
+  INSN(sve_whilels, 0b111);  // While incrementing unsigned scalar lower than or the same as scalar
+#undef INSN
+
+  // Predicate scan (SVE)
+
+  // Break after the first true condition
+  void sve_brka(PRegister pd, PRegister pg, PRegister pn, bool isMerge) {
+    starti;
+    f(0b00100101, 31, 24), f(0b00, 23, 22), f(0b01000001, 21, 14),
+    prf(pg, 10), f(0b0, 9), prf(pn, 5), f(isMerge ? 1 : 0, 4), prf(pd, 0);
+  }
+
+// Element count and increment scalar (SVE)
+#define INSN(NAME, TYPE)                                                             \
+  void NAME(Register Xdn, unsigned imm4 = 1, int pattern = 0b11111) {                \
+    starti;                                                                          \
+    f(0b00000100, 31, 24), f(TYPE, 23, 22), f(0b10, 21, 20);                         \
+    f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(0, 10), f(pattern, 9, 5), rf(Xdn, 0); \
+  }
+
+  INSN(sve_cntb, B);  // Set scalar to multiple of 8-bit predicate constraint element count
+  INSN(sve_cnth, H);  // Set scalar to multiple of 16-bit predicate constraint element count
+  INSN(sve_cntw, S);  // Set scalar to multiple of 32-bit predicate constraint element count
+  INSN(sve_cntd, D);  // Set scalar to multiple of 64-bit predicate constraint element count
+#undef INSN
+
+  // Predicate count and increment scalar (SVE)
+
+  // Set scalar to the number of Active predicate elements that are TRUE
+  void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
+    starti;
+    assert(T != Q, "invalid size");
+    f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
+    prf(pg, 5), rf(rd, 0);
+  }
+
   Assembler(CodeBuffer* code) : AbstractAssembler(code) {
   }
 
@@ -3265,9 +3298,19 @@ inline Assembler::Membar_mask_bits operator|(Assembler::Membar_mask_bits a,
 }
 
 Instruction_aarch64::~Instruction_aarch64() {
-  assem->emit();
+  assem->emit_int32(insn);
+  assert_cond(get_bits() == 0xffffffff);
 }
 
+#undef f
+#undef sf
+#undef rf
+#undef srf
+#undef zrf
+#undef prf
+#undef pgrf
+#undef fixed
+
 #undef starti
 
 // Invert a condition
@@ -3275,8 +3318,6 @@ inline const Assembler::Condition operator~(const Assembler::Condition cond) {
   return Assembler::Condition(int(cond) ^ 1);
 }
 
-class BiasedLockingCounters;
-
 extern "C" void das(uint64_t start, int len);
 
 #endif // CPU_AARCH64_ASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
index ac12ba9e23d..6f9425e43ac 100644
--- a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
@@ -45,5 +45,9 @@ extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_relaxed_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_4_relaxed_impl;
 extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_relaxed_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_4_release_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_release_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_4_seq_cst_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_seq_cst_impl;
 
 #endif // CPU_AARCH64_ATOMIC_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
index cccfaca5c8c..1d189cf3435 100644
--- a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
@@ -449,8 +449,10 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->add_call_info_here(info());
 
 #ifndef PRODUCT
-  __ lea(rscratch2, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
-  __ incrementw(Address(rscratch2));
+  if (PrintC1Statistics) {
+    __ lea(rscratch2, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+    __ incrementw(Address(rscratch2));
+  }
 #endif
 
   __ b(_continuation);
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index b77439b7b95..f2b0a39b1d1 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -811,7 +811,7 @@ void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool po
 }
 
 
-void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
+void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
   LIR_Address* to_addr = dest->as_address_ptr();
   PatchingStub* patch = NULL;
   Register compressed_src = rscratch1;
@@ -969,7 +969,7 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 }
 
 
-void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
   LIR_Address* addr = src->as_address_ptr();
   LIR_Address* from_addr = src->as_address_ptr();
 
@@ -2789,13 +2789,9 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   if (!UseFastLocking) {
     __ b(*op->stub()->entry());
   } else if (op->code() == lir_lock) {
-    Register scratch = noreg;
-    if (UseBiasedLocking) {
-      scratch = op->scratch_opr()->as_register();
-    }
     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // add debug info for NullPointerException only if one is possible
-    int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
+    int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
     if (op->info() != NULL) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }
@@ -3049,7 +3045,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
         }
 #endif
         // first time here. Set profile type.
-        __ ldr(tmp, mdo_addr);
+        __ str(tmp, mdo_addr);
       } else {
         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
@@ -3143,7 +3139,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
   if (dest->is_address() || src->is_address()) {
     move_op(src, dest, type, lir_patch_none, info,
-            /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
+            /*pop_fpu_stack*/false, /*wide*/false);
   } else {
     ShouldNotReachHere();
   }
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index 67b5e27f3cc..517d41bbfc6 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -338,9 +338,9 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
 
   // "lock" stores the address of the monitor stack slot, so this is not an oop
   LIR_Opr lock = new_register(T_INT);
-  // Need a scratch register for biased locking
+  // Need a scratch register for inline type
   LIR_Opr scratch = LIR_OprFact::illegalOpr;
-  if (UseBiasedLocking || x->maybe_inlinetype()) {
+  if (EnableValhalla && x->maybe_inlinetype()) {
     scratch = new_register(T_INT);
   }
 
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index f4ba39d2f6b..f0a24fa2ee0 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -35,7 +35,6 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
@@ -63,7 +62,7 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
   }
 }
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
@@ -84,18 +83,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     br(Assembler::NE, slow_case);
   }
 
-  if (UseBiasedLocking) {
-    assert(scratch != noreg, "should have scratch register at this point");
-    biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
-  }
-
   // Load object header
   ldr(hdr, Address(obj, hdr_offset));
   // and mark it as unlocked
   orr(hdr, hdr, markWord::unlocked_value);
 
   if (EnableValhalla) {
-    assert(!UseBiasedLocking, "Not compatible with biased-locking");
     // Mask always_locked bit such that we go to the slow path if object is an inline type
     andr(hdr, hdr, ~markWord::inline_type_bit_in_place);
   }
@@ -131,10 +124,6 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
   cbnz(hdr, slow_case);
   // done
   bind(done);
-  if (PrintBiasedLockingStatistics) {
-    lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
-    addmw(Address(rscratch2, 0), 1, rscratch1);
-  }
   return null_check_offset;
 }
 
@@ -145,21 +134,13 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
-  if (UseBiasedLocking) {
-    // load object
-    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-    biased_locking_exit(obj, hdr, done);
-  }
-
   // load displaced header
   ldr(hdr, Address(disp_hdr, 0));
   // if the loaded hdr is NULL we had recursive locking
   // if we had recursive locking, we are done
   cbz(hdr, done);
-  if (!UseBiasedLocking) {
-    // load object
-    ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-  }
+  // load object
+  ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
   verify_oop(obj);
   // test if object header is pointing to the displaced header, and if so, restore
   // the displaced header in the object - if the object header is not pointing to
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
index 7f47181342a..95c2e1a914b 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
@@ -58,9 +58,8 @@ using MacroAssembler::null_check;
   // hdr     : must be r0, contents destroyed
   // obj     : must point to the object to lock, contents preserved
   // disp_hdr: must point to the displaced header location, contents preserved
-  // scratch : scratch register, contents destroyed
   // returns code offset at which to add null check debug information
-  int lock_object  (Register swap, Register obj, Register disp_hdr, Register scratch, Label& slow_case);
+  int lock_object  (Register swap, Register obj, Register disp_hdr, Label& slow_case);
 
   // unlocking
   // hdr     : contents destroyed
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index fab8ff669b1..46c1145654d 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 #include "asm/assembler.inline.hpp"
 #include "opto/c2_MacroAssembler.hpp"
 #include "opto/intrinsicnode.hpp"
+#include "opto/subnode.hpp"
 #include "runtime/stubRoutines.hpp"
 
 #ifdef PRODUCT
@@ -539,6 +540,75 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
   BIND(DONE);
 }
 
+void C2_MacroAssembler::string_indexof_char_sve(Register str1, Register cnt1,
+                                                Register ch, Register result,
+                                                FloatRegister ztmp1,
+                                                FloatRegister ztmp2,
+                                                PRegister tmp_pg,
+                                                PRegister tmp_pdn, bool isL)
+{
+  // Note that `tmp_pdn` should *NOT* be used as governing predicate register.
+  assert(tmp_pg->is_governing(),
+         "this register has to be a governing predicate register");
+
+  Label LOOP, MATCH, DONE, NOMATCH;
+  Register vec_len = rscratch1;
+  Register idx = rscratch2;
+
+  SIMD_RegVariant T = (isL == true) ? B : H;
+
+  cbz(cnt1, NOMATCH);
+
+  // Assign the particular char throughout the vector.
+  sve_dup(ztmp2, T, ch);
+  if (isL) {
+    sve_cntb(vec_len);
+  } else {
+    sve_cnth(vec_len);
+  }
+  mov(idx, 0);
+
+  // Generate a predicate to control the reading of input string.
+  sve_whilelt(tmp_pg, T, idx, cnt1);
+
+  BIND(LOOP);
+    // Read a vector of 8- or 16-bit data depending on the string type. Note
+    // that inactive elements indicated by the predicate register won't cause
+    // a data read from memory to the destination vector.
+    if (isL) {
+      sve_ld1b(ztmp1, T, tmp_pg, Address(str1, idx));
+    } else {
+      sve_ld1h(ztmp1, T, tmp_pg, Address(str1, idx, Address::lsl(1)));
+    }
+    add(idx, idx, vec_len);
+
+    // Perform the comparison. An element of the destination predicate is set
+    // to active if the particular char is matched.
+    sve_cmpeq(tmp_pdn, T, tmp_pg, ztmp1, ztmp2);
+
+    // Branch if the particular char is found.
+    br(NE, MATCH);
+
+    sve_whilelt(tmp_pg, T, idx, cnt1);
+
+    // Loop back if the particular char not found.
+    br(MI, LOOP);
+
+  BIND(NOMATCH);
+    mov(result, -1);
+    b(DONE);
+
+  BIND(MATCH);
+    // Undo the index increment.
+    sub(idx, idx, vec_len);
+
+    // Crop the vector to find its location.
+    sve_brka(tmp_pdn, tmp_pg, tmp_pdn, false /* isMerge */);
+    add(result, idx, -1);
+    sve_incp(result, T, tmp_pdn);
+  BIND(DONE);
+}
+
 void C2_MacroAssembler::stringL_indexof_char(Register str1, Register cnt1,
                                             Register ch, Register result,
                                             Register tmp1, Register tmp2, Register tmp3)
@@ -832,3 +902,45 @@ void C2_MacroAssembler::string_compare(Register str1, Register str2,
 
   BLOCK_COMMENT("} string_compare");
 }
+
+void C2_MacroAssembler::neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
+                                     FloatRegister src2, int cond, bool isQ) {
+  SIMD_Arrangement size = esize2arrangement(type2aelembytes(bt), isQ);
+  if (bt == T_FLOAT || bt == T_DOUBLE) {
+    switch (cond) {
+      case BoolTest::eq: fcmeq(dst, size, src1, src2); break;
+      case BoolTest::ne: {
+        fcmeq(dst, size, src1, src2);
+        notr(dst, T16B, dst);
+        break;
+      }
+      case BoolTest::ge: fcmge(dst, size, src1, src2); break;
+      case BoolTest::gt: fcmgt(dst, size, src1, src2); break;
+      case BoolTest::le: fcmge(dst, size, src2, src1); break;
+      case BoolTest::lt: fcmgt(dst, size, src2, src1); break;
+      default:
+        assert(false, "unsupported");
+        ShouldNotReachHere();
+    }
+  } else {
+    switch (cond) {
+      case BoolTest::eq: cmeq(dst, size, src1, src2); break;
+      case BoolTest::ne: {
+        cmeq(dst, size, src1, src2);
+        notr(dst, T16B, dst);
+        break;
+      }
+      case BoolTest::ge: cmge(dst, size, src1, src2); break;
+      case BoolTest::gt: cmgt(dst, size, src1, src2); break;
+      case BoolTest::le: cmge(dst, size, src2, src1); break;
+      case BoolTest::lt: cmgt(dst, size, src2, src1); break;
+      case BoolTest::uge: cmhs(dst, size, src1, src2); break;
+      case BoolTest::ugt: cmhi(dst, size, src1, src2); break;
+      case BoolTest::ult: cmhi(dst, size, src2, src1); break;
+      case BoolTest::ule: cmhs(dst, size, src2, src1); break;
+      default:
+        assert(false, "unsupported");
+        ShouldNotReachHere();
+    }
+  }
+}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index b2f6226bf9e..2b550df074a 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,7 +46,16 @@
                            Register tmp1, Register tmp2, Register tmp3);
 
   void stringL_indexof_char(Register str1, Register cnt1,
-                           Register ch, Register result,
-                           Register tmp1, Register tmp2, Register tmp3);
+                            Register ch, Register result,
+                            Register tmp1, Register tmp2, Register tmp3);
+
+  void string_indexof_char_sve(Register str1, Register cnt1,
+                               Register ch, Register result,
+                               FloatRegister ztmp1, FloatRegister ztmp2,
+                               PRegister pgtmp, PRegister ptmp, bool isL);
+
+  // SIMD&FP comparison
+  void neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
+                    FloatRegister src2, int cond, bool isQ);
 
 #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
index f15b6faa79d..34e6e688abb 100644
--- a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         3);
-define_pd_global(intx, FLOATPRESSURE,                32);
 define_pd_global(intx, FreqInlineSize,               325);
 define_pd_global(intx, MinJumpTableSize,             10);
-define_pd_global(intx, INTPRESSURE,                  24);
 define_pd_global(intx, InteriorEntryAlignment,       16);
 define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
 define_pd_global(intx, LoopUnrollLimit,              60);
diff --git a/src/hotspot/cpu/aarch64/foreign_globals_aarch64.cpp b/src/hotspot/cpu/aarch64/foreign_globals_aarch64.cpp
index 6531eb03edc..d08afc79a52 100644
--- a/src/hotspot/cpu/aarch64/foreign_globals_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/foreign_globals_aarch64.cpp
@@ -45,17 +45,18 @@ bool ABIDescriptor::is_volatile_reg(FloatRegister reg) const {
 const ABIDescriptor ForeignGlobals::parse_abi_descriptor_impl(jobject jabi) const {
   oop abi_oop = JNIHandles::resolve_non_null(jabi);
   ABIDescriptor abi;
+  const Register (*to_Register)(int) = as_Register;
 
   objArrayOop inputStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.inputStorage_offset));
-  loadArray(inputStorage, INTEGER_TYPE, abi._integer_argument_registers, as_Register);
+  loadArray(inputStorage, INTEGER_TYPE, abi._integer_argument_registers, to_Register);
   loadArray(inputStorage, VECTOR_TYPE, abi._vector_argument_registers, as_FloatRegister);
 
   objArrayOop outputStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.outputStorage_offset));
-  loadArray(outputStorage, INTEGER_TYPE, abi._integer_return_registers, as_Register);
+  loadArray(outputStorage, INTEGER_TYPE, abi._integer_return_registers, to_Register);
   loadArray(outputStorage, VECTOR_TYPE, abi._vector_return_registers, as_FloatRegister);
 
   objArrayOop volatileStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.volatileStorage_offset));
-  loadArray(volatileStorage, INTEGER_TYPE, abi._integer_additional_volatile_registers, as_Register);
+  loadArray(volatileStorage, INTEGER_TYPE, abi._integer_additional_volatile_registers, to_Register);
   loadArray(volatileStorage, VECTOR_TYPE, abi._vector_additional_volatile_registers, as_FloatRegister);
 
   abi._stack_alignment_bytes = abi_oop->int_field(ABI.stackAlignment_offset);
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
index 7b194570214..d3a7a1647b6 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
@@ -365,11 +365,16 @@ frame frame::sender_for_entry_frame(RegisterMap* map) const {
   return fr;
 }
 
-JavaFrameAnchor* OptimizedEntryBlob::jfa_for_frame(const frame& frame) const {
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
   ShouldNotCallThis();
   return nullptr;
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const {
   ShouldNotCallThis();
   return {};
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index 85e4846653f..65803b1ba97 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -365,7 +365,7 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
   __ cbz(pre_val_reg, *stub->continuation());
   ce->store_parameter(stub->pre_val()->as_register(), 0);
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
index 8598fb7e7c1..4ce9869a157 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
@@ -116,7 +116,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
     log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
                                 nm->method()->name_and_sig_as_C_string(),
                                 nm, *(address *) return_address_ptr, nm->is_osr_method(), thread,
-                                thread->get_thread_name(), frame.sp(), nm->verified_entry_point());
+                                thread->name(), frame.sp(), nm->verified_entry_point());
   }
 
   new_frame->sp = frame.sp();
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
index 0aca4c3ccba..bb5ec7e89d3 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -613,7 +613,7 @@ void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, Shen
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
   __ cbz(pre_val_reg, *stub->continuation());
   ce->store_parameter(stub->pre_val()->as_register(), 0);
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 545511371b0..bc170cad1e9 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -40,7 +40,6 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -849,15 +848,10 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       br(Assembler::NE, slow_case);
     }
 
-    if (UseBiasedLocking) {
-      biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
-    }
-
     // Load (object->mark() | 1) into swap_reg
     ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
     orr(swap_reg, rscratch1, 1);
     if (EnableValhalla) {
-      assert(!UseBiasedLocking, "Not compatible with biased-locking");
       // Mask inline_type bit such that we go to the slow path if object is an inline type
       andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
     }
@@ -869,17 +863,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
            "displached header must be first word in BasicObjectLock");
 
     Label fail;
-    if (PrintBiasedLockingStatistics) {
-      Label fast;
-      cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
-      bind(fast);
-      atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
-                  rscratch2, rscratch1, tmp);
-      b(done);
-      bind(fail);
-    } else {
-      cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
-    }
+    cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
 
     // Fast check for recursive lock.
     //
@@ -916,12 +900,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
 
     // Save the test result, for recursive case, the result is zero
     str(swap_reg, Address(lock_reg, mark_offset));
-
-    if (PrintBiasedLockingStatistics) {
-      br(Assembler::NE, slow_case);
-      atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
-                  rscratch2, rscratch1, tmp);
-    }
     br(Assembler::EQ, done);
 
     bind(slow_case);
@@ -972,10 +950,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
     // Free entry
     str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
 
-    if (UseBiasedLocking) {
-      biased_locking_exit(obj_reg, header_reg, done);
-    }
-
     // Load the old header from BasicLock structure
     ldr(header_reg, Address(swap_reg,
                             BasicLock::displaced_header_offset_in_bytes()));
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 4560bdba6ec..eebac9b85d6 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -45,7 +45,6 @@
 #include "oops/accessDecorators.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/klass.inline.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/icache.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
@@ -445,178 +444,6 @@ void MacroAssembler::reserved_stack_check() {
     bind(no_reserved_zone_enabling);
 }
 
-void MacroAssembler::biased_locking_enter(Register lock_reg,
-                                          Register obj_reg,
-                                          Register swap_reg,
-                                          Register tmp_reg,
-                                          bool swap_reg_contains_mark,
-                                          Label& done,
-                                          Label* slow_case,
-                                          BiasedLockingCounters* counters) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-  assert_different_registers(lock_reg, obj_reg, swap_reg);
-
-  if (PrintBiasedLockingStatistics && counters == NULL)
-    counters = BiasedLocking::counters();
-
-  assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);
-  assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout");
-  Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
-  Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());
-  Address saved_mark_addr(lock_reg, 0);
-
-  // Biased locking
-  // See whether the lock is currently biased toward our thread and
-  // whether the epoch is still valid
-  // Note that the runtime guarantees sufficient alignment of JavaThread
-  // pointers to allow age to be placed into low bits
-  // First check to see whether biasing is even enabled for this object
-  Label cas_label;
-  if (!swap_reg_contains_mark) {
-    ldr(swap_reg, mark_addr);
-  }
-  andr(tmp_reg, swap_reg, markWord::biased_lock_mask_in_place);
-  cmp(tmp_reg, (u1)markWord::biased_lock_pattern);
-  br(Assembler::NE, cas_label);
-  // The bias pattern is present in the object's header. Need to check
-  // whether the bias owner and the epoch are both still current.
-  load_prototype_header(tmp_reg, obj_reg);
-  orr(tmp_reg, tmp_reg, rthread);
-  eor(tmp_reg, swap_reg, tmp_reg);
-  andr(tmp_reg, tmp_reg, ~((int) markWord::age_mask_in_place));
-  if (counters != NULL) {
-    Label around;
-    cbnz(tmp_reg, around);
-    atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);
-    b(done);
-    bind(around);
-  } else {
-    cbz(tmp_reg, done);
-  }
-
-  Label try_revoke_bias;
-  Label try_rebias;
-
-  // At this point we know that the header has the bias pattern and
-  // that we are not the bias owner in the current epoch. We need to
-  // figure out more details about the state of the header in order to
-  // know what operations can be legally performed on the object's
-  // header.
-
-  // If the low three bits in the xor result aren't clear, that means
-  // the prototype header is no longer biased and we have to revoke
-  // the bias on this object.
-  andr(rscratch1, tmp_reg, markWord::biased_lock_mask_in_place);
-  cbnz(rscratch1, try_revoke_bias);
-
-  // Biasing is still enabled for this data type. See whether the
-  // epoch of the current bias is still valid, meaning that the epoch
-  // bits of the mark word are equal to the epoch bits of the
-  // prototype header. (Note that the prototype header's epoch bits
-  // only change at a safepoint.) If not, attempt to rebias the object
-  // toward the current thread. Note that we must be absolutely sure
-  // that the current epoch is invalid in order to do this because
-  // otherwise the manipulations it performs on the mark word are
-  // illegal.
-  andr(rscratch1, tmp_reg, markWord::epoch_mask_in_place);
-  cbnz(rscratch1, try_rebias);
-
-  // The epoch of the current bias is still valid but we know nothing
-  // about the owner; it might be set or it might be clear. Try to
-  // acquire the bias of the object using an atomic operation. If this
-  // fails we will go in to the runtime to revoke the object's bias.
-  // Note that we first construct the presumed unbiased header so we
-  // don't accidentally blow away another thread's valid bias.
-  {
-    Label here;
-    mov(rscratch1, markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place);
-    andr(swap_reg, swap_reg, rscratch1);
-    orr(tmp_reg, swap_reg, rthread);
-    cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
-    // If the biasing toward our thread failed, this means that
-    // another thread succeeded in biasing it toward itself and we
-    // need to revoke that bias. The revocation will occur in the
-    // interpreter runtime in the slow case.
-    bind(here);
-    if (counters != NULL) {
-      atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()),
-                  tmp_reg, rscratch1, rscratch2);
-    }
-  }
-  b(done);
-
-  bind(try_rebias);
-  // At this point we know the epoch has expired, meaning that the
-  // current "bias owner", if any, is actually invalid. Under these
-  // circumstances _only_, we are allowed to use the current header's
-  // value as the comparison value when doing the cas to acquire the
-  // bias in the current epoch. In other words, we allow transfer of
-  // the bias from one thread to another directly in this situation.
-  //
-  // FIXME: due to a lack of registers we currently blow away the age
-  // bits in this situation. Should attempt to preserve them.
-  {
-    Label here;
-    load_prototype_header(tmp_reg, obj_reg);
-    orr(tmp_reg, rthread, tmp_reg);
-    cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
-    // If the biasing toward our thread failed, then another thread
-    // succeeded in biasing it toward itself and we need to revoke that
-    // bias. The revocation will occur in the runtime in the slow case.
-    bind(here);
-    if (counters != NULL) {
-      atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()),
-                  tmp_reg, rscratch1, rscratch2);
-    }
-  }
-  b(done);
-
-  bind(try_revoke_bias);
-  // The prototype mark in the klass doesn't have the bias bit set any
-  // more, indicating that objects of this data type are not supposed
-  // to be biased any more. We are going to try to reset the mark of
-  // this object to the prototype value and fall through to the
-  // CAS-based locking scheme. Note that if our CAS fails, it means
-  // that another thread raced us for the privilege of revoking the
-  // bias of this particular object, so it's okay to continue in the
-  // normal locking code.
-  //
-  // FIXME: due to a lack of registers we currently blow away the age
-  // bits in this situation. Should attempt to preserve them.
-  {
-    Label here, nope;
-    load_prototype_header(tmp_reg, obj_reg);
-    cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, &nope);
-    bind(here);
-
-    // Fall through to the normal CAS-based lock, because no matter what
-    // the result of the above CAS, some thread must have succeeded in
-    // removing the bias bit from the object's header.
-    if (counters != NULL) {
-      atomic_incw(Address((address)counters->revoked_lock_entry_count_addr()), tmp_reg,
-                  rscratch1, rscratch2);
-    }
-    bind(nope);
-  }
-
-  bind(cas_label);
-}
-
-void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-
-  // Check for biased locking unlock case, which is a no-op
-  // Note: we do not have to check the thread ID for two reasons.
-  // First, the interpreter checks for IllegalMonitorStateException at
-  // a higher level. Second, if the bias was revoked while we held the
-  // lock, the object could not be rebiased toward another thread, so
-  // the bias bit would be clear.
-  ldr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
-  andr(temp_reg, temp_reg, markWord::biased_lock_mask_in_place);
-  cmp(temp_reg, (u1)markWord::biased_lock_pattern);
-  br(Assembler::EQ, done);
-}
-
 static void pass_arg0(MacroAssembler* masm, Register arg) {
   if (c_rarg0 != arg ) {
     masm->mov(c_rarg0, arg);
@@ -5789,6 +5616,7 @@ int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from
 
 // Move a value between registers/stack slots and update the reg_state
 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
+  assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
   if (reg_state[to->value()] == reg_written) {
     return true; // Already written
   }
@@ -5873,7 +5701,7 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
                                           RegState reg_state[]) {
   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
-  assert(from->is_valid(), "source must bevalid");
+  assert(from->is_valid(), "source must be valid");
   Register tmp1 = r10, tmp2 = r11;
   Register fromReg;
   if (from->is_reg()) {
@@ -5890,6 +5718,7 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
   VMReg toReg;
   BasicType bt;
   while (stream.next(toReg, bt)) {
+    assert(toReg->is_valid(), "destination must be valid");
     int off = sig->at(stream.sig_index())._offset;
     assert(off > 0, "offset in object should be positive");
     Address fromAddr = Address(fromReg, off);
@@ -5974,6 +5803,7 @@ bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int&
   VMReg fromReg;
   BasicType bt;
   while (stream.next(fromReg, bt)) {
+    assert(fromReg->is_valid(), "source must be valid");
     int off = sig->at(stream.sig_index())._offset;
     assert(off > 0, "offset in object should be positive");
     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
@@ -6074,49 +5904,6 @@ void MacroAssembler::safepoint_isb() {
 #endif
 }
 
-void MacroAssembler::neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
-                                  FloatRegister src2, int cond, bool isQ) {
-  SIMD_Arrangement size = esize2arrangement(type2aelembytes(bt), isQ);
-  if (bt == T_FLOAT || bt == T_DOUBLE) {
-    switch (cond) {
-      case BoolTest::eq: fcmeq(dst, size, src1, src2); break;
-      case BoolTest::ne: {
-        fcmeq(dst, size, src1, src2);
-        notr(dst, T16B, dst);
-        break;
-      }
-      case BoolTest::ge: fcmge(dst, size, src1, src2); break;
-      case BoolTest::gt: fcmgt(dst, size, src1, src2); break;
-      case BoolTest::le: fcmge(dst, size, src2, src1); break;
-      case BoolTest::lt: fcmgt(dst, size, src2, src1); break;
-      default:
-        assert(false, "unsupported");
-        ShouldNotReachHere();
-    }
-  } else {
-    switch (cond) {
-      case BoolTest::eq: cmeq(dst, size, src1, src2); break;
-      case BoolTest::ne: {
-        cmeq(dst, size, src1, src2);
-        notr(dst, T16B, dst);
-        break;
-      }
-      case BoolTest::ge: cmge(dst, size, src1, src2); break;
-      case BoolTest::gt: cmgt(dst, size, src1, src2); break;
-      case BoolTest::le: cmge(dst, size, src2, src1); break;
-      case BoolTest::lt: cmgt(dst, size, src2, src1); break;
-      case BoolTest::uge: cmhs(dst, size, src1, src2); break;
-      case BoolTest::ugt: cmhi(dst, size, src1, src2); break;
-      case BoolTest::ult: cmhi(dst, size, src2, src1); break;
-      case BoolTest::ule: cmhs(dst, size, src2, src1); break;
-      default:
-        assert(false, "unsupported");
-        ShouldNotReachHere();
-    }
-  }
-}
-
-
 #ifndef PRODUCT
 void MacroAssembler::verify_cross_modify_fence_not_required() {
   if (VerifyCrossModifyFence) {
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 9841830f79f..3759fcbbf3a 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -110,20 +110,6 @@ class MacroAssembler: public Assembler {
 
   void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod);
 
-  // Biased locking support
-  // lock_reg and obj_reg must be loaded up with the appropriate values.
-  // swap_reg is killed.
-  // tmp_reg must be supplied and must not be rscratch1 or rscratch2
-  // Optional slow case is for implementations (interpreter and C1) which branch to
-  // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
-  void biased_locking_enter(Register lock_reg, Register obj_reg,
-                            Register swap_reg, Register tmp_reg,
-                            bool swap_reg_contains_mark,
-                            Label& done, Label* slow_case = NULL,
-                            BiasedLockingCounters* counters = NULL);
-  void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
-
-
   // Helper functions for statistics gathering.
   // Unconditional atomic increment.
   void atomic_incw(Register counter_addr, Register tmp, Register tmp2);
@@ -1117,8 +1103,6 @@ class MacroAssembler: public Assembler {
                bool acquire, bool release, bool weak,
                Register result);
 
-  // SIMD&FP comparison
-  void neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1, FloatRegister src2, int cond, bool isQ);
 private:
   void compare_eq(Register rn, Register rm, enum operand_size size);
 
diff --git a/src/hotspot/cpu/aarch64/register_aarch64.hpp b/src/hotspot/cpu/aarch64/register_aarch64.hpp
index 5a152d62777..479bd1f37c4 100644
--- a/src/hotspot/cpu/aarch64/register_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/register_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -36,7 +36,7 @@ typedef VMRegImpl* VMReg;
 class RegisterImpl;
 typedef RegisterImpl* Register;
 
-inline Register as_Register(int encoding) {
+inline const Register as_Register(int encoding) {
   return (Register)(intptr_t) encoding;
 }
 
@@ -53,7 +53,7 @@ class RegisterImpl: public AbstractRegisterImpl {
   Register successor() const                          { return as_Register(encoding() + 1); }
 
   // construction
-  inline friend Register as_Register(int encoding);
+  inline friend const Register as_Register(int encoding);
 
   VMReg as_VMReg();
 
@@ -242,6 +242,7 @@ class PRegisterImpl: public AbstractRegisterImpl {
  public:
   enum {
     number_of_registers = 16,
+    number_of_governing_registers = 8,
     max_slots_per_register = 1
   };
 
@@ -257,6 +258,7 @@ class PRegisterImpl: public AbstractRegisterImpl {
   int   encoding() const          { assert(is_valid(), "invalid register"); return (intptr_t)this; }
   int   encoding_nocheck() const  { return (intptr_t)this; }
   bool  is_valid() const          { return 0 <= (intptr_t)this && (intptr_t)this < number_of_registers; }
+  bool  is_governing() const      { return 0 <= (intptr_t)this && (intptr_t)this < number_of_governing_registers; }
   const char* name() const;
 };
 
@@ -420,4 +422,8 @@ inline FloatRegister AbstractRegSet<FloatRegister>::first() {
   return first ? as_FloatRegister(exact_log2(first)) : fnoreg;
 }
 
+inline Register as_Register(FloatRegister reg) {
+  return as_Register(reg->encoding());
+}
+
 #endif // CPU_AARCH64_REGISTER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index c465083bbb5..2da3640bc13 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -961,7 +961,7 @@ static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
 
 
 // ---------------------------------------------------------------
-AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
+AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
                                                             int comp_args_on_stack,
                                                             const GrowableArray<SigEntry>* sig,
                                                             const VMRegPair* regs,
@@ -970,7 +970,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
                                                             const GrowableArray<SigEntry>* sig_cc_ro,
                                                             const VMRegPair* regs_cc_ro,
                                                             AdapterFingerPrint* fingerprint,
-                                                            AdapterBlob*& new_adapter) {
+                                                            AdapterBlob*& new_adapter,
+                                                            bool allocate_code_blob) {
 
   address i2c_entry = __ pc();
   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
@@ -1035,9 +1036,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
 
   // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
   // the GC knows about the location of oop argument locations passed to the c2i adapter.
-
-  bool caller_must_gc_arguments = (regs != regs_cc);
-  new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
+  if (allocate_code_blob) {
+    bool caller_must_gc_arguments = (regs != regs_cc);
+    new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
+  }
 
   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
 }
@@ -2042,15 +2044,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ ldr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (UseBiasedLocking) {
-      __ biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, lock_done, &slow_path_lock);
-    }
-
     // Load (object->mark() | 1) into swap_reg %r0
     __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
     __ orr(swap_reg, rscratch1, 1);
     if (EnableValhalla) {
-      assert(!UseBiasedLocking, "Not compatible with biased-locking");
       // Mask inline_type bit such that we go to the slow path if object is an inline type
       __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
     }
@@ -2199,11 +2196,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ ldr(obj_reg, Address(oop_handle_reg, 0));
 
     Label done;
-
-    if (UseBiasedLocking) {
-      __ biased_locking_exit(obj_reg, old_hdr, done);
-    }
-
     // Simple recursive lock?
 
     __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index 80c526d60ec..94a3ff55ea7 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -5709,6 +5709,7 @@ class StubGenerator: public StubCodeGenerator {
    *  c_rarg3   - dest_start
    *  c_rarg4   - dest_offset
    *  c_rarg5   - isURL
+   *  c_rarg6   - isMIME
    *
    */
   address generate_base64_decodeBlock() {
@@ -5791,12 +5792,13 @@ class StubGenerator: public StubCodeGenerator {
     StubCodeMark mark(this, "StubRoutines", "decodeBlock");
     address start = __ pc();
 
-    Register src   = c_rarg0;  // source array
-    Register soff  = c_rarg1;  // source start offset
-    Register send  = c_rarg2;  // source end offset
-    Register dst   = c_rarg3;  // dest array
-    Register doff  = c_rarg4;  // position for writing to dest array
-    Register isURL = c_rarg5;  // Base64 or URL character set
+    Register src    = c_rarg0;  // source array
+    Register soff   = c_rarg1;  // source start offset
+    Register send   = c_rarg2;  // source end offset
+    Register dst    = c_rarg3;  // dest array
+    Register doff   = c_rarg4;  // position for writing to dest array
+    Register isURL  = c_rarg5;  // Base64 or URL character set
+    Register isMIME = c_rarg6;  // Decoding MIME block - unused in this implementation
 
     Register length = send;    // reuse send as length of source data to process
 
@@ -5980,6 +5982,10 @@ class StubGenerator: public StubCodeGenerator {
         acquire = false;
         release = false;
         break;
+      case memory_order_release:
+        acquire = false;
+        release = true;
+        break;
       default:
         acquire = true;
         release = true;
@@ -6061,6 +6067,20 @@ class StubGenerator: public StubCodeGenerator {
       (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
 
+    AtomicStubMark mark_cmpxchg_4_release
+      (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
+    gen_cas_entry(MacroAssembler::word, memory_order_release);
+    AtomicStubMark mark_cmpxchg_8_release
+      (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
+    gen_cas_entry(MacroAssembler::xword, memory_order_release);
+
+    AtomicStubMark mark_cmpxchg_4_seq_cst
+      (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
+    gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
+    AtomicStubMark mark_cmpxchg_8_seq_cst
+      (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
+    gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
+
     ICache::invalidate_range(first_entry, __ pc() - first_entry);
   }
 #endif // LINUX
@@ -7362,6 +7382,10 @@ DEFAULT_ATOMIC_OP(cmpxchg, 8, )
 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
+DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
+DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
+DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
+DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
 
 #undef DEFAULT_ATOMIC_OP
 
diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
index ae5d709b8b7..cb2adc8c6a6 100644
--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
@@ -3823,10 +3823,15 @@ void TemplateTable::withfield() {
   transition(vtos, atos);
   resolve_cache_and_index(f2_byte, c_rarg1 /*cache*/, c_rarg2 /*index*/, sizeof(u2));
 
+  ByteSize cp_base_offset = ConstantPoolCache::base_offset();
+
   // n.b. unlike x86 cache is now rcpool plus the indexed offset
-  // so using rcpool to meet shared code expectations
+  __ lea(c_rarg1, Address(c_rarg1, in_bytes(cp_base_offset)));
 
-  call_VM(r1, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), rcpool);
+  __ lea(c_rarg2, at_tos());
+  call_VM(r1, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), c_rarg1, c_rarg2);
+  // new value type is returned in r1
+  // stack adjustment is returned in r0
   __ verify_oop(r1);
   __ add(esp, esp, r0);
   __ mov(r0, r1);
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 06267046028..1eb14630e7a 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1001,10 +1001,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Vector width in bytes
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   return MaxVectorSize;
@@ -1055,7 +1051,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd,
   return NULL;
 }
 
-bool Matcher::is_generic_reg2reg_move(MachNode* m) {
+bool Matcher::is_reg2reg_move(MachNode* m) {
   ShouldNotReachHere();  // generic vector operands not supported
   return false;
 }
@@ -1100,6 +1096,16 @@ bool Matcher::is_spillable_arg( int reg ) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 12 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   return false;
 }
@@ -5454,7 +5460,6 @@ instruct storeXConditional( memoryex mem, iRegX oldval, iRegX newval, iRegX tmp,
     __ cmp($tmp$$Register, 1, eq);
     __ b(loop, eq);
     __ teq($tmp$$Register, 0);
-    // used by biased locking only. Requires a membar.
     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::LoadLoad), noreg);
   %}
   ins_pipe( long_memory_op );
@@ -8954,7 +8959,6 @@ instruct partialSubtypeCheck( R0RegP index, R1RegP sub, R2RegP super, flagsRegP
 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch )
 %{
   match(Set pcc (FastLock object box));
-  predicate(!(UseBiasedLocking && !UseOptoBiasInlining));
 
   effect(TEMP scratch, TEMP scratch2);
   ins_cost(DEFAULT_COST*3);
@@ -8966,22 +8970,6 @@ instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRe
   ins_pipe(long_memory_op);
 %}
 
-instruct cmpFastLock_noBiasInline(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2,
-                                  iRegP scratch, iRegP scratch3) %{
-  match(Set pcc (FastLock object box));
-  predicate(UseBiasedLocking && !UseOptoBiasInlining);
-
-  effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
-  ins_cost(DEFAULT_COST*5);
-
-  format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
-  ins_encode %{
-    __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
-  %}
-  ins_pipe(long_memory_op);
-%}
-
-
 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch ) %{
   match(Set pcc (FastUnlock object box));
   effect(TEMP scratch, TEMP scratch2);
diff --git a/src/hotspot/cpu/arm/assembler_arm.cpp b/src/hotspot/cpu/arm/assembler_arm.cpp
index 3336d9f7d11..d23207a6e34 100644
--- a/src/hotspot/cpu/arm/assembler_arm.cpp
+++ b/src/hotspot/cpu/arm/assembler_arm.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,6 @@
 #include "memory/resourceArea.hpp"
 #include "prims/jvm_misc.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
diff --git a/src/hotspot/cpu/arm/assembler_arm_32.cpp b/src/hotspot/cpu/arm/assembler_arm_32.cpp
index 0166d85a21a..5dd9d39392a 100644
--- a/src/hotspot/cpu/arm/assembler_arm_32.cpp
+++ b/src/hotspot/cpu/arm/assembler_arm_32.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,6 @@
 #include "memory/resourceArea.hpp"
 #include "prims/jvm_misc.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
diff --git a/src/hotspot/cpu/arm/c1_Defs_arm.hpp b/src/hotspot/cpu/arm/c1_Defs_arm.hpp
index fd165edb6dd..b8ed431891d 100644
--- a/src/hotspot/cpu/arm/c1_Defs_arm.hpp
+++ b/src/hotspot/cpu/arm/c1_Defs_arm.hpp
@@ -76,6 +76,5 @@ enum {
 
 #define PATCHED_ADDR (204)
 #define CARDTABLEBARRIERSET_POST_BARRIER_HELPER
-#define GENERATE_ADDRESS_IS_PREFERRED
 
 #endif // CPU_ARM_C1_DEFS_ARM_HPP
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index 33e3bce65b5..67438391e33 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -247,7 +247,7 @@ int LIR_Assembler::emit_unwind_handler() {
   if (method()->is_synchronized()) {
     monitor_address(0, FrameMap::R0_opr);
     stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
-    __ unlock_object(R2, R1, R0, Rtemp, *stub->entry());
+    __ unlock_object(R2, R1, R0, *stub->entry());
     __ bind(*stub->continuation());
   }
 
@@ -494,8 +494,7 @@ void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool po
 
 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
                             LIR_PatchCode patch_code, CodeEmitInfo* info,
-                            bool pop_fpu_stack, bool wide,
-                            bool unaligned) {
+                            bool pop_fpu_stack, bool wide) {
   LIR_Address* to_addr = dest->as_address_ptr();
   Register base_reg = to_addr->base()->as_pointer_register();
   const bool needs_patching = (patch_code != lir_patch_none);
@@ -695,7 +694,7 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 
 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
                             LIR_PatchCode patch_code, CodeEmitInfo* info,
-                            bool wide, bool unaligned) {
+                            bool wide) {
   assert(src->is_address(), "should not call otherwise");
   assert(dest->is_register(), "should not call otherwise");
   LIR_Address* addr = src->as_address_ptr();
@@ -2429,19 +2428,17 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   Register obj = op->obj_opr()->as_pointer_register();
   Register hdr = op->hdr_opr()->as_pointer_register();
   Register lock = op->lock_opr()->as_pointer_register();
-  Register tmp = op->scratch_opr()->is_illegal() ? noreg :
-                 op->scratch_opr()->as_pointer_register();
 
   if (!UseFastLocking) {
     __ b(*op->stub()->entry());
   } else if (op->code() == lir_lock) {
     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
-    int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
+    int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
     if (op->info() != NULL) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }
   } else if (op->code() == lir_unlock) {
-    __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry());
+    __ unlock_object(hdr, obj, lock, *op->stub()->entry());
   } else {
     ShouldNotReachHere();
   }
diff --git a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
index b16986ee2a2..7697934b718 100644
--- a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
@@ -412,21 +412,13 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
   LIR_Opr lock = new_pointer_register();
   LIR_Opr hdr  = new_pointer_register();
 
-  // Need a scratch register for biased locking on arm
-  LIR_Opr scratch = LIR_OprFact::illegalOpr;
-  if(UseBiasedLocking) {
-    scratch = new_pointer_register();
-  } else {
-    scratch = atomicLockOpr();
-  }
-
   CodeEmitInfo* info_for_exception = NULL;
   if (x->needs_null_check()) {
     info_for_exception = state_for(x);
   }
 
   CodeEmitInfo* info = state_for(x, x->state(), true);
-  monitor_enter(obj.result(), lock, hdr, scratch,
+  monitor_enter(obj.result(), lock, hdr, LIR_OprFact::illegalOpr,
                 x->monitor_no(), info_for_exception, info);
 }
 
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
index ac7137ca66d..7f1d4341872 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
@@ -31,7 +31,6 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
@@ -90,11 +89,7 @@ void C1_MacroAssembler::try_allocate(Register obj, Register obj_end, Register tm
 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp) {
   assert_different_registers(obj, klass, len, tmp);
 
-  if(UseBiasedLocking && !len->is_valid()) {
-    ldr(tmp, Address(klass, Klass::prototype_header_offset()));
-  } else {
-    mov(tmp, (intptr_t)markWord::prototype().value());
-  }
+  mov(tmp, (intptr_t)markWord::prototype().value());
 
   str(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
   str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
@@ -187,14 +182,12 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len,
   initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB);
 }
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj,
-                                   Register disp_hdr, Register tmp1,
-                                   Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   Label done, fast_lock, fast_lock_done;
   int null_check_offset = 0;
 
   const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level
-  assert_different_registers(hdr, obj, disp_hdr, tmp1, tmp2);
+  assert_different_registers(hdr, obj, disp_hdr, tmp2);
 
   assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code");
   const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
@@ -211,10 +204,6 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj,
     b(slow_case, ne);
   }
 
-  if (UseBiasedLocking) {
-    biased_locking_enter(obj, hdr/*scratched*/, tmp1, false, tmp2, done, slow_case);
-  }
-
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
   // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
@@ -234,8 +223,9 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj,
   // -2- test (hdr - SP) if the low two bits are 0
   sub(tmp2, hdr, SP, eq);
   movs(tmp2, AsmOperand(tmp2, lsr, exact_log2(os::vm_page_size())), eq);
-  // If 'eq' then OK for recursive fast locking: store 0 into a lock record.
-  str(tmp2, Address(disp_hdr, mark_offset), eq);
+  // If still 'eq' then recursive locking OK
+  // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
+  str(tmp2, Address(disp_hdr, mark_offset));
   b(fast_lock_done, eq);
   // else need slow case
   b(slow_case);
@@ -248,23 +238,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj,
   cas_for_lock_acquire(hdr, disp_hdr, obj, tmp2, slow_case);
 
   bind(fast_lock_done);
-
-#ifndef PRODUCT
-  if (PrintBiasedLockingStatistics) {
-    cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
-  }
-#endif // !PRODUCT
-
   bind(done);
 
   return null_check_offset;
 }
 
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj,
-                                      Register disp_hdr, Register tmp,
-                                      Label& slow_case) {
-  // Note: this method is not using its 'tmp' argument
-
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   assert_different_registers(hdr, obj, disp_hdr, Rtemp);
   Register tmp2 = Rtemp;
 
@@ -273,11 +252,6 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj,
   const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
 
   Label done;
-  if (UseBiasedLocking) {
-    // load object
-    ldr(obj, Address(disp_hdr, obj_offset));
-    biased_locking_exit(obj, hdr, done);
-  }
 
   assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
 
@@ -286,10 +260,8 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj,
   // If hdr is NULL, we've got recursive locking and there's nothing more to do
   cbz(hdr, done);
 
-  if(!UseBiasedLocking) {
-    // load object
-    ldr(obj, Address(disp_hdr, obj_offset));
-  }
+  // load object
+  ldr(obj, Address(disp_hdr, obj_offset));
 
   // Restore the object header
   cas_for_lock_release(disp_hdr, hdr, obj, tmp2, slow_case);
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
index faf546db698..acf8fb9ef8d 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
@@ -59,9 +59,9 @@
     max_array_allocation_length = 0x01000000
   };
 
-  int lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case);
+  int lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
 
-  void unlock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case);
+  void unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
 
   // This platform only uses signal-based null checks. The Label is not needed.
   void null_check(Register r, Label *Lnull = NULL) { MacroAssembler::null_check(r); }
diff --git a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp
index 2211d5c5fa3..07d22a1fce9 100644
--- a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -78,7 +78,7 @@ void C2_MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
   // mov(result_reg, 1);  //equal
 }
 
-void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2, Register scratch3) {
+void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratch, Register Rscratch2) {
   assert(VM_Version::supports_ldrex(), "unsupported, yet?");
 
   Register Rmark      = Rscratch2;
@@ -97,14 +97,6 @@ void C2_MacroAssembler::fast_lock(Register Roop, Register Rbox, Register Rscratc
     b(done, ne);
   }
 
-  if (UseBiasedLocking && !UseOptoBiasInlining) {
-    assert(scratch3 != noreg, "need extra temporary for -XX:-UseOptoBiasInlining");
-    biased_locking_enter(Roop, Rmark, Rscratch, false, scratch3, done, done);
-    // Fall through if lock not biased otherwise branch to done
-  }
-
-  // Invariant: Rmark loaded below does not contain biased lock pattern
-
   ldr(Rmark, Address(Roop, oopDesc::mark_offset_in_bytes()));
   tst(Rmark, markWord::unlocked_value);
   b(fast_lock, ne);
@@ -148,10 +140,6 @@ void C2_MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscra
 
   Label done;
 
-  if (UseBiasedLocking && !UseOptoBiasInlining) {
-    biased_locking_exit(Roop, Rscratch, done);
-  }
-
   ldr(Rmark, Address(Rbox, BasicLock::displaced_header_offset_in_bytes()));
   // If hdr is NULL, we've got recursive locking and there's nothing more to do
   cmp(Rmark, 0);
diff --git a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.hpp b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.hpp
index 2f43e6e4e24..c2fe811bf32 100644
--- a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.hpp
@@ -33,7 +33,7 @@
                           Register limit, Register result,
                           Register chr1, Register chr2, Label& Ldone);
 
-  void fast_lock(Register obj, Register box, Register scratch, Register scratch2, Register scratch3 = noreg);
+  void fast_lock(Register obj, Register box, Register scratch, Register scratch2);
   void fast_unlock(Register obj, Register box, Register scratch, Register scratch2);
 
 #endif // CPU_ARM_C2_MACROASSEMBLER_ARM_HPP
diff --git a/src/hotspot/cpu/arm/c2_globals_arm.hpp b/src/hotspot/cpu/arm/c2_globals_arm.hpp
index 7754001dd0a..1d666357b30 100644
--- a/src/hotspot/cpu/arm/c2_globals_arm.hpp
+++ b/src/hotspot/cpu/arm/c2_globals_arm.hpp
@@ -45,9 +45,7 @@ define_pd_global(intx, CompileThreshold,             10000);
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         4);
 // C2 gets to use all the float/double registers
-define_pd_global(intx, FLOATPRESSURE,                30);
 define_pd_global(intx, FreqInlineSize,               175);
-define_pd_global(intx, INTPRESSURE,                  12);
 define_pd_global(intx, InteriorEntryAlignment,       16);  // = CodeEntryAlignment
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 // The default setting 16/16 seems to work best.
diff --git a/src/hotspot/cpu/arm/frame_arm.cpp b/src/hotspot/cpu/arm/frame_arm.cpp
index 17841a8cf71..6b8f77bf088 100644
--- a/src/hotspot/cpu/arm/frame_arm.cpp
+++ b/src/hotspot/cpu/arm/frame_arm.cpp
@@ -313,6 +313,16 @@ frame frame::sender_for_entry_frame(RegisterMap* map) const {
   return fr;
 }
 
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
+  ShouldNotCallThis();
+  return nullptr;
+}
+
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 //------------------------------------------------------------------------------
 // frame::verify_deopt_original_pc
 //
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
index b67f39e2e6b..63bf45b8db5 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
@@ -332,7 +332,7 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
 
   __ cbz(pre_val_reg, *stub->continuation());
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.cpp b/src/hotspot/cpu/arm/interp_masm_arm.cpp
index 48efb6a8036..2562f51f6fe 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.cpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.cpp
@@ -40,7 +40,6 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -890,11 +889,6 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
       b(slow_case, ne);
     }
 
-    if (UseBiasedLocking) {
-      biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case);
-    }
-
-
     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
     // That would be acceptable as ether CAS or slow case path is taken in that case.
     // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
@@ -912,12 +906,6 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
 
     cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case);
 
-#ifndef PRODUCT
-    if (PrintBiasedLockingStatistics) {
-      cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
-    }
-#endif //!PRODUCT
-
     b(done);
 
     // If we got here that means the object is locked by ether calling thread or another thread.
@@ -962,13 +950,6 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
     // If still 'eq' then recursive locking OK: store 0 into lock record
     str(R0, Address(Rlock, mark_offset), eq);
 
-
-#ifndef PRODUCT
-    if (PrintBiasedLockingStatistics) {
-      cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr());
-    }
-#endif // !PRODUCT
-
     b(done, eq);
 
     bind(slow_case);
@@ -1010,10 +991,6 @@ void InterpreterMacroAssembler::unlock_object(Register Rlock) {
     // Free entry
     str(Rzero, Address(Rlock, obj_offset));
 
-    if (UseBiasedLocking) {
-      biased_locking_exit(Robj, Rmark, done);
-    }
-
     // Load the old header from BasicLock structure
     ldr(Rmark, Address(Rlock, mark_offset));
 
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.cpp b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
index 1896a940b19..12dfc1adf0d 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
@@ -40,7 +40,6 @@
 #include "oops/accessDecorators.hpp"
 #include "oops/klass.inline.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.hpp"
 #include "runtime/objectMonitor.hpp"
@@ -1288,221 +1287,6 @@ void MacroAssembler::cond_atomic_inc32(AsmCondition cond, int* counter_addr) {
 
 #endif // !PRODUCT
 
-
-// Building block for CAS cases of biased locking: makes CAS and records statistics.
-// The slow_case label is used to transfer control if CAS fails. Otherwise leaves condition codes set.
-void MacroAssembler::biased_locking_enter_with_cas(Register obj_reg, Register old_mark_reg, Register new_mark_reg,
-                                                 Register tmp, Label& slow_case, int* counter_addr) {
-
-  cas_for_lock_acquire(old_mark_reg, new_mark_reg, obj_reg, tmp, slow_case);
-#ifdef ASSERT
-  breakpoint(ne); // Fallthrough only on success
-#endif
-#ifndef PRODUCT
-  if (counter_addr != NULL) {
-    cond_atomic_inc32(al, counter_addr);
-  }
-#endif // !PRODUCT
-}
-
-void MacroAssembler::biased_locking_enter(Register obj_reg, Register swap_reg, Register tmp_reg,
-                                          bool swap_reg_contains_mark,
-                                          Register tmp2,
-                                          Label& done, Label& slow_case,
-                                          BiasedLockingCounters* counters) {
-  // obj_reg must be preserved (at least) if the bias locking fails
-  // tmp_reg is a temporary register
-  // swap_reg was used as a temporary but contained a value
-  //   that was used afterwards in some call pathes. Callers
-  //   have been fixed so that swap_reg no longer needs to be
-  //   saved.
-  // Rtemp in no longer scratched
-
-  assert(UseBiasedLocking, "why call this otherwise?");
-  assert_different_registers(obj_reg, swap_reg, tmp_reg, tmp2);
-  guarantee(swap_reg!=tmp_reg, "invariant");
-  assert(tmp_reg != noreg, "must supply tmp_reg");
-
-#ifndef PRODUCT
-  if (PrintBiasedLockingStatistics && (counters == NULL)) {
-    counters = BiasedLocking::counters();
-  }
-#endif
-
-  assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout");
-  Address mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
-
-  // Biased locking
-  // See whether the lock is currently biased toward our thread and
-  // whether the epoch is still valid
-  // Note that the runtime guarantees sufficient alignment of JavaThread
-  // pointers to allow age to be placed into low bits
-  // First check to see whether biasing is even enabled for this object
-  Label cas_label;
-
-  if (!swap_reg_contains_mark) {
-    ldr(swap_reg, mark_addr);
-  }
-
-  // On MP platform loads could return 'stale' values in some cases.
-  // That is acceptable since either CAS or slow case path is taken in the worst case.
-
-  andr(tmp_reg, swap_reg, markWord::biased_lock_mask_in_place);
-  cmp(tmp_reg, markWord::biased_lock_pattern);
-
-  b(cas_label, ne);
-
-  // The bias pattern is present in the object's header. Need to check
-  // whether the bias owner and the epoch are both still current.
-  load_klass(tmp_reg, obj_reg);
-  ldr(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));
-  orr(tmp_reg, tmp_reg, Rthread);
-  eor(tmp_reg, tmp_reg, swap_reg);
-
-  bics(tmp_reg, tmp_reg, ((int) markWord::age_mask_in_place));
-
-#ifndef PRODUCT
-  if (counters != NULL) {
-    cond_atomic_inc32(eq, counters->biased_lock_entry_count_addr());
-  }
-#endif // !PRODUCT
-
-  b(done, eq);
-
-  Label try_revoke_bias;
-  Label try_rebias;
-
-  // At this point we know that the header has the bias pattern and
-  // that we are not the bias owner in the current epoch. We need to
-  // figure out more details about the state of the header in order to
-  // know what operations can be legally performed on the object's
-  // header.
-
-  // If the low three bits in the xor result aren't clear, that means
-  // the prototype header is no longer biased and we have to revoke
-  // the bias on this object.
-  tst(tmp_reg, markWord::biased_lock_mask_in_place);
-  b(try_revoke_bias, ne);
-
-  // Biasing is still enabled for this data type. See whether the
-  // epoch of the current bias is still valid, meaning that the epoch
-  // bits of the mark word are equal to the epoch bits of the
-  // prototype header. (Note that the prototype header's epoch bits
-  // only change at a safepoint.) If not, attempt to rebias the object
-  // toward the current thread. Note that we must be absolutely sure
-  // that the current epoch is invalid in order to do this because
-  // otherwise the manipulations it performs on the mark word are
-  // illegal.
-  tst(tmp_reg, markWord::epoch_mask_in_place);
-  b(try_rebias, ne);
-
-  // tmp_reg has the age, epoch and pattern bits cleared
-  // The remaining (owner) bits are (Thread ^ current_owner)
-
-  // The epoch of the current bias is still valid but we know nothing
-  // about the owner; it might be set or it might be clear. Try to
-  // acquire the bias of the object using an atomic operation. If this
-  // fails we will go in to the runtime to revoke the object's bias.
-  // Note that we first construct the presumed unbiased header so we
-  // don't accidentally blow away another thread's valid bias.
-
-  // Note that we know the owner is not ourself. Hence, success can
-  // only happen when the owner bits is 0
-
-  // until the assembler can be made smarter, we need to make some assumptions about the values
-  // so we can optimize this:
-  assert((markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place) == 0x1ff, "biased bitmasks changed");
-
-  mov(swap_reg, AsmOperand(swap_reg, lsl, 23));
-  mov(swap_reg, AsmOperand(swap_reg, lsr, 23)); // markWord with thread bits cleared (for CAS)
-
-  orr(tmp_reg, swap_reg, Rthread); // new mark
-
-  biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
-        (counters != NULL) ? counters->anonymously_biased_lock_entry_count_addr() : NULL);
-
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-
-  b(done);
-
-  bind(try_rebias);
-
-  // At this point we know the epoch has expired, meaning that the
-  // current "bias owner", if any, is actually invalid. Under these
-  // circumstances _only_, we are allowed to use the current header's
-  // value as the comparison value when doing the cas to acquire the
-  // bias in the current epoch. In other words, we allow transfer of
-  // the bias from one thread to another directly in this situation.
-
-  // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
-
-  eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
-
-  // owner bits 'random'. Set them to Rthread.
-  mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
-  mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
-
-  orr(tmp_reg, tmp_reg, Rthread); // new mark
-
-  biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
-        (counters != NULL) ? counters->rebiased_lock_entry_count_addr() : NULL);
-
-  // If the biasing toward our thread failed, then another thread
-  // succeeded in biasing it toward itself and we need to revoke that
-  // bias. The revocation will occur in the runtime in the slow case.
-
-  b(done);
-
-  bind(try_revoke_bias);
-
-  // The prototype mark in the klass doesn't have the bias bit set any
-  // more, indicating that objects of this data type are not supposed
-  // to be biased any more. We are going to try to reset the mark of
-  // this object to the prototype value and fall through to the
-  // CAS-based locking scheme. Note that if our CAS fails, it means
-  // that another thread raced us for the privilege of revoking the
-  // bias of this particular object, so it's okay to continue in the
-  // normal locking code.
-
-  // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
-
-  eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
-
-  // owner bits 'random'. Clear them
-  mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
-  mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
-
-  biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, cas_label,
-        (counters != NULL) ? counters->revoked_lock_entry_count_addr() : NULL);
-
-  // Fall through to the normal CAS-based lock, because no matter what
-  // the result of the above CAS, some thread must have succeeded in
-  // removing the bias bit from the object's header.
-
-  bind(cas_label);
-}
-
-
-void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-
-  // Check for biased locking unlock case, which is a no-op
-  // Note: we do not have to check the thread ID for two reasons.
-  // First, the interpreter checks for IllegalMonitorStateException at
-  // a higher level. Second, if the bias was revoked while we held the
-  // lock, the object could not be rebiased toward another thread, so
-  // the bias bit would be clear.
-  ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
-
-  andr(tmp_reg, tmp_reg, markWord::biased_lock_mask_in_place);
-  cmp(tmp_reg, markWord::biased_lock_pattern);
-  b(done, eq);
-}
-
-
 void MacroAssembler::resolve_jobject(Register value,
                                      Register tmp1,
                                      Register tmp2) {
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.hpp b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
index a27a54e1c71..b07782332db 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,8 +28,6 @@
 #include "code/relocInfo.hpp"
 #include "utilities/powerOfTwo.hpp"
 
-class BiasedLockingCounters;
-
 // Introduced AddressLiteral and its subclasses to ease portability from
 // x86 and avoid relocation issues
 class AddressLiteral {
@@ -359,29 +357,6 @@ class MacroAssembler: public Assembler {
     ShouldNotReachHere();
   }
 
-  // Biased locking support
-  // lock_reg and obj_reg must be loaded up with the appropriate values.
-  // swap_reg must be supplied.
-  // tmp_reg must be supplied.
-  // Done label is branched to with condition code EQ set if the lock is
-  // biased and we acquired it. Slow case label is branched to with
-  // condition code NE set if the lock is biased but we failed to acquire
-  // it. Otherwise fall through.
-  // Notes:
-  // - swap_reg and tmp_reg are scratched
-  // - Rtemp was (implicitly) scratched and can now be specified as the tmp2
-  void biased_locking_enter(Register obj_reg, Register swap_reg, Register tmp_reg,
-                            bool swap_reg_contains_mark,
-                            Register tmp2,
-                            Label& done, Label& slow_case,
-                            BiasedLockingCounters* counters = NULL);
-  void biased_locking_exit(Register obj_reg, Register temp_reg, Label& done);
-
-  // Building block for CAS cases of biased locking: makes CAS and records statistics.
-  // Optional slow_case label is used to transfer control if CAS fails. Otherwise leaves condition codes set.
-  void biased_locking_enter_with_cas(Register obj_reg, Register old_mark_reg, Register new_mark_reg,
-                                     Register tmp, Label& slow_case, int* counter_addr);
-
   void resolve_jobject(Register value, Register tmp1, Register tmp2);
 
   void nop() {
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index 58cbac7419c..09ae8e0e804 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -862,11 +862,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     assert(markWord::unlocked_value == 1, "adjust this code");
     __ tbz(Rtemp, exact_log2(markWord::unlocked_value), slow_case);
 
-    if (UseBiasedLocking) {
-      assert(is_power_of_2(markWord::biased_lock_bit_in_place), "adjust this code");
-      __ tbnz(Rtemp, exact_log2(markWord::biased_lock_bit_in_place), slow_case);
-    }
-
     __ bics(Rtemp, Rtemp, ~markWord::hash_mask_in_place);
     __ mov(R0, AsmOperand(Rtemp, lsr, markWord::hash_shift), ne);
     __ bx(LR, ne);
@@ -1151,17 +1146,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   const Register disp_hdr    = altFP_7_11;
   const Register tmp         = R8;
 
-  Label slow_lock, slow_lock_biased, lock_done, fast_lock;
+  Label slow_lock, lock_done, fast_lock;
   if (method->is_synchronized()) {
     // The first argument is a handle to sync object (a class or an instance)
     __ ldr(sync_obj, Address(R1));
     // Remember the handle for the unlocking code
     __ mov(sync_handle, R1);
 
-    if(UseBiasedLocking) {
-      __ biased_locking_enter(sync_obj, tmp, disp_hdr/*scratched*/, false, Rtemp, lock_done, slow_lock_biased);
-    }
-
     const Register mark = tmp;
     // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
     // That would be acceptable as either CAS or slow case path is taken in that case
@@ -1180,8 +1171,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // -2- test (hdr - SP) if the low two bits are 0
     __ sub(Rtemp, mark, SP, eq);
     __ movs(Rtemp, AsmOperand(Rtemp, lsr, exact_log2(os::vm_page_size())), eq);
-    // If still 'eq' then recursive locking OK: set displaced header to 0
-    __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()), eq);
+    // If still 'eq' then recursive locking OK
+    // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
+    __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
     __ b(lock_done, eq);
     __ b(slow_lock);
 
@@ -1242,12 +1234,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   if (method->is_synchronized()) {
     __ ldr(sync_obj, Address(sync_handle));
 
-    if(UseBiasedLocking) {
-      __ biased_locking_exit(sync_obj, Rtemp, unlock_done);
-      // disp_hdr may not have been saved on entry with biased locking
-      __ sub(disp_hdr, FP, lock_slot_fp_offset);
-    }
-
     // See C1_MacroAssembler::unlock_object() for more comments
     __ ldr(R2, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
     __ cbz(R2, unlock_done);
@@ -1303,11 +1289,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   if (method->is_synchronized()) {
     // Locking slow case
-    if(UseBiasedLocking) {
-      __ bind(slow_lock_biased);
-      __ sub(disp_hdr, FP, lock_slot_fp_offset);
-    }
-
     __ bind(slow_lock);
 
     push_param_registers(masm, fp_regs_in_arguments);
diff --git a/src/hotspot/cpu/arm/templateTable_arm.cpp b/src/hotspot/cpu/arm/templateTable_arm.cpp
index 30649a5e104..85bf0ce812e 100644
--- a/src/hotspot/cpu/arm/templateTable_arm.cpp
+++ b/src/hotspot/cpu/arm/templateTable_arm.cpp
@@ -490,29 +490,30 @@ void TemplateTable::ldc2_w() {
   __ add(Rtemp, Rtags, tags_offset);
   __ ldrb(Rtemp, Address(Rtemp, Rindex));
 
-  Label Condy, exit;
-#ifdef __ABI_HARD__
-  Label NotDouble;
+  Label Done, NotLong, NotDouble;
   __ cmp(Rtemp, JVM_CONSTANT_Double);
   __ b(NotDouble, ne);
+#ifdef __SOFTFP__
+  __ ldr(R0_tos_lo, Address(Rbase, base_offset + 0 * wordSize));
+  __ ldr(R1_tos_hi, Address(Rbase, base_offset + 1 * wordSize));
+#else // !__SOFTFP__
   __ ldr_double(D0_tos, Address(Rbase, base_offset));
-
+#endif // __SOFTFP__
   __ push(dtos);
-  __ b(exit);
+  __ b(Done);
   __ bind(NotDouble);
-#endif
 
   __ cmp(Rtemp, JVM_CONSTANT_Long);
-  __ b(Condy, ne);
+  __ b(NotLong, ne);
   __ ldr(R0_tos_lo, Address(Rbase, base_offset + 0 * wordSize));
   __ ldr(R1_tos_hi, Address(Rbase, base_offset + 1 * wordSize));
   __ push(ltos);
-  __ b(exit);
+  __ b(Done);
+  __ bind(NotLong);
 
-  __ bind(Condy);
-  condy_helper(exit);
+  condy_helper(Done);
 
-  __ bind(exit);
+  __ bind(Done);
 }
 
 
@@ -3967,11 +3968,7 @@ void TemplateTable::_new() {
 
     // initialize object header only.
     __ bind(initialize_header);
-    if (UseBiasedLocking) {
-      __ ldr(Rtemp, Address(Rklass, Klass::prototype_header_offset()));
-    } else {
-      __ mov_slow(Rtemp, (intptr_t)markWord::prototype().value());
-    }
+    __ mov_slow(Rtemp, (intptr_t)markWord::prototype().value());
     // mark
     __ str(Rtemp, Address(Robj, oopDesc::mark_offset_in_bytes()));
 
diff --git a/src/hotspot/cpu/arm/vm_version_arm.hpp b/src/hotspot/cpu/arm/vm_version_arm.hpp
index 8b50d6d21e0..a6e2dc8b442 100644
--- a/src/hotspot/cpu/arm/vm_version_arm.hpp
+++ b/src/hotspot/cpu/arm/vm_version_arm.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -94,8 +94,6 @@ class VM_Version: public Abstract_VM_Version {
   static bool supports_compare_and_exchange() { return true; }
   static bool supports_kuser_cmpxchg32() { return _kuser_helper_version >= KUSER_VERSION_CMPXCHG32; }
   static bool supports_kuser_cmpxchg64() { return _kuser_helper_version >= KUSER_VERSION_CMPXCHG64; }
-  // Override Abstract_VM_Version implementation
-  static bool use_biased_locking();
 
   static bool has_vfp()             { return (_features & vfp_m) != 0; }
   static bool has_vfp3_32()         { return (_features & vfp3_32_m) != 0; }
diff --git a/src/hotspot/cpu/arm/vm_version_arm_32.cpp b/src/hotspot/cpu/arm/vm_version_arm_32.cpp
index e6fd8b98668..73d64d02fe7 100644
--- a/src/hotspot/cpu/arm/vm_version_arm_32.cpp
+++ b/src/hotspot/cpu/arm/vm_version_arm_32.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -348,16 +348,3 @@ void VM_Version::initialize() {
 
   _is_initialized = true;
 }
-
-bool VM_Version::use_biased_locking() {
-  get_os_cpu_info();
-  // The cost of CAS on uniprocessor ARM v6 and later is low compared to the
-  // overhead related to slightly longer Biased Locking execution path.
-  // Testing shows no improvement when running with Biased Locking enabled
-  // on an ARMv6 and higher uniprocessor systems.  The situation is different on
-  // ARMv5 and MP systems.
-  //
-  // Therefore the Biased Locking is enabled on ARMv5 and ARM MP only.
-  //
-  return (!os::is_MP() && (arm_arch() > 5)) ? false : true;
-}
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.cpp b/src/hotspot/cpu/ppc/assembler_ppc.cpp
index 6a6be870991..ff6788f6241 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2015 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -30,7 +30,6 @@
 #include "interpreter/interpreter.hpp"
 #include "memory/resourceArea.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index ba5c55f6b8a..d6d58262de5 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -576,6 +576,7 @@ class Assembler : public AbstractAssembler {
     XVNMSUBASP_OPCODE=(60u<< OPCODE_SHIFT |  209u << 3),
     XVNMSUBADP_OPCODE=(60u<< OPCODE_SHIFT |  241u << 3),
     XVRDPI_OPCODE  = (60u << OPCODE_SHIFT |  201u << 2),
+    XVRDPIC_OPCODE = (60u << OPCODE_SHIFT |  235u << 2),
     XVRDPIM_OPCODE = (60u << OPCODE_SHIFT |  249u << 2),
     XVRDPIP_OPCODE = (60u << OPCODE_SHIFT |  233u << 2),
 
@@ -823,6 +824,10 @@ class Assembler : public AbstractAssembler {
     // Prefixed addi/li
     PADDI_PREFIX_OPCODE   = PREFIX_PRIMARY_OPCODE | (2u << PRE_TYPE_SHIFT),
     PADDI_SUFFIX_OPCODE   = ADDI_OPCODE,
+
+    // xxpermx
+    XXPERMX_PREFIX_OPCODE = PREFIX_PRIMARY_OPCODE | (1u << PRE_TYPE_SHIFT),
+    XXPERMX_SUFFIX_OPCODE = (34u << OPCODE_SHIFT),
   };
 
   // Trap instructions TO bits
@@ -2347,6 +2352,7 @@ class Assembler : public AbstractAssembler {
   inline void mtvrd(    VectorRegister  d, Register a);
   inline void mfvrd(    Register        a, VectorRegister d);
   inline void xxperm(   VectorSRegister d, VectorSRegister a, VectorSRegister b);
+  inline void xxpermx(  VectorSRegister d, VectorSRegister a, VectorSRegister b, VectorSRegister c, int ui3);
   inline void xxpermdi( VectorSRegister d, VectorSRegister a, VectorSRegister b, int dm);
   inline void xxmrghw(  VectorSRegister d, VectorSRegister a, VectorSRegister b);
   inline void xxmrglw(  VectorSRegister d, VectorSRegister a, VectorSRegister b);
@@ -2384,6 +2390,7 @@ class Assembler : public AbstractAssembler {
   inline void xvnmsubasp(VectorSRegister d, VectorSRegister a, VectorSRegister b);
   inline void xvnmsubadp(VectorSRegister d, VectorSRegister a, VectorSRegister b);
   inline void xvrdpi(   VectorSRegister d, VectorSRegister b);
+  inline void xvrdpic(  VectorSRegister d, VectorSRegister b);
   inline void xvrdpim(  VectorSRegister d, VectorSRegister b);
   inline void xvrdpip(  VectorSRegister d, VectorSRegister b);
 
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 79709d01bf1..2f28ac06751 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -142,6 +142,11 @@ inline void Assembler::paddi_r0ok(Register d, Register a, long si34, bool r = fa
   emit_int32(PADDI_SUFFIX_OPCODE | rt(d)   | ra(a)   | d1_eo(si34));
 }
 
+inline void Assembler::xxpermx( VectorSRegister d, VectorSRegister a, VectorSRegister b, VectorSRegister c, int ui3) {
+  emit_int32(XXPERMX_PREFIX_OPCODE | uimm(ui3, 3));
+  emit_int32(XXPERMX_SUFFIX_OPCODE | vsrt(d) | vsra(a) | vsrb(b) | vsrc(c));
+}
+
 // Fixed-Point Arithmetic Instructions with Overflow detection
 inline void Assembler::addo(    Register d, Register a, Register b) { emit_int32(ADD_OPCODE    | rt(d) | ra(a) | rb(b) | oe(1) | rc(0)); }
 inline void Assembler::addo_(   Register d, Register a, Register b) { emit_int32(ADD_OPCODE    | rt(d) | ra(a) | rb(b) | oe(1) | rc(1)); }
@@ -848,6 +853,7 @@ inline void Assembler::xvmsubadp( VectorSRegister d, VectorSRegister a, VectorSR
 inline void Assembler::xvnmsubasp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVNMSUBASP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
 inline void Assembler::xvnmsubadp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVNMSUBADP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
 inline void Assembler::xvrdpi(    VectorSRegister d, VectorSRegister b)                  { emit_int32( XVRDPI_OPCODE  | vsrt(d) | vsrb(b)); }
+inline void Assembler::xvrdpic(   VectorSRegister d, VectorSRegister b)                  { emit_int32( XVRDPIC_OPCODE | vsrt(d) | vsrb(b)); }
 inline void Assembler::xvrdpim(   VectorSRegister d, VectorSRegister b)                  { emit_int32( XVRDPIM_OPCODE | vsrt(d) | vsrb(b)); }
 inline void Assembler::xvrdpip(   VectorSRegister d, VectorSRegister b)                  { emit_int32( XVRDPIP_OPCODE | vsrt(d) | vsrb(b)); }
 
diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
index 8eedca7886c..65df14c2c8a 100644
--- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
@@ -491,12 +491,14 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->verify_oop_map(info());
 
 #ifndef PRODUCT
-  const address counter = (address)&Runtime1::_arraycopy_slowcase_cnt;
-  const Register tmp = R3, tmp2 = R4;
-  int simm16_offs = __ load_const_optimized(tmp, counter, tmp2, true);
-  __ lwz(tmp2, simm16_offs, tmp);
-  __ addi(tmp2, tmp2, 1);
-  __ stw(tmp2, simm16_offs, tmp);
+  if (PrintC1Statistics) {
+    const address counter = (address)&Runtime1::_arraycopy_slowcase_cnt;
+    const Register tmp = R3, tmp2 = R4;
+    int simm16_offs = __ load_const_optimized(tmp, counter, tmp2, true);
+    __ lwz(tmp2, simm16_offs, tmp);
+    __ addi(tmp2, tmp2, 1);
+    __ stw(tmp2, simm16_offs, tmp);
+  }
 #endif
 
   __ b(_continuation);
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 5aaff199a24..6d8acbe846c 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -714,7 +714,7 @@ void LIR_Assembler::explicit_null_check(Register addr, CodeEmitInfo* info) {
 
 
 // Attention: caller must encode oop if needed
-int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide, bool unaligned) {
+int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide) {
   int store_offset;
   if (!Assembler::is_simm16(offset)) {
     // For offsets larger than a simm16 we setup the offset.
@@ -794,7 +794,7 @@ int LIR_Assembler::store(LIR_Opr from_reg, Register base, Register disp, BasicTy
 }
 
 
-int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide, bool unaligned) {
+int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide) {
   int load_offset;
   if (!Assembler::is_simm16(offset)) {
     // For offsets larger than a simm16 we setup the offset.
@@ -965,7 +965,7 @@ void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmi
     offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
   } else {
     assert(Assembler::is_simm16(addr->disp()), "can't handle larger addresses");
-    offset = store(tmp, base, addr->disp(), type, wide, false);
+    offset = store(tmp, base, addr->disp(), type, wide);
   }
 
   if (info != NULL) {
@@ -1120,7 +1120,7 @@ Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 
 
 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type,
-                            LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool unaligned) {
+                            LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 
   assert(type != T_METADATA, "load of metadata ptr not supported");
   LIR_Address* addr = src_opr->as_address_ptr();
@@ -1170,9 +1170,8 @@ void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type,
 
   if (disp_reg == noreg) {
     assert(Assembler::is_simm16(disp_value), "should have set this up");
-    offset = load(src, disp_value, to_reg, type, wide, unaligned);
+    offset = load(src, disp_value, to_reg, type, wide);
   } else {
-    assert(!unaligned, "unexpected");
     offset = load(src, disp_reg, to_reg, type, wide);
   }
 
@@ -1193,8 +1192,7 @@ void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
     addr = frame_map()->address_for_double_slot(src->double_stack_ix());
   }
 
-  bool unaligned = addr.disp() % 8 != 0;
-  load(addr.base(), addr.disp(), dest, dest->type(), true /*wide*/, unaligned);
+  load(addr.base(), addr.disp(), dest, dest->type(), true /*wide*/);
 }
 
 
@@ -1205,8 +1203,8 @@ void LIR_Assembler::reg2stack(LIR_Opr from_reg, LIR_Opr dest, BasicType type, bo
   } else if (dest->is_double_word())  {
     addr = frame_map()->address_for_slot(dest->double_stack_ix());
   }
-  bool unaligned = addr.disp() % 8 != 0;
-  store(from_reg, addr.base(), addr.disp(), from_reg->type(), true /*wide*/, unaligned);
+
+  store(from_reg, addr.base(), addr.disp(), from_reg->type(), true /*wide*/);
 }
 
 
@@ -1242,7 +1240,7 @@ void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
 
 void LIR_Assembler::reg2mem(LIR_Opr from_reg, LIR_Opr dest, BasicType type,
                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
-                            bool wide, bool unaligned) {
+                            bool wide) {
   assert(type != T_METADATA, "store of metadata ptr not supported");
   LIR_Address* addr = dest->as_address_ptr();
 
@@ -1299,9 +1297,8 @@ void LIR_Assembler::reg2mem(LIR_Opr from_reg, LIR_Opr dest, BasicType type,
 
   if (disp_reg == noreg) {
     assert(Assembler::is_simm16(disp_value), "should have set this up");
-    offset = store(from_reg, src, disp_value, type, wide, unaligned);
+    offset = store(from_reg, src, disp_value, type, wide);
   } else {
-    assert(!unaligned, "unexpected");
     offset = store(from_reg, src, disp_reg, type, wide);
   }
 
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp
index 861430b79eb..1cb40d63fdd 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.hpp
@@ -38,10 +38,10 @@
 
   void explicit_null_check(Register addr, CodeEmitInfo* info);
 
-  int store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide, bool unaligned);
+  int store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide);
   int store(LIR_Opr from_reg, Register base, Register disp, BasicType type, bool wide);
 
-  int load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide, bool unaligned);
+  int load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide);
   int load(Register base, Register disp, LIR_Opr to_reg, BasicType type, bool wide);
 
   int shift_amount(BasicType t);
diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
index 2c685920367..6d71c87d3e7 100644
--- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
@@ -309,12 +309,7 @@ bool LIRGenerator::strength_reduce_multiply(LIR_Opr left, jint c, LIR_Opr result
 void LIRGenerator::store_stack_parameter(LIR_Opr item, ByteSize offset_from_sp) {
   BasicType t = item->type();
   LIR_Opr sp_opr = FrameMap::SP_opr;
-  if ((t == T_LONG || t == T_DOUBLE) &&
-      (in_bytes(offset_from_sp) % 8 != 0)) {
-    __ unaligned_move(item, new LIR_Address(sp_opr, in_bytes(offset_from_sp), t));
-  } else {
-    __ move(item, new LIR_Address(sp_opr, in_bytes(offset_from_sp), t));
-  }
+  __ move(item, new LIR_Address(sp_opr, in_bytes(offset_from_sp), t));
 }
 
 
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 47cee695490..2e7fb125eb3 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -33,7 +33,6 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
@@ -115,10 +114,6 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
     bne(CCR0, slow_int);
   }
 
-  if (UseBiasedLocking) {
-    biased_locking_enter(CCR0, Roop, Rmark, Rscratch, R0, done, &slow_int);
-  }
-
   // ... and mark it unlocked.
   ori(Rmark, Rmark, markWord::unlocked_value);
 
@@ -164,21 +159,14 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
   Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
   assert(mark_addr.disp() == 0, "cas must take a zero displacement");
 
-  if (UseBiasedLocking) {
-    // Load the object out of the BasicObjectLock.
-    ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
-    verify_oop(Roop, FILE_AND_LINE);
-    biased_locking_exit(CCR0, Roop, R0, done);
-  }
   // Test first it it is a fast recursive unlock.
   ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
   cmpdi(CCR0, Rmark, 0);
   beq(CCR0, done);
-  if (!UseBiasedLocking) {
-    // Load object.
-    ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
-    verify_oop(Roop, FILE_AND_LINE);
-  }
+
+  // Load object.
+  ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
+  verify_oop(Roop, FILE_AND_LINE);
 
   // Check if it is still a light weight lock, this is is true if we see
   // the stack address of the basicLock in the markWord of the object.
@@ -222,11 +210,7 @@ void C1_MacroAssembler::try_allocate(
 
 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
   assert_different_registers(obj, klass, len, t1, t2);
-  if (UseBiasedLocking && !len->is_valid()) {
-    ld(t1, in_bytes(Klass::prototype_header_offset()), klass);
-  } else {
-    load_const_optimized(t1, (intx)markWord::prototype().value());
-  }
+  load_const_optimized(t1, (intx)markWord::prototype().value());
   std(t1, oopDesc::mark_offset_in_bytes(), obj);
   store_klass(obj, klass);
   if (len->is_valid()) {
diff --git a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
index bb103cdf609..00a92ff6b62 100644
--- a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         3);
-define_pd_global(intx, FLOATPRESSURE,                28);
 define_pd_global(intx, FreqInlineSize,               175);
 define_pd_global(intx, MinJumpTableSize,             10);
-define_pd_global(intx, INTPRESSURE,                  26);
 define_pd_global(intx, InteriorEntryAlignment,       16);
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 define_pd_global(intx, RegisterCostAreaRatio,        16000);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp
index 0f4e34dc9e7..36585185812 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.cpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.cpp
@@ -197,6 +197,16 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return fr;
 }
 
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
+  ShouldNotCallThis();
+  return nullptr;
+}
+
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_interpreter_frame(RegisterMap *map) const {
   // Pass callers initial_caller_sp as unextended_sp.
   return frame(sender_sp(), sender_pc(), (intptr_t*)get_ijava_state()->sender_sp);
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
index a95197aeed7..a9328e8d616 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
@@ -393,7 +393,7 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
 
   __ cmpdi(CCR0, pre_val_reg, 0);
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
index 5810e12139f..ff4b8e80ca8 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
@@ -943,10 +943,6 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
       bne(CCR0, slow_case);
     }
 
-    if (UseBiasedLocking) {
-      biased_locking_enter(CCR0, object, displaced_header, tmp, current_header, done, &slow_case);
-    }
-
     // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
     ori(displaced_header, displaced_header, markWord::unlocked_value);
 
@@ -1048,13 +1044,6 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {
 
     assert_different_registers(object, displaced_header, object_mark_addr, current_header);
 
-    if (UseBiasedLocking) {
-      // The object address from the monitor is in object.
-      ld(object, BasicObjectLock::obj_offset_in_bytes(), monitor);
-      assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-      biased_locking_exit(CCR0, object, displaced_header, free_slot);
-    }
-
     // Test first if we are in the fast recursive case.
     ld(displaced_header, BasicObjectLock::lock_offset_in_bytes() +
            BasicLock::displaced_header_offset_in_bytes(), monitor);
@@ -1070,7 +1059,7 @@ void InterpreterMacroAssembler::unlock_object(Register monitor) {
     // If we still have a lightweight lock, unlock the object and be done.
 
     // The object address from the monitor is in object.
-    if (!UseBiasedLocking) { ld(object, BasicObjectLock::obj_offset_in_bytes(), monitor); }
+    ld(object, BasicObjectLock::obj_offset_in_bytes(), monitor);
     addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
 
     // We have the displaced header in displaced_header. If the lock is still
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 918ad1d7c89..98565003691 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -35,7 +35,6 @@
 #include "oops/klass.inline.hpp"
 #include "oops/methodData.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/icache.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
@@ -106,6 +105,10 @@ void MacroAssembler::align(int modulus, int max, int rem) {
   for (int c = (padding >> 2); c > 0; --c) { nop(); }
 }
 
+void MacroAssembler::align_prefix() {
+  if (is_aligned(offset() + BytesPerInstWord, 64)) { nop(); }
+}
+
 // Issue instructions that calculate given TOC from global TOC.
 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16,
                                                        bool add_relocation, bool emit_dummy_addr) {
@@ -2073,218 +2076,6 @@ RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
   }
 }
 
-// Supports temp2_reg = R0.
-void MacroAssembler::biased_locking_enter(ConditionRegister cr_reg, Register obj_reg,
-                                          Register mark_reg, Register temp_reg,
-                                          Register temp2_reg, Label& done, Label* slow_case) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-
-#ifdef ASSERT
-  assert_different_registers(obj_reg, mark_reg, temp_reg, temp2_reg);
-#endif
-
-  Label cas_label;
-
-  // Branch to done if fast path fails and no slow_case provided.
-  Label *slow_case_int = (slow_case != NULL) ? slow_case : &done;
-
-  // Biased locking
-  // See whether the lock is currently biased toward our thread and
-  // whether the epoch is still valid
-  // Note that the runtime guarantees sufficient alignment of JavaThread
-  // pointers to allow age to be placed into low bits
-  assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits,
-         "biased locking makes assumptions about bit layout");
-
-  if (PrintBiasedLockingStatistics) {
-    load_const(temp2_reg, (address) BiasedLocking::total_entry_count_addr(), temp_reg);
-    lwzx(temp_reg, temp2_reg);
-    addi(temp_reg, temp_reg, 1);
-    stwx(temp_reg, temp2_reg);
-  }
-
-  andi(temp_reg, mark_reg, markWord::biased_lock_mask_in_place);
-  cmpwi(cr_reg, temp_reg, markWord::biased_lock_pattern);
-  bne(cr_reg, cas_label);
-
-  load_klass(temp_reg, obj_reg);
-
-  load_const_optimized(temp2_reg, ~((int) markWord::age_mask_in_place));
-  ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
-  orr(temp_reg, R16_thread, temp_reg);
-  xorr(temp_reg, mark_reg, temp_reg);
-  andr(temp_reg, temp_reg, temp2_reg);
-  cmpdi(cr_reg, temp_reg, 0);
-  if (PrintBiasedLockingStatistics) {
-    Label l;
-    bne(cr_reg, l);
-    load_const(temp2_reg, (address) BiasedLocking::biased_lock_entry_count_addr());
-    lwzx(mark_reg, temp2_reg);
-    addi(mark_reg, mark_reg, 1);
-    stwx(mark_reg, temp2_reg);
-    // restore mark_reg
-    ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg);
-    bind(l);
-  }
-  beq(cr_reg, done);
-
-  Label try_revoke_bias;
-  Label try_rebias;
-
-  // At this point we know that the header has the bias pattern and
-  // that we are not the bias owner in the current epoch. We need to
-  // figure out more details about the state of the header in order to
-  // know what operations can be legally performed on the object's
-  // header.
-
-  // If the low three bits in the xor result aren't clear, that means
-  // the prototype header is no longer biased and we have to revoke
-  // the bias on this object.
-  andi(temp2_reg, temp_reg, markWord::biased_lock_mask_in_place);
-  cmpwi(cr_reg, temp2_reg, 0);
-  bne(cr_reg, try_revoke_bias);
-
-  // Biasing is still enabled for this data type. See whether the
-  // epoch of the current bias is still valid, meaning that the epoch
-  // bits of the mark word are equal to the epoch bits of the
-  // prototype header. (Note that the prototype header's epoch bits
-  // only change at a safepoint.) If not, attempt to rebias the object
-  // toward the current thread. Note that we must be absolutely sure
-  // that the current epoch is invalid in order to do this because
-  // otherwise the manipulations it performs on the mark word are
-  // illegal.
-
-  int shift_amount = 64 - markWord::epoch_shift;
-  // rotate epoch bits to right (little) end and set other bits to 0
-  // [ big part | epoch | little part ] -> [ 0..0 | epoch ]
-  rldicl_(temp2_reg, temp_reg, shift_amount, 64 - markWord::epoch_bits);
-  // branch if epoch bits are != 0, i.e. they differ, because the epoch has been incremented
-  bne(CCR0, try_rebias);
-
-  // The epoch of the current bias is still valid but we know nothing
-  // about the owner; it might be set or it might be clear. Try to
-  // acquire the bias of the object using an atomic operation. If this
-  // fails we will go in to the runtime to revoke the object's bias.
-  // Note that we first construct the presumed unbiased header so we
-  // don't accidentally blow away another thread's valid bias.
-  andi(mark_reg, mark_reg, (markWord::biased_lock_mask_in_place |
-                                markWord::age_mask_in_place |
-                                markWord::epoch_mask_in_place));
-  orr(temp_reg, R16_thread, mark_reg);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg).
-  cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
-           /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
-           /*where=*/obj_reg,
-           MacroAssembler::MemBarAcq,
-           MacroAssembler::cmpxchgx_hint_acquire_lock(),
-           noreg, slow_case_int); // bail out if failed
-
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-  if (PrintBiasedLockingStatistics) {
-    load_const(temp2_reg, (address) BiasedLocking::anonymously_biased_lock_entry_count_addr(), temp_reg);
-    lwzx(temp_reg, temp2_reg);
-    addi(temp_reg, temp_reg, 1);
-    stwx(temp_reg, temp2_reg);
-  }
-  b(done);
-
-  bind(try_rebias);
-  // At this point we know the epoch has expired, meaning that the
-  // current "bias owner", if any, is actually invalid. Under these
-  // circumstances _only_, we are allowed to use the current header's
-  // value as the comparison value when doing the cas to acquire the
-  // bias in the current epoch. In other words, we allow transfer of
-  // the bias from one thread to another directly in this situation.
-  load_klass(temp_reg, obj_reg);
-  andi(temp2_reg, mark_reg, markWord::age_mask_in_place);
-  orr(temp2_reg, R16_thread, temp2_reg);
-  ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
-  orr(temp_reg, temp2_reg, temp_reg);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
-                 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
-                 /*where=*/obj_reg,
-                 MacroAssembler::MemBarAcq,
-                 MacroAssembler::cmpxchgx_hint_acquire_lock(),
-                 noreg, slow_case_int); // bail out if failed
-
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-  if (PrintBiasedLockingStatistics) {
-    load_const(temp2_reg, (address) BiasedLocking::rebiased_lock_entry_count_addr(), temp_reg);
-    lwzx(temp_reg, temp2_reg);
-    addi(temp_reg, temp_reg, 1);
-    stwx(temp_reg, temp2_reg);
-  }
-  b(done);
-
-  bind(try_revoke_bias);
-  // The prototype mark in the klass doesn't have the bias bit set any
-  // more, indicating that objects of this data type are not supposed
-  // to be biased any more. We are going to try to reset the mark of
-  // this object to the prototype value and fall through to the
-  // CAS-based locking scheme. Note that if our CAS fails, it means
-  // that another thread raced us for the privilege of revoking the
-  // bias of this particular object, so it's okay to continue in the
-  // normal locking code.
-  load_klass(temp_reg, obj_reg);
-  ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
-  andi(temp2_reg, mark_reg, markWord::age_mask_in_place);
-  orr(temp_reg, temp_reg, temp2_reg);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg).
-  cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
-                 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
-                 /*where=*/obj_reg,
-                 MacroAssembler::MemBarAcq,
-                 MacroAssembler::cmpxchgx_hint_acquire_lock());
-
-  // reload markWord in mark_reg before continuing with lightweight locking
-  ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg);
-
-  // Fall through to the normal CAS-based lock, because no matter what
-  // the result of the above CAS, some thread must have succeeded in
-  // removing the bias bit from the object's header.
-  if (PrintBiasedLockingStatistics) {
-    Label l;
-    bne(cr_reg, l);
-    load_const(temp2_reg, (address) BiasedLocking::revoked_lock_entry_count_addr(), temp_reg);
-    lwzx(temp_reg, temp2_reg);
-    addi(temp_reg, temp_reg, 1);
-    stwx(temp_reg, temp2_reg);
-    bind(l);
-  }
-
-  bind(cas_label);
-}
-
-void MacroAssembler::biased_locking_exit (ConditionRegister cr_reg, Register mark_addr, Register temp_reg, Label& done) {
-  // Check for biased locking unlock case, which is a no-op
-  // Note: we do not have to check the thread ID for two reasons.
-  // First, the interpreter checks for IllegalMonitorStateException at
-  // a higher level. Second, if the bias was revoked while we held the
-  // lock, the object could not be rebiased toward another thread, so
-  // the bias bit would be clear.
-
-  ld(temp_reg, 0, mark_addr);
-  andi(temp_reg, temp_reg, markWord::biased_lock_mask_in_place);
-
-  cmpwi(cr_reg, temp_reg, markWord::biased_lock_pattern);
-  beq(cr_reg, done);
-}
-
 // allocation (for C1)
 void MacroAssembler::eden_allocate(
   Register obj,                      // result: pointer to object after successful allocation
@@ -2695,14 +2486,13 @@ void MacroAssembler::rtm_stack_locking(ConditionRegister flag,
                                        Metadata* method_data, bool profile_rtm,
                                        Label& DONE_LABEL, Label& IsInflated) {
   assert(UseRTMForStackLocks, "why call this otherwise?");
-  assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
   Label L_rtm_retry, L_decrement_retry, L_on_abort;
 
   if (RTMRetryCount > 0) {
     load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
     bind(L_rtm_retry);
   }
-  andi_(R0, mark_word, markWord::monitor_value);  // inflated vs stack-locked|neutral|biased
+  andi_(R0, mark_word, markWord::monitor_value);  // inflated vs stack-locked|neutral
   bne(CCR0, IsInflated);
 
   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
@@ -2720,10 +2510,10 @@ void MacroAssembler::rtm_stack_locking(ConditionRegister flag,
   }
   tbegin_();
   beq(CCR0, L_on_abort);
-  ld(mark_word, oopDesc::mark_offset_in_bytes(), obj);      // Reload in transaction, conflicts need to be tracked.
-  andi(R0, mark_word, markWord::biased_lock_mask_in_place); // look at 3 lock bits
-  cmpwi(flag, R0, markWord::unlocked_value);                // bits = 001 unlocked
-  beq(flag, DONE_LABEL);                                    // all done if unlocked
+  ld(mark_word, oopDesc::mark_offset_in_bytes(), obj);   // Reload in transaction, conflicts need to be tracked.
+  andi(R0, mark_word, markWord::lock_mask_in_place);     // look at 2 lock bits
+  cmpwi(flag, R0, markWord::unlocked_value);             // bits = 01 unlocked
+  beq(flag, DONE_LABEL);                                 // all done if unlocked
 
   if (UseRTMXendForLockBusy) {
     tend_();
@@ -2837,7 +2627,6 @@ void MacroAssembler::rtm_inflated_locking(ConditionRegister flag,
 // "The box" is the space on the stack where we copy the object mark.
 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
                                                Register temp, Register displaced_header, Register current_header,
-                                               bool try_bias,
                                                RTMLockingCounters* rtm_counters,
                                                RTMLockingCounters* stack_rtm_counters,
                                                Metadata* method_data,
@@ -2858,10 +2647,6 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
     bne(flag, cont);
   }
 
-  if (try_bias) {
-    biased_locking_enter(flag, oop, displaced_header, temp, current_header, cont);
-  }
-
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
     rtm_stack_locking(flag, oop, displaced_header, temp, /*temp*/ current_header,
@@ -2964,26 +2749,21 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
 
 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
                                                  Register temp, Register displaced_header, Register current_header,
-                                                 bool try_bias, bool use_rtm) {
+                                                 bool use_rtm) {
   assert_different_registers(oop, box, temp, displaced_header, current_header);
   assert(flag != CCR0, "bad condition register");
   Label cont;
   Label object_has_monitor;
 
-  if (try_bias) {
-    biased_locking_exit(flag, oop, current_header, cont);
-  }
-
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
-    assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
     Label L_regular_unlock;
-    ld(current_header, oopDesc::mark_offset_in_bytes(), oop);      // fetch markword
-    andi(R0, current_header, markWord::biased_lock_mask_in_place); // look at 3 lock bits
-    cmpwi(flag, R0, markWord::unlocked_value);                     // bits = 001 unlocked
-    bne(flag, L_regular_unlock);                                   // else RegularLock
-    tend_();                                                       // otherwise end...
-    b(cont);                                                       // ... and we're done
+    ld(current_header, oopDesc::mark_offset_in_bytes(), oop);   // fetch markword
+    andi(R0, current_header, markWord::lock_mask_in_place);     // look at 2 lock bits
+    cmpwi(flag, R0, markWord::unlocked_value);                  // bits = 01 unlocked
+    bne(flag, L_regular_unlock);                                // else RegularLock
+    tend_();                                                    // otherwise end...
+    b(cont);                                                    // ... and we're done
     bind(L_regular_unlock);
   }
 #endif
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 55cefd28436..5c53beede16 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -88,6 +88,16 @@ class MacroAssembler: public Assembler {
   // nop padding
   void align(int modulus, int max = 252, int rem = 0);
 
+  // Align prefix opcode to make sure it's not on the last word of a
+  // 64-byte block.
+  //
+  // Note: do not call align_prefix() in a .ad file (e.g. ppc.ad).  Instead
+  // add ins_alignment(2) to the instruct definition and implement the
+  // compute_padding() method of the instruct node to use
+  // compute_prefix_padding().  See loadConI32Node::compute_padding() in
+  // ppc.ad for an example.
+  void align_prefix();
+
   //
   // Constants, loading constants, TOC support
   //
@@ -589,24 +599,6 @@ class MacroAssembler: public Assembler {
   // Method handle support (JSR 292).
   RegisterOrConstant argument_offset(RegisterOrConstant arg_slot, Register temp_reg, int extra_slot_offset = 0);
 
-  // Biased locking support
-  // Upon entry,obj_reg must contain the target object, and mark_reg
-  // must contain the target object's header.
-  // Destroys mark_reg if an attempt is made to bias an anonymously
-  // biased lock. In this case a failure will go either to the slow
-  // case or fall through with the notEqual condition code set with
-  // the expectation that the slow case in the runtime will be called.
-  // In the fall-through case where the CAS-based lock is done,
-  // mark_reg is not destroyed.
-  void biased_locking_enter(ConditionRegister cr_reg, Register obj_reg, Register mark_reg, Register temp_reg,
-                            Register temp2_reg, Label& done, Label* slow_case = NULL);
-  // Upon entry, the base register of mark_addr must contain the oop.
-  // Destroys temp_reg.
-  // If allow_delay_slot_filling is set to true, the next instruction
-  // emitted after this one will go in an annulled delay slot if the
-  // biased locking exit case failed.
-  void biased_locking_exit(ConditionRegister cr_reg, Register mark_addr, Register temp_reg, Label& done);
-
   // allocation (for C1)
   void eden_allocate(
     Register obj,                      // result: pointer to object after successful allocation
@@ -655,7 +647,6 @@ class MacroAssembler: public Assembler {
 
   void compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
                                  Register tmp1, Register tmp2, Register tmp3,
-                                 bool try_bias = UseBiasedLocking,
                                  RTMLockingCounters* rtm_counters = NULL,
                                  RTMLockingCounters* stack_rtm_counters = NULL,
                                  Metadata* method_data = NULL,
@@ -663,7 +654,7 @@ class MacroAssembler: public Assembler {
 
   void compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
                                    Register tmp1, Register tmp2, Register tmp3,
-                                   bool try_bias = UseBiasedLocking, bool use_rtm = false);
+                                   bool use_rtm = false);
 
   // Check if safepoint requested and if so branch
   void safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod);
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 0b66ce7489a..9a883c72310 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -2108,6 +2108,8 @@ const bool Matcher::match_rule_supported(int opcode) {
   switch (opcode) {
     case Op_SqrtD:
       return VM_Version::has_fsqrt();
+    case Op_RoundDoubleMode:
+      return VM_Version::has_vsx();
     case Op_CountLeadingZerosI:
     case Op_CountLeadingZerosL:
       return UseCountLeadingZerosInstructionsPPC64;
@@ -2193,10 +2195,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Vector width in bytes.
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   if (SuperwordUseVSX) {
@@ -2272,7 +2270,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd,
   return NULL;
 }
 
-bool Matcher::is_generic_reg2reg_move(MachNode* m) {
+bool Matcher::is_reg2reg_move(MachNode* m) {
   ShouldNotReachHere();  // generic vector operands not supported
   return false;
 }
@@ -2363,6 +2361,16 @@ bool Matcher::is_spillable_arg(int reg) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 26 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }
@@ -12117,8 +12125,7 @@ instruct cmpFastLock(flagsReg crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1, iR
   format %{ "FASTLOCK  $oop, $box, $tmp1, $tmp2" %}
   ins_encode %{
     __ compiler_fast_lock_object($crx$$CondRegister, $oop$$Register, $box$$Register,
-                                 $tmp1$$Register, $tmp2$$Register, /*tmp3*/ R0,
-                                 UseBiasedLocking && !UseOptoBiasInlining);
+                                 $tmp1$$Register, $tmp2$$Register, /*tmp3*/ R0);
     // If locking was successfull, crx should indicate 'EQ'.
     // The compiler generates a branch to the runtime call to
     // _complete_monitor_locking_Java for the case where crx is 'NE'.
@@ -12136,10 +12143,9 @@ instruct cmpFastLock_tm(flagsReg crx, iRegPdst oop, rarg2RegP box, iRegPdst tmp1
   ins_encode %{
     __ compiler_fast_lock_object($crx$$CondRegister, $oop$$Register, $box$$Register,
                                  $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
-                                 /*Biased Locking*/ false,
                                  _rtm_counters, _stack_rtm_counters,
                                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
-                                 /*TM*/ true, ra_->C->profile_rtm());
+                                 /*RTM*/ true, ra_->C->profile_rtm());
     // If locking was successfull, crx should indicate 'EQ'.
     // The compiler generates a branch to the runtime call to
     // _complete_monitor_locking_Java for the case where crx is 'NE'.
@@ -12156,7 +12162,6 @@ instruct cmpFastUnlock(flagsReg crx, iRegPdst oop, iRegPdst box, iRegPdst tmp1,
   ins_encode %{
     __ compiler_fast_unlock_object($crx$$CondRegister, $oop$$Register, $box$$Register,
                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
-                                   UseBiasedLocking && !UseOptoBiasInlining,
                                    false);
     // If unlocking was successfull, crx should indicate 'EQ'.
     // The compiler generates a branch to the runtime call to
@@ -12174,7 +12179,7 @@ instruct cmpFastUnlock_tm(flagsReg crx, iRegPdst oop, iRegPdst box, iRegPdst tmp
   ins_encode %{
     __ compiler_fast_unlock_object($crx$$CondRegister, $oop$$Register, $box$$Register,
                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
-                                   /*Biased Locking*/ false, /*TM*/ true);
+                                   /*RTM*/ true);
     // If unlocking was successfull, crx should indicate 'EQ'.
     // The compiler generates a branch to the runtime call to
     // _complete_monitor_unlocking_Java for the case where crx is 'NE'.
@@ -13961,7 +13966,7 @@ instruct roundD_reg(regD dst, regD src, immI8 rmode) %{
   ins_encode %{
     switch ($rmode$$constant) {
       case RoundDoubleModeNode::rmode_rint:
-        __ frin($dst$$FloatRegister, $src$$FloatRegister);
+        __ xvrdpic($dst$$FloatRegister->to_vsr(), $src$$FloatRegister->to_vsr());
         break;
       case RoundDoubleModeNode::rmode_floor:
         __ frim($dst$$FloatRegister, $src$$FloatRegister);
@@ -13985,7 +13990,7 @@ instruct vround2D_reg(vecX dst, vecX src, immI8 rmode) %{
   ins_encode %{
     switch ($rmode$$constant) {
       case RoundDoubleModeNode::rmode_rint:
-        __ xvrdpi($dst$$VectorSRegister, $src$$VectorSRegister);
+        __ xvrdpic($dst$$VectorSRegister, $src$$VectorSRegister);
         break;
       case RoundDoubleModeNode::rmode_floor:
         __ xvrdpim($dst$$VectorSRegister, $src$$VectorSRegister);
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 94869ae7ca2..377df777511 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -2154,14 +2154,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
     // Get the lock box slot's address.
     __ addi(r_box, R1_SP, lock_offset);
 
-#   ifdef ASSERT
-    if (UseBiasedLocking) {
-      // Making the box point to itself will make it clear it went unused
-      // but also be obviously invalid.
-      __ std(r_box, 0, r_box);
-    }
-#   endif // ASSERT
-
     // Try fastpath for locking.
     // fast_lock kills r_temp_1, r_temp_2, r_temp_3.
     __ compiler_fast_lock_object(r_flag, r_oop, r_box, r_temp_1, r_temp_2, r_temp_3);
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index c4e60e9a513..0e68a38dbaf 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -3643,8 +3643,14 @@ class StubGenerator: public StubCodeGenerator {
 // Underscore (URL = 1)
 #define US  (signed char)((-'_' + 63) & 0xff)
 
+// For P10 (or later) only
+#define VALID_B64 0x80
+#define VB64(x) (VALID_B64 | x)
+
 #define VEC_ALIGN __attribute__ ((aligned(16)))
 
+#define BLK_OFFSETOF(x) (offsetof(constant_block, x))
+
 // In little-endian mode, the lxv instruction loads the element at EA into
 // element 15 of the the vector register, EA+1 goes into element 14, and so
 // on.
@@ -3660,95 +3666,123 @@ class StubGenerator: public StubCodeGenerator {
     StubCodeMark mark(this, "StubRoutines", "base64_decodeBlock");
     address start   = __ function_entry();
 
-    static const signed char VEC_ALIGN offsetLUT_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      0,   0, PLS, DIG,  UC,  UC,  LC,  LC,
-      0,   0,   0,   0,   0,   0,   0,   0 ) };
-
-    static const signed char VEC_ALIGN offsetLUT_URL_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      0,   0, HYP, DIG,  UC,  UC,  LC,  LC,
-      0,   0,   0,   0,   0,   0,   0,   0 ) };
-
-    static const unsigned char VEC_ALIGN maskLUT_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      /* 0        */ (unsigned char)0b10101000,
-      /* 1 .. 9   */ (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
-                     (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
-                     (unsigned char)0b11111000,
-      /* 10       */ (unsigned char)0b11110000,
-      /* 11       */ (unsigned char)0b01010100,
-      /* 12 .. 14 */ (unsigned char)0b01010000, (unsigned char)0b01010000, (unsigned char)0b01010000,
-      /* 15       */ (unsigned char)0b01010100 ) };
-
-    static const unsigned char VEC_ALIGN maskLUT_URL_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      /* 0        */ (unsigned char)0b10101000,
-      /* 1 .. 9   */ (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
-                     (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
-                     (unsigned char)0b11111000,
-      /* 10       */ (unsigned char)0b11110000,
-      /* 11 .. 12 */ (unsigned char)0b01010000, (unsigned char)0b01010000,
-      /* 13       */ (unsigned char)0b01010100,
-      /* 14       */ (unsigned char)0b01010000,
-      /* 15       */ (unsigned char)0b01110000 ) };
-
-    static const unsigned char VEC_ALIGN bitposLUT_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, (unsigned char)0x80,
-      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ) };
-
-    static const unsigned char VEC_ALIGN pack_lshift_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2 ) };
-
-    static const unsigned char VEC_ALIGN pack_rshift_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-      0, 2, 4, 0, 0, 2, 4, 0, 0, 2, 4, 0, 0, 2, 4, 0 ) };
-
-    // The first 4 index values are "don't care" because
-    // we only use the first 12 bytes of the vector,
-    // which are decoded from 16 bytes of Base64 characters.
-    static const unsigned char VEC_ALIGN pack_permute_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-       0, 0, 0, 0,
-       0,  1,  2,
-       4,  5,  6,
-       8,  9, 10,
-      12, 13, 14 ) };
-
-    static const unsigned char VEC_ALIGN p10_pack_permute_val[16] = {
-      ARRAY_TO_LXV_ORDER(
-       0,  0,  0,  0,  7,  6,  5,  4,
-       3,  2, 15, 14, 13, 12, 11, 10 ) };
-
-    // loop_unrolls needs to be a power of two so that the rounding can be
-    // done using a mask.
-    //
-    // The amount of loop unrolling was determined by running a benchmark
-    // that decodes a 20k block of Base64 data on a Power9 machine:
-    // loop_unrolls = 1 :
-    // (min, avg, max) = (108639.215, 110530.479, 110779.920), stdev = 568.437
-    // loop_unrolls = 2 :
-    // (min, avg, max) = (108259.029, 110174.202, 110399.642), stdev = 561.729
-    // loop_unrolls = 4 :
-    // (min, avg, max) = (106514.175, 108373.110, 108514.786), stdev = 392.237
-    // loop_unrolls = 8 :
-    // (min, avg, max) = (106281.283, 108316.668, 108539.953), stdev = 553.938
-    // loop_unrolls = 16 :
-    // (min, avg, max) = (108580.768, 110631.161, 110766.237), stdev = 430.510
-    //
-    // Comparing only the max values, there's no reason to go past
-    // loop_unrolls = 1.  Performance at loop_unrolls = 16 is similar but
-    // has the disadvantage of requiring a larger minimum block of data to
-    // work with.  A value of 1 gives a minimum of (16 + 12) = 28 bytes
-    // before the intrinsic will decode any data.  See the reason for the
-    // +12 in the following logic.
-    const unsigned loop_unrolls = 1;
-
-    const unsigned vec_size = 16; // size of vector registers in bytes
-    const unsigned block_size = vec_size * loop_unrolls;  // number of bytes to process in each pass through the loop
-    const unsigned block_size_shift = exact_log2(block_size);
+    typedef struct {
+      signed char offsetLUT_val[16];
+      signed char offsetLUT_URL_val[16];
+      unsigned char maskLUT_val[16];
+      unsigned char maskLUT_URL_val[16];
+      unsigned char bitposLUT_val[16];
+      unsigned char table_32_47_val[16];
+      unsigned char table_32_47_URL_val[16];
+      unsigned char table_48_63_val[16];
+      unsigned char table_64_79_val[16];
+      unsigned char table_80_95_val[16];
+      unsigned char table_80_95_URL_val[16];
+      unsigned char table_96_111_val[16];
+      unsigned char table_112_127_val[16];
+      unsigned char pack_lshift_val[16];
+      unsigned char pack_rshift_val[16];
+      unsigned char pack_permute_val[16];
+    } constant_block;
+
+    static const constant_block VEC_ALIGN const_block = {
+
+      .offsetLUT_val = {
+        ARRAY_TO_LXV_ORDER(
+        0,   0, PLS, DIG,  UC,  UC,  LC,  LC,
+        0,   0,   0,   0,   0,   0,   0,   0 ) },
+
+      .offsetLUT_URL_val = {
+        ARRAY_TO_LXV_ORDER(
+        0,   0, HYP, DIG,  UC,  UC,  LC,  LC,
+        0,   0,   0,   0,   0,   0,   0,   0 ) },
+
+      .maskLUT_val = {
+        ARRAY_TO_LXV_ORDER(
+        /* 0        */ (unsigned char)0b10101000,
+        /* 1 .. 9   */ (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
+                       (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
+                       (unsigned char)0b11111000,
+        /* 10       */ (unsigned char)0b11110000,
+        /* 11       */ (unsigned char)0b01010100,
+        /* 12 .. 14 */ (unsigned char)0b01010000, (unsigned char)0b01010000, (unsigned char)0b01010000,
+        /* 15       */ (unsigned char)0b01010100 ) },
+
+      .maskLUT_URL_val = {
+        ARRAY_TO_LXV_ORDER(
+        /* 0        */ (unsigned char)0b10101000,
+        /* 1 .. 9   */ (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
+                       (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000, (unsigned char)0b11111000,
+                       (unsigned char)0b11111000,
+        /* 10       */ (unsigned char)0b11110000,
+        /* 11 .. 12 */ (unsigned char)0b01010000, (unsigned char)0b01010000,
+        /* 13       */ (unsigned char)0b01010100,
+        /* 14       */ (unsigned char)0b01010000,
+        /* 15       */ (unsigned char)0b01110000 ) },
+
+      .bitposLUT_val = {
+        ARRAY_TO_LXV_ORDER(
+        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, (unsigned char)0x80,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ) },
+
+      // In the following table_*_val constants, a 0 value means the
+      // character is not in the Base64 character set
+      .table_32_47_val = {
+        ARRAY_TO_LXV_ORDER (
+         /* space .. '*' = 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '+' = 62 */ VB64(62), /* ',' .. '.' = 0 */ 0, 0, 0, /* '/' = 63 */ VB64(63) ) },
+
+      .table_32_47_URL_val = {
+        ARRAY_TO_LXV_ORDER(
+         /* space .. ',' = 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* '-' = 62 */ VB64(62), /* '.' .. '/' */ 0, 0 ) },
+
+      .table_48_63_val = {
+        ARRAY_TO_LXV_ORDER(
+         /* '0' .. '9' = 52 .. 61 */ VB64(52), VB64(53), VB64(54), VB64(55), VB64(56), VB64(57), VB64(58), VB64(59), VB64(60), VB64(61),
+         /* ':' .. '?' = 0 */ 0, 0, 0, 0, 0, 0 ) },
+
+      .table_64_79_val = {
+        ARRAY_TO_LXV_ORDER(
+         /* '@' = 0 */ 0, /* 'A' .. 'O' = 0 .. 14 */ VB64(0), VB64(1), VB64(2), VB64(3), VB64(4), VB64(5), VB64(6), VB64(7), VB64(8),
+         VB64(9), VB64(10), VB64(11), VB64(12), VB64(13), VB64(14) ) },
+
+      .table_80_95_val = {
+        ARRAY_TO_LXV_ORDER(/* 'P' .. 'Z' = 15 .. 25 */ VB64(15), VB64(16), VB64(17), VB64(18), VB64(19), VB64(20), VB64(21), VB64(22),
+        VB64(23), VB64(24), VB64(25), /* '[' .. '_' = 0 */ 0, 0, 0, 0, 0 ) },
+
+      .table_80_95_URL_val = {
+        ARRAY_TO_LXV_ORDER(/* 'P' .. 'Z' = 15 .. 25 */ VB64(15), VB64(16), VB64(17), VB64(18), VB64(19), VB64(20), VB64(21), VB64(22),
+        VB64(23), VB64(24), VB64(25), /* '[' .. '^' = 0 */ 0, 0, 0, 0, /* '_' = 63 */ VB64(63) ) },
+
+      .table_96_111_val = {
+        ARRAY_TO_LXV_ORDER(/* '`' = 0 */ 0, /* 'a' .. 'o' = 26 .. 40 */ VB64(26), VB64(27), VB64(28), VB64(29), VB64(30), VB64(31),
+        VB64(32), VB64(33), VB64(34), VB64(35), VB64(36), VB64(37), VB64(38), VB64(39), VB64(40) ) },
+
+      .table_112_127_val = {
+        ARRAY_TO_LXV_ORDER(/* 'p' .. 'z' = 41 .. 51 */ VB64(41), VB64(42), VB64(43), VB64(44), VB64(45), VB64(46), VB64(47), VB64(48),
+        VB64(49), VB64(50), VB64(51), /* '{' .. DEL = 0 */ 0, 0, 0, 0, 0 ) },
+
+      .pack_lshift_val = {
+        ARRAY_TO_LXV_ORDER(
+        0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2 ) },
+
+      .pack_rshift_val = {
+        ARRAY_TO_LXV_ORDER(
+        0, 2, 4, 0, 0, 2, 4, 0, 0, 2, 4, 0, 0, 2, 4, 0 ) },
+
+      // The first 4 index values are "don't care" because
+      // we only use the first 12 bytes of the vector,
+      // which are decoded from 16 bytes of Base64 characters.
+      .pack_permute_val = {
+        ARRAY_TO_LXV_ORDER(
+         0, 0, 0, 0,
+         0,  1,  2,
+         4,  5,  6,
+         8,  9, 10,
+        12, 13, 14 ) }
+    };
+
+    const unsigned block_size = 16;  // number of bytes to process in each pass through the loop
+    const unsigned block_size_shift = 4;
 
     // According to the ELF V2 ABI, registers r3-r12 are volatile and available for use without save/restore
     Register s      = R3_ARG1; // source starting address of Base64 characters
@@ -3757,6 +3791,7 @@ class StubGenerator: public StubCodeGenerator {
     Register d      = R6_ARG4; // destination address
     Register dp     = R7_ARG5; // destination offset
     Register isURL  = R8_ARG6; // boolean, if non-zero indicates use of RFC 4648 base64url encoding
+    Register isMIME = R9_ARG7; // boolean, if non-zero indicates use of RFC 2045 MIME encoding - not used
 
     // Local variables
     Register const_ptr     = R9;  // used for loading constants
@@ -3774,8 +3809,6 @@ class StubGenerator: public StubCodeGenerator {
     VectorRegister  vec_special_case_char   = VR3;
     VectorRegister  pack_rshift             = VR4;
     VectorRegister  pack_lshift             = VR5;
-    // P10+
-    VectorRegister  vec_0x3fs               = VR4; // safe to reuse pack_rshift's register
 
     // VSR Constants
     VectorSRegister offsetLUT               = VSR0;
@@ -3785,26 +3818,40 @@ class StubGenerator: public StubCodeGenerator {
     VectorSRegister vec_special_case_offset = VSR4;
     VectorSRegister pack_permute            = VSR5;
 
-    // Variables for lookup
-    // VR
+    // P10 (or later) VSR lookup constants
+    VectorSRegister table_32_47             = VSR0;
+    VectorSRegister table_48_63             = VSR1;
+    VectorSRegister table_64_79             = VSR2;
+    VectorSRegister table_80_95             = VSR3;
+    VectorSRegister table_96_111            = VSR4;
+    VectorSRegister table_112_127           = VSR6;
+
+    // Data read in and later converted
     VectorRegister  input                   = VR6;
+    // Variable for testing Base64 validity
+    VectorRegister  non_match               = VR10;
+
+    // P9 VR Variables for lookup
     VectorRegister  higher_nibble           = VR7;
     VectorRegister  eq_special_case_char    = VR8;
     VectorRegister  offsets                 = VR9;
-    VectorRegister  non_match               = VR10;
 
-    // VSR
+    // P9 VSR lookup variables
     VectorSRegister bit                     = VSR6;
     VectorSRegister lower_nibble            = VSR7;
     VectorSRegister M                       = VSR8;
 
+    // P10 (or later) VSR lookup variables
+    VectorSRegister  xlate_a                = VSR7;
+    VectorSRegister  xlate_b                = VSR8;
+
     // Variables for pack
     // VR
     VectorRegister  l                       = VR7;  // reuse higher_nibble's register
     VectorRegister  r                       = VR8;  // reuse eq_special_case_char's register
-    VectorRegister  gathered                = VR9;  // reuse offsets's register
+    VectorRegister  gathered                = VR10; // reuse non_match's register
 
-    Label not_URL, calculate_size, unrolled_loop_start, unrolled_loop_exit, return_zero;
+    Label not_URL, calculate_size, loop_start, loop_exit, return_zero;
 
     // The upper 32 bits of the non-pointer parameter registers are not
     // guaranteed to be zero, so mask off those upper bits.
@@ -3823,7 +3870,7 @@ class StubGenerator: public StubCodeGenerator {
     __ sub(sl, sl, sp);
     __ subi(sl, sl, 12);
 
-    // Load CTR with the number of passes through the unrolled loop
+    // Load CTR with the number of passes through the loop
     // = sl >> block_size_shift.  After the shift, if sl <= 0, there's too
     // little data to be processed by this intrinsic.
     __ srawi_(sl, sl, block_size_shift);
@@ -3835,26 +3882,33 @@ class StubGenerator: public StubCodeGenerator {
     __ clrldi(dp, dp, 32);
 
     // Load constant vec registers that need to be loaded from memory
-    __ load_const_optimized(const_ptr, (address)&bitposLUT_val, tmp_reg);
-    __ lxv(bitposLUT, 0, const_ptr);
-    if (PowerArchitecturePPC64 >= 10) {
-        __ load_const_optimized(const_ptr, (address)&p10_pack_permute_val, tmp_reg);
-    } else {
-        __ load_const_optimized(const_ptr, (address)&pack_rshift_val, tmp_reg);
-        __ lxv(pack_rshift->to_vsr(), 0, const_ptr);
-        __ load_const_optimized(const_ptr, (address)&pack_lshift_val, tmp_reg);
-        __ lxv(pack_lshift->to_vsr(), 0, const_ptr);
-        __ load_const_optimized(const_ptr, (address)&pack_permute_val, tmp_reg);
-    }
-    __ lxv(pack_permute, 0, const_ptr);
+    __ load_const_optimized(const_ptr, (address)&const_block, tmp_reg);
+    __ lxv(bitposLUT, BLK_OFFSETOF(bitposLUT_val), const_ptr);
+    __ lxv(pack_rshift->to_vsr(), BLK_OFFSETOF(pack_rshift_val), const_ptr);
+    __ lxv(pack_lshift->to_vsr(), BLK_OFFSETOF(pack_lshift_val), const_ptr);
+    __ lxv(pack_permute, BLK_OFFSETOF(pack_permute_val), const_ptr);
 
     // Splat the constants that can use xxspltib
     __ xxspltib(vec_0s->to_vsr(), 0);
-    __ xxspltib(vec_4s->to_vsr(), 4);
     __ xxspltib(vec_8s->to_vsr(), 8);
-    __ xxspltib(vec_0xfs, 0xf);
     if (PowerArchitecturePPC64 >= 10) {
-        __ xxspltib(vec_0x3fs->to_vsr(), 0x3f);
+      // Using VALID_B64 for the offsets effectively strips the upper bit
+      // of each byte that was selected from the table.  Setting the upper
+      // bit gives us a way to distinguish between the 6-bit value of 0
+      // from an error code of 0, which will happen if the character is
+      // outside the range of the lookup, or is an illegal Base64
+      // character, such as %.
+      __ xxspltib(offsets->to_vsr(), VALID_B64);
+
+      __ lxv(table_48_63, BLK_OFFSETOF(table_48_63_val), const_ptr);
+      __ lxv(table_64_79, BLK_OFFSETOF(table_64_79_val), const_ptr);
+      __ lxv(table_80_95, BLK_OFFSETOF(table_80_95_val), const_ptr);
+      __ lxv(table_96_111, BLK_OFFSETOF(table_96_111_val), const_ptr);
+      __ lxv(table_112_127, BLK_OFFSETOF(table_112_127_val), const_ptr);
+    } else {
+      __ xxspltib(vec_4s->to_vsr(), 4);
+      __ xxspltib(vec_0xfs, 0xf);
+      __ lxv(bitposLUT, BLK_OFFSETOF(bitposLUT_val), const_ptr);
     }
 
     // The rest of the constants use different values depending on the
@@ -3863,22 +3917,28 @@ class StubGenerator: public StubCodeGenerator {
     __ beq(CCR0, not_URL);
 
     // isURL != 0 (true)
-    __ load_const_optimized(const_ptr, (address)&offsetLUT_URL_val, tmp_reg);
-    __ lxv(offsetLUT, 0, const_ptr);
-    __ load_const_optimized(const_ptr, (address)&maskLUT_URL_val, tmp_reg);
-    __ lxv(maskLUT, 0, const_ptr);
-    __ xxspltib(vec_special_case_char->to_vsr(), '_');
-    __ xxspltib(vec_special_case_offset, (unsigned char)US);
+    if (PowerArchitecturePPC64 >= 10) {
+      __ lxv(table_32_47, BLK_OFFSETOF(table_32_47_URL_val), const_ptr);
+      __ lxv(table_80_95, BLK_OFFSETOF(table_80_95_URL_val), const_ptr);
+    } else {
+      __ lxv(offsetLUT, BLK_OFFSETOF(offsetLUT_URL_val), const_ptr);
+      __ lxv(maskLUT, BLK_OFFSETOF(maskLUT_URL_val), const_ptr);
+      __ xxspltib(vec_special_case_char->to_vsr(), '_');
+      __ xxspltib(vec_special_case_offset, (unsigned char)US);
+    }
     __ b(calculate_size);
 
     // isURL = 0 (false)
     __ bind(not_URL);
-    __ load_const_optimized(const_ptr, (address)&offsetLUT_val, tmp_reg);
-    __ lxv(offsetLUT, 0, const_ptr);
-    __ load_const_optimized(const_ptr, (address)&maskLUT_val, tmp_reg);
-    __ lxv(maskLUT, 0, const_ptr);
-    __ xxspltib(vec_special_case_char->to_vsr(), '/');
-    __ xxspltib(vec_special_case_offset, (unsigned char)SLS);
+    if (PowerArchitecturePPC64 >= 10) {
+      __ lxv(table_32_47, BLK_OFFSETOF(table_32_47_val), const_ptr);
+      __ lxv(table_80_95, BLK_OFFSETOF(table_80_95_val), const_ptr);
+    } else {
+      __ lxv(offsetLUT, BLK_OFFSETOF(offsetLUT_val), const_ptr);
+      __ lxv(maskLUT, BLK_OFFSETOF(maskLUT_val), const_ptr);
+      __ xxspltib(vec_special_case_char->to_vsr(), '/');
+      __ xxspltib(vec_special_case_offset, (unsigned char)SLS);
+    }
 
     __ bind(calculate_size);
 
@@ -3889,177 +3949,156 @@ class StubGenerator: public StubCodeGenerator {
     __ add(in, s, sp);
 
     __ align(32);
-    __ bind(unrolled_loop_start);
-    for (unsigned unroll_cnt=0; unroll_cnt < loop_unrolls; unroll_cnt++) {
-        // We can use a static displacement in the load since it's always a
-        // multiple of 16, which is a requirement of lxv/stxv.  This saves
-        // an addi instruction.
-        __ lxv(input->to_vsr(), unroll_cnt * 16, in);
-        //
-        // Lookup
-        //
-        // Isolate the upper 4 bits of each character by shifting it right 4 bits
-        __ vsrb(higher_nibble, input, vec_4s);
-        // Isolate the lower 4 bits by masking
-        __ xxland(lower_nibble, input->to_vsr(), vec_0xfs);
-
-        // Get the offset (the value to subtract from the byte) by using
-        // a lookup table indexed by the upper 4 bits of the character
-        __ xxperm(offsets->to_vsr(), offsetLUT, higher_nibble->to_vsr());
-
-        // Find out which elements are the special case character (isURL ? '/' : '-')
-        __ vcmpequb(eq_special_case_char, input, vec_special_case_char);
-
-        // For each character in the input which is a special case
-        // character, replace its offset with one that is special for that
-        // character.
-        __ xxsel(offsets->to_vsr(), offsets->to_vsr(), vec_special_case_offset, eq_special_case_char->to_vsr());
-
-        // Use the lower_nibble to select a mask "M" from the lookup table.
-        __ xxperm(M, maskLUT, lower_nibble);
-
-        // "bit" is used to isolate which of the bits in M is relevant.
-        __ xxperm(bit, bitposLUT, higher_nibble->to_vsr());
-
-        // Each element of non_match correspond to one each of the 16 input
-        // characters.  Those elements that become 0x00 after the xxland
-        // instuction are invalid Base64 characters.
-        __ xxland(non_match->to_vsr(), M, bit);
-
-        // Compare each element to zero
-        //
-        // vmcmpequb_ sets the EQ bit of CCR6 if no elements compare equal.
-        // Any element comparing equal to zero means there is an error in
-        // that element.  Note that the comparison result register
-        // non_match is not referenced again.  Only CCR6-EQ matters.
-        __ vcmpequb_(non_match, non_match, vec_0s);
-        __ bne_predict_not_taken(CCR6, unrolled_loop_exit);
-
-        // The Base64 characters had no errors, so add the offsets
-        __ vaddubm(input, input, offsets);
-
-        // Pack
-        //
-        // In the tables below, b0, b1, .. b15 are the bytes of decoded
-        // binary data, the first line of each of the cells (except for
-        // the constants) uses the bit-field nomenclature from the
-        // above-linked paper, whereas the second line is more specific
-        // about which exact bits are present, and is constructed using the
-        // Power ISA 3.x document style, where:
-        //
-        // * The specifier after the colon depicts which bits are there.
-        // * The bit numbering is big endian style (bit 0 is the most
-        //   significant).
-        // * || is a concatenate operator.
-        // * Strings of 0's are a field of zeros with the shown length, and
-        //   likewise for strings of 1's.
-
-        if (PowerArchitecturePPC64 >= 10) {
-            // Note that only e8..e15 are shown here because the extract bit
-            // pattern is the same in e0..e7.
-            //
-            // +===============+=============+======================+======================+=============+=============+======================+======================+=============+
-            // |    Vector     |     e8      |          e9          |         e10          |     e11     |     e12     |         e13          |         e14          |     e15     |
-            // |    Element    |             |                      |                      |             |             |                      |                      |             |
-            // +===============+=============+======================+======================+=============+=============+======================+======================+=============+
-            // | after vaddudb |  00hhhhhh   |       00gggggg       |       00ffffff       |  00eeeeee   |  00dddddd   |       00cccccc       |       00bbbbbb       |  00aaaaaa   |
-            // |               | 00||b5:2..7 | 00||b4:4..7||b5:0..1 | 00||b3:6..7||b4:0..3 | 00||b3:0..5 | 00||b2:2..7 | 00||b1:4..7||b2:0..1 | 00||b0:6..7||b1:0..3 | 00||b0:0..5 |
-            // +---------------+-------------+----------------------+----------------------+-------------+-------------+----------------------+----------------------+-------------+
-            // |  after xxbrd  |  00aaaaaa   |       00bbbbbb       |       00cccccc       |  00dddddd   |  00eeeeee   |       00ffffff       |       00gggggg       |  00hhhhhh   |
-            // |               | 00||b0:0..5 | 00||b0:6..7||b1:0..3 | 00||b1:4..7||b2:0..1 | 00||b2:2..7 | 00||b3:0..5 | 00||b3:6..7||b4:0..3 | 00||b4:4..7||b5:0..1 | 00||b5:2..7 |
-            // +---------------+-------------+----------------------+----------------------+-------------+-------------+----------------------+----------------------+-------------+
-            // |   vec_0x3fs   |  00111111   |       00111111       |       00111111       |  00111111   |  00111111   |       00111111       |       00111111       |  00111111   |
-            // +---------------+-------------+----------------------+----------------------+-------------+-------------+----------------------+----------------------+-------------+
-            // | after vpextd  |  00000000   |       00000000       |       aaaaaabb       |  bbbbcccc   |  ccdddddd   |       eeeeeeff       |       ffffgggg       |  gghhhhhh   |
-            // |               |  00000000   |       00000000       |       b0:0..7        |   b1:0..7   |   b2:0..7   |       b3:0..7        |       b4:0..7        |   b5:0..7   |
-            // +===============+=============+======================+======================+=============+=============+======================+======================+=============+
-
-            __ xxbrd(input->to_vsr(), input->to_vsr());
-            __ vpextd(gathered, input, vec_0x3fs);
-
-            // Final rearrangement of bytes into their correct positions.
-            // +==================+====+====+====+====+=====+=====+=====+=====+====+====+=====+=====+=====+=====+=====+=====+
-            // |      Vector      | e0 | e1 | e2 | e3 | e4  | e5  | e6  | e7  | e8 | e9 | e10 | e11 | e12 | e13 | e14 | e15 |
-            // |     Elements     |    |    |    |    |     |     |     |     |    |    |     |     |     |     |     |     |
-            // +==================+====+====+====+====+=====+=====+=====+=====+====+====+=====+=====+=====+=====+=====+=====+
-            // |   after vpextd   | 0  | 0  | b6 | b7 | b8  | b9  | b10 | b11 | 0  | 0  | b0  | b1  | b2  | b3  | b4  | b5  |
-            // +------------------+----+----+----+----+-----+-----+-----+-----+----+----+-----+-----+-----+-----+-----+-----+
-            // | p10_pack_permute | 0  | 0  | 0  | 0  |  7  |  6  |  5  |  4  | 3  | 2  | 15  | 14  | 13  | 12  | 11  | 10  |
-            // +------------------+----+----+----+----+-----+-----+-----+-----+----+----+-----+-----+-----+-----+-----+-----+
-            // |   after xxperm   | 0  | 0  | 0  | 0  | b11 | b10 | b9  | b8  | b7 | b6 | b5  | b4  | b3  | b2  | b1  | b0  |
-            // +==================+====+====+====+====+=====+=====+=====+=====+====+====+=====+=====+=====+=====+=====+=====+
-
-        } else {
-            // Note that only e12..e15 are shown here because the shifting
-            // and OR'ing pattern replicates for e8..e11, e4..7, and
-            // e0..e3.
-            //
-            // +======================+=================+======================+======================+=============+
-            // |        Vector        |       e12       |         e13          |         e14          |     e15     |
-            // |       Element        |                 |                      |                      |             |
-            // +======================+=================+======================+======================+=============+
-            // |    after vaddubm     |    00dddddd     |       00cccccc       |       00bbbbbb       |  00aaaaaa   |
-            // |                      |   00||b2:2..7   | 00||b1:4..7||b2:0..1 | 00||b0:6..7||b1:0..3 | 00||b0:0..5 |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // |     pack_lshift      |                 |         << 6         |         << 4         |    << 2     |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // |     l after vslb     |    00dddddd     |       cc000000       |       bbbb0000       |  aaaaaa00   |
-            // |                      |   00||b2:2..7   |   b2:0..1||000000    |    b1:0..3||0000     | b0:0..5||00 |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // |     l after vslo     |    cc000000     |       bbbb0000       |       aaaaaa00       |  00000000   |
-            // |                      | b2:0..1||000000 |    b1:0..3||0000     |     b0:0..5||00      |  00000000   |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // |     pack_rshift      |                 |         >> 2         |         >> 4         |             |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // |     r after vsrb     |    00dddddd     |       0000cccc       |       000000bb       |  00aaaaaa   |
-            // |                      |   00||b2:2..7   |    0000||b1:4..7     |   000000||b0:6..7    | 00||b0:0..5 |
-            // +----------------------+-----------------+----------------------+----------------------+-------------+
-            // | gathered after xxlor |    ccdddddd     |       bbbbcccc       |       aaaaaabb       |  00aaaaaa   |
-            // |                      |     b2:0..7     |       b1:0..7        |       b0:0..7        | 00||b0:0..5 |
-            // +======================+=================+======================+======================+=============+
-            //
-            // Note: there is a typo in the above-linked paper that shows the result of the gathering process is:
-            // [ddddddcc|bbbbcccc|aaaaaabb]
-            // but should be:
-            // [ccdddddd|bbbbcccc|aaaaaabb]
-            //
-            __ vslb(l, input, pack_lshift);
-            // vslo of vec_8s shifts the vector by one octet toward lower
-            // element numbers, discarding element 0.  This means it actually
-            // shifts to the right (not left) according to the order of the
-            // table above.
-            __ vslo(l, l, vec_8s);
-            __ vsrb(r, input, pack_rshift);
-            __ xxlor(gathered->to_vsr(), l->to_vsr(), r->to_vsr());
-
-            // Final rearrangement of bytes into their correct positions.
-            // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
-            // |    Vector    |  e0  |  e1  |  e2  |  e3  | e4  | e5  | e6 | e7 | e8 | e9 | e10 | e11 | e12 | e13 | e14 | e15 |
-            // |   Elements   |      |      |      |      |     |     |    |    |    |    |     |     |     |     |     |     |
-            // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
-            // | after xxlor  | b11  | b10  |  b9  |  xx  | b8  | b7  | b6 | xx | b5 | b4 | b3  | xx  | b2  | b1  | b0  | xx  |
-            // +--------------+------+------+------+------+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+
-            // | pack_permute |  0   |  0   |  0   |  0   |  0  |  1  | 2  | 4  | 5  | 6  |  8  |  9  | 10  | 12  | 13  | 14  |
-            // +--------------+------+------+------+------+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+
-            // | after xxperm | b11* | b11* | b11* | b11* | b11 | b10 | b9 | b8 | b7 | b6 | b5  | b4  | b3  | b2  | b1  | b0  |
-            // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
-            // xx bytes are not used to form the final data
-            // b0..b15 are the decoded and reassembled 8-bit bytes of data
-            // b11 with asterisk is a "don't care", because these bytes will be
-            // overwritten on the next iteration.
-        }
-        __ xxperm(gathered->to_vsr(), gathered->to_vsr(), pack_permute);
+    __ bind(loop_start);
+    __ lxv(input->to_vsr(), 0, in); // offset=0
 
-        // We cannot use a static displacement on the store, since it's a
-        // multiple of 12, not 16.  Note that this stxv instruction actually
-        // writes 16 bytes, even though only the first 12 are valid data.
-        __ stxv(gathered->to_vsr(), 0, out);
-        __ addi(out, out, 12);
+    //
+    // Lookup
+    //
+    if (PowerArchitecturePPC64 >= 10) {
+      // Use xxpermx to do a lookup of each Base64 character in the
+      // input vector and translate it to a 6-bit value + 0x80.
+      // Characters which are not valid Base64 characters will result
+      // in a zero in the corresponding byte.
+      //
+      // Note that due to align(32) call above, the xxpermx instructions do
+      // not require align_prefix() calls, since the final xxpermx
+      // prefix+opcode is at byte 24.
+      __ xxpermx(xlate_a, table_32_47, table_48_63, input->to_vsr(), 1);    // offset=4
+      __ xxpermx(xlate_b, table_64_79, table_80_95, input->to_vsr(), 2);    // offset=12
+      __ xxlor(xlate_b, xlate_a, xlate_b);                                  // offset=20
+      __ xxpermx(xlate_a, table_96_111, table_112_127, input->to_vsr(), 3); // offset=24
+      __ xxlor(input->to_vsr(), xlate_a, xlate_b);
+      // Check for non-Base64 characters by comparing each byte to zero.
+      __ vcmpequb_(non_match, input, vec_0s);
+    } else {
+      // Isolate the upper 4 bits of each character by shifting it right 4 bits
+      __ vsrb(higher_nibble, input, vec_4s);
+      // Isolate the lower 4 bits by masking
+      __ xxland(lower_nibble, input->to_vsr(), vec_0xfs);
+
+      // Get the offset (the value to subtract from the byte) by using
+      // a lookup table indexed by the upper 4 bits of the character
+      __ xxperm(offsets->to_vsr(), offsetLUT, higher_nibble->to_vsr());
+
+      // Find out which elements are the special case character (isURL ? '/' : '-')
+      __ vcmpequb(eq_special_case_char, input, vec_special_case_char);
+
+      // For each character in the input which is a special case
+      // character, replace its offset with one that is special for that
+      // character.
+      __ xxsel(offsets->to_vsr(), offsets->to_vsr(), vec_special_case_offset, eq_special_case_char->to_vsr());
+
+      // Use the lower_nibble to select a mask "M" from the lookup table.
+      __ xxperm(M, maskLUT, lower_nibble);
+
+      // "bit" is used to isolate which of the bits in M is relevant.
+      __ xxperm(bit, bitposLUT, higher_nibble->to_vsr());
+
+      // Each element of non_match correspond to one each of the 16 input
+      // characters.  Those elements that become 0x00 after the xxland
+      // instuction are invalid Base64 characters.
+      __ xxland(non_match->to_vsr(), M, bit);
+
+      // Compare each element to zero
+      //
+      __ vcmpequb_(non_match, non_match, vec_0s);
     }
-    __ addi(in, in, 16 * loop_unrolls);
-    __ bdnz(unrolled_loop_start);
+    // vmcmpequb_ sets the EQ bit of CCR6 if no elements compare equal.
+    // Any element comparing equal to zero means there is an error in
+    // that element.  Note that the comparison result register
+    // non_match is not referenced again.  Only CCR6-EQ matters.
+    __ bne_predict_not_taken(CCR6, loop_exit);
+
+    // The Base64 characters had no errors, so add the offsets, which in
+    // the case of Power10 is a constant vector of all 0x80's (see earlier
+    // comment where the offsets register is loaded).
+    __ vaddubm(input, input, offsets);
+
+    // Pack
+    //
+    // In the tables below, b0, b1, .. b15 are the bytes of decoded
+    // binary data, the first line of each of the cells (except for
+    // the constants) uses the bit-field nomenclature from the
+    // above-linked paper, whereas the second line is more specific
+    // about which exact bits are present, and is constructed using the
+    // Power ISA 3.x document style, where:
+    //
+    // * The specifier after the colon depicts which bits are there.
+    // * The bit numbering is big endian style (bit 0 is the most
+    //   significant).
+    // * || is a concatenate operator.
+    // * Strings of 0's are a field of zeros with the shown length, and
+    //   likewise for strings of 1's.
+
+    // Note that only e12..e15 are shown here because the shifting
+    // and OR'ing pattern replicates for e8..e11, e4..7, and
+    // e0..e3.
+    //
+    // +======================+=================+======================+======================+=============+
+    // |        Vector        |       e12       |         e13          |         e14          |     e15     |
+    // |       Element        |                 |                      |                      |             |
+    // +======================+=================+======================+======================+=============+
+    // |    after vaddubm     |    00dddddd     |       00cccccc       |       00bbbbbb       |  00aaaaaa   |
+    // |                      |   00||b2:2..7   | 00||b1:4..7||b2:0..1 | 00||b0:6..7||b1:0..3 | 00||b0:0..5 |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // |     pack_lshift      |                 |         << 6         |         << 4         |    << 2     |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // |     l after vslb     |    00dddddd     |       cc000000       |       bbbb0000       |  aaaaaa00   |
+    // |                      |   00||b2:2..7   |   b2:0..1||000000    |    b1:0..3||0000     | b0:0..5||00 |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // |     l after vslo     |    cc000000     |       bbbb0000       |       aaaaaa00       |  00000000   |
+    // |                      | b2:0..1||000000 |    b1:0..3||0000     |     b0:0..5||00      |  00000000   |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // |     pack_rshift      |                 |         >> 2         |         >> 4         |             |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // |     r after vsrb     |    00dddddd     |       0000cccc       |       000000bb       |  00aaaaaa   |
+    // |                      |   00||b2:2..7   |    0000||b1:4..7     |   000000||b0:6..7    | 00||b0:0..5 |
+    // +----------------------+-----------------+----------------------+----------------------+-------------+
+    // | gathered after xxlor |    ccdddddd     |       bbbbcccc       |       aaaaaabb       |  00aaaaaa   |
+    // |                      |     b2:0..7     |       b1:0..7        |       b0:0..7        | 00||b0:0..5 |
+    // +======================+=================+======================+======================+=============+
+    //
+    // Note: there is a typo in the above-linked paper that shows the result of the gathering process is:
+    // [ddddddcc|bbbbcccc|aaaaaabb]
+    // but should be:
+    // [ccdddddd|bbbbcccc|aaaaaabb]
+    //
+    __ vslb(l, input, pack_lshift);
+    // vslo of vec_8s shifts the vector by one octet toward lower
+    // element numbers, discarding element 0.  This means it actually
+    // shifts to the right (not left) according to the order of the
+    // table above.
+    __ vslo(l, l, vec_8s);
+    __ vsrb(r, input, pack_rshift);
+    __ xxlor(gathered->to_vsr(), l->to_vsr(), r->to_vsr());
+
+    // Final rearrangement of bytes into their correct positions.
+    // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
+    // |    Vector    |  e0  |  e1  |  e2  |  e3  | e4  | e5  | e6 | e7 | e8 | e9 | e10 | e11 | e12 | e13 | e14 | e15 |
+    // |   Elements   |      |      |      |      |     |     |    |    |    |    |     |     |     |     |     |     |
+    // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
+    // | after xxlor  | b11  | b10  |  b9  |  xx  | b8  | b7  | b6 | xx | b5 | b4 | b3  | xx  | b2  | b1  | b0  | xx  |
+    // +--------------+------+------+------+------+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+
+    // | pack_permute |  0   |  0   |  0   |  0   |  0  |  1  | 2  | 4  | 5  | 6  |  8  |  9  | 10  | 12  | 13  | 14  |
+    // +--------------+------+------+------+------+-----+-----+----+----+----+----+-----+-----+-----+-----+-----+-----+
+    // | after xxperm | b11* | b11* | b11* | b11* | b11 | b10 | b9 | b8 | b7 | b6 | b5  | b4  | b3  | b2  | b1  | b0  |
+    // +==============+======+======+======+======+=====+=====+====+====+====+====+=====+=====+=====+=====+=====+=====+
+    // xx bytes are not used to form the final data
+    // b0..b15 are the decoded and reassembled 8-bit bytes of data
+    // b11 with asterisk is a "don't care", because these bytes will be
+    // overwritten on the next iteration.
+    __ xxperm(gathered->to_vsr(), gathered->to_vsr(), pack_permute);
+
+    // We cannot use a static displacement on the store, since it's a
+    // multiple of 12, not 16.  Note that this stxv instruction actually
+    // writes 16 bytes, even though only the first 12 are valid data.
+    __ stxv(gathered->to_vsr(), 0, out);
+    __ addi(out, out, 12);
+    __ addi(in, in, 16);
+    __ bdnz(loop_start);
 
-    __ bind(unrolled_loop_exit);
+    __ bind(loop_exit);
 
     // Return the number of out bytes produced, which is (out - (d + dp)) == out - d - dp;
     __ sub(R3_RET, out, d);
@@ -4187,10 +4226,12 @@ class StubGenerator: public StubCodeGenerator {
 // at each location, all values in expanded are compared to 31.  Using
 // vsel, values higher than 31 use the results from the upper 32 bytes of
 // the lookup operation, while values less than or equal to 31 use the
-// lower 32 bytes of the lookup operation.  Power10 and beyond can save the
-// compare instruction, because the comparison is done within xxpermx
-// itself. TODO: use xxpermx,xxpermx,vor on P10 when instruction prefixes are
-// available in assembler_ppc.*
+// lower 32 bytes of the lookup operation.
+//
+// Note: it's tempting to use a xxpermx,xxpermx,vor sequence here on
+// Power10 (or later), but experiments doing so on Power10 yielded a slight
+// performance drop, perhaps due to the need for xxpermx instruction
+// prefixes.
 
 #define ENCODE_CORE                                                        \
     __ xxperm(input->to_vsr(), input->to_vsr(), expand_permute);           \
@@ -4282,7 +4323,6 @@ class StubGenerator: public StubCodeGenerator {
         ARRAY_TO_LXV_ORDER(
         'w','x','y','z','0','1','2','3','4','5','6','7','8','9','-','_' ) }
     };
-    #define BLK_OFFSETOF(x) (offsetof(constant_block, x))
 
     // Number of bytes to process in each pass through the main loop.
     // 12 of the 16 bytes from each lxv are encoded to 16 Base64 bytes.
@@ -4305,7 +4345,7 @@ class StubGenerator: public StubCodeGenerator {
     Register block_modulo   = R12; // == block_size (reuse const_ptr)
     Register remaining      = R12; // bytes remaining to process after the blocks are completed (reuse block_modulo's reg)
     Register in             = R4;  // current input (source) pointer (reuse sp's register)
-    Register num_blocks     = R11; // number of blocks to be processed by the unrolled loop
+    Register num_blocks     = R11; // number of blocks to be processed by the loop
     Register out            = R8;  // current output (destination) pointer (reuse const_ptr's register)
     Register three          = R9;  // constant divisor (reuse size's register)
     Register bytes_to_write = R10; // number of bytes to write with the stxvl instr (reused blocked_size's register)
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 2210e05a410..124cb66cc5d 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -3793,11 +3793,7 @@ void TemplateTable::_new() {
     // --------------------------------------------------------------------------
     // Init2: Initialize the header: mark, klass
     // Init mark.
-    if (UseBiasedLocking) {
-      __ ld(Rscratch, in_bytes(Klass::prototype_header_offset()), RinstanceKlass);
-    } else {
-      __ load_const_optimized(Rscratch, markWord::prototype().value(), R0);
-    }
+    __ load_const_optimized(Rscratch, markWord::prototype().value(), R0);
     __ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject);
 
     // Init klass.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 43b655cf9d6..0b2aa63bcea 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2020 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -378,11 +378,7 @@ void VM_Version::initialize() {
 
   // Adjust RTM (Restricted Transactional Memory) flags.
   if (UseRTMLocking) {
-    // If CPU or OS do not support TM:
-    // Can't continue because UseRTMLocking affects UseBiasedLocking flag
-    // setting during arguments processing. See use_biased_locking().
-    // VM_Version_init() is executed after UseBiasedLocking is used
-    // in Thread::allocate().
+    // If CPU or OS do not support RTM:
     if (PowerArchitecturePPC64 < 8) {
       vm_exit_during_initialization("RTM instructions are not available on this CPU.");
     }
@@ -399,8 +395,6 @@ void VM_Version::initialize() {
     }
 #else
     // Only C2 does RTM locking optimization.
-    // Can't continue because UseRTMLocking affects UseBiasedLocking flag
-    // setting during arguments processing. See use_biased_locking().
     vm_exit_during_initialization("RTM locking optimization is not supported in this VM");
 #endif
   } else { // !UseRTMLocking
@@ -544,27 +538,6 @@ void VM_Version::print_platform_virtualization_info(outputStream* st) {
 #endif
 }
 
-bool VM_Version::use_biased_locking() {
-#if INCLUDE_RTM_OPT
-  // RTM locking is most useful when there is high lock contention and
-  // low data contention. With high lock contention the lock is usually
-  // inflated and biased locking is not suitable for that case.
-  // RTM locking code requires that biased locking is off.
-  // Note: we can't switch off UseBiasedLocking in get_processor_features()
-  // because it is used by Thread::allocate() which is called before
-  // VM_Version::initialize().
-  if (UseRTMLocking && UseBiasedLocking) {
-    if (FLAG_IS_DEFAULT(UseBiasedLocking)) {
-      FLAG_SET_DEFAULT(UseBiasedLocking, false);
-    } else {
-      warning("Biased locking is not supported with RTM locking; ignoring UseBiasedLocking flag." );
-      UseBiasedLocking = false;
-    }
-  }
-#endif
-  return UseBiasedLocking;
-}
-
 void VM_Version::print_features() {
   tty->print_cr("Version: %s L1_data_cache_line_size=%d", features_string(), L1_data_cache_line_size());
 
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index 439be6cc8d2..89352192615 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -93,9 +93,6 @@ class VM_Version: public Abstract_VM_Version {
   // Override Abstract_VM_Version implementation
   static void print_platform_virtualization_info(outputStream*);
 
-  // Override Abstract_VM_Version implementation
-  static bool use_biased_locking();
-
   // PPC64 supports fast class initialization checks for static methods.
   static bool supports_fast_class_init_checks() { return true; }
   constexpr static bool supports_stack_watermark_barrier() { return true; }
diff --git a/src/hotspot/cpu/s390/assembler_s390.cpp b/src/hotspot/cpu/s390/assembler_s390.cpp
index d587c9d659b..4defc959618 100644
--- a/src/hotspot/cpu/s390/assembler_s390.cpp
+++ b/src/hotspot/cpu/s390/assembler_s390.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -31,7 +31,6 @@
 #include "gc/shared/cardTableBarrierSet.hpp"
 #include "memory/resourceArea.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
diff --git a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
index 329c163f313..ccc2364007b 100644
--- a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
@@ -454,8 +454,10 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->verify_oop_map(info());
 
 #ifndef PRODUCT
-  __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_slowcase_cnt);
-  __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
+  if (PrintC1Statistics) {
+    __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_slowcase_cnt);
+    __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
+  }
 #endif
 
   __ branch_optimized(Assembler::bcondAlways, _continuation);
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 37eba3b6b6f..a19d5da501b 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -879,7 +879,7 @@ Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 }
 
 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code,
-                            CodeEmitInfo* info, bool wide, bool unaligned) {
+                            CodeEmitInfo* info, bool wide) {
 
   assert(type != T_METADATA, "load of metadata ptr not supported");
   LIR_Address* addr = src_opr->as_address_ptr();
@@ -1079,7 +1079,7 @@ void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
 
 void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
-                            bool wide, bool unaligned) {
+                            bool wide) {
   assert(type != T_METADATA, "store of metadata ptr not supported");
   LIR_Address* addr = dest_opr->as_address_ptr();
 
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 89cea9598a4..aaad1575a82 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -33,7 +33,6 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
@@ -97,10 +96,6 @@ void C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hd
     z_btrue(slow_case);
   }
 
-  if (UseBiasedLocking) {
-    biased_locking_enter(obj, hdr, Z_R1_scratch, Z_R0_scratch, done, &slow_case);
-  }
-
   // and mark it as unlocked.
   z_oill(hdr, markWord::unlocked_value);
   // Save unlocked object header into the displaced header location on the stack.
@@ -110,13 +105,6 @@ void C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hd
   // object header instead.
   z_csg(hdr, disp_hdr, hdr_offset, obj);
   // If the object header was the same, we're done.
-  if (PrintBiasedLockingStatistics) {
-    Unimplemented();
-#if 0
-    cond_inc32(Assembler::equal,
-               ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
-#endif
-  }
   branch_optimized(Assembler::bcondEqual, done);
   // If the object header was not the same, it is now in the hdr register.
   // => Test if it is a stack pointer into the same stack (recursive locking), i.e.:
@@ -150,20 +138,12 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert_different_registers(hdr, obj, disp_hdr);
   NearLabel done;
 
-  if (UseBiasedLocking) {
-    // Load object.
-    z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-    biased_locking_exit(obj, hdr, done);
-  }
-
   // Load displaced header.
   z_ltg(hdr, Address(disp_hdr, (intptr_t)0));
   // If the loaded hdr is NULL we had recursive locking, and we are done.
   z_bre(done);
-  if (!UseBiasedLocking) {
-    // Load object.
-    z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-  }
+  // Load object.
+  z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
   verify_oop(obj, FILE_AND_LINE);
   // Test if object header is pointing to the displaced header, and if so, restore
   // the displaced header in the object. If the object header is not pointing to
@@ -193,13 +173,8 @@ void C1_MacroAssembler::try_allocate(
 
 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register Rzero, Register t1) {
   assert_different_registers(obj, klass, len, t1, Rzero);
-  if (UseBiasedLocking && !len->is_valid()) {
-    assert_different_registers(obj, klass, len, t1);
-    z_lg(t1, Address(klass, Klass::prototype_header_offset()));
-  } else {
-    // This assumes that all prototype bits fit in an int32_t.
-    load_const_optimized(t1, (intx)markWord::prototype().value());
-  }
+  // This assumes that all prototype bits fit in an int32_t.
+  load_const_optimized(t1, (intx)markWord::prototype().value());
   z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
 
   if (len->is_valid()) {
diff --git a/src/hotspot/cpu/s390/c2_globals_s390.hpp b/src/hotspot/cpu/s390/c2_globals_s390.hpp
index e747f6c8c51..0192cb716ba 100644
--- a/src/hotspot/cpu/s390/c2_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c2_globals_s390.hpp
@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         4);
-define_pd_global(intx, FLOATPRESSURE,                15);
 define_pd_global(intx, FreqInlineSize,               175);
 // 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
-define_pd_global(intx, INTPRESSURE,                  10); // Medium size register set, 6 special purpose regs, 3 SOE regs.
 define_pd_global(intx, InteriorEntryAlignment,       2);
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 define_pd_global(intx, RegisterCostAreaRatio,        12000);
diff --git a/src/hotspot/cpu/s390/frame_s390.cpp b/src/hotspot/cpu/s390/frame_s390.cpp
index 2c1155e6de5..2486c6c6360 100644
--- a/src/hotspot/cpu/s390/frame_s390.cpp
+++ b/src/hotspot/cpu/s390/frame_s390.cpp
@@ -208,6 +208,16 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return fr;
 }
 
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
+  ShouldNotCallThis();
+  return nullptr;
+}
+
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_interpreter_frame(RegisterMap *map) const {
   // Pass callers sender_sp as unextended_sp.
   return frame(sender_sp(), sender_pc(), (intptr_t*)(ijava_state()->sender_sp));
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index ba4bfdc9864..7258630bb0b 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -427,7 +427,7 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
 
   __ z_ltgr(Z_R1_scratch, pre_val_reg); // Pass oop in Z_R1_scratch to Runtime1::g1_pre_barrier_slow_id.
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index 98f16b0f004..b4b13a14dd4 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -38,7 +38,6 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -1005,10 +1004,6 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
     z_btrue(slow_case);
   }
 
-  if (UseBiasedLocking) {
-    biased_locking_enter(object, displaced_header, Z_R1, Z_R0, done, &slow_case);
-  }
-
   // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
   z_oill(displaced_header, markWord::unlocked_value);
 
@@ -1116,12 +1111,6 @@ void InterpreterMacroAssembler::unlock_object(Register monitor, Register object)
 
   clear_mem(obj_entry, sizeof(oop));
 
-  if (UseBiasedLocking) {
-    // The object address from the monitor is in object.
-    assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-    biased_locking_exit(object, displaced_header, done);
-  }
-
   // Test first if we are in the fast recursive case.
   MacroAssembler::load_and_test_long(displaced_header,
                                      Address(monitor, BasicObjectLock::lock_offset_in_bytes() +
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index c098cf09604..06567a511ed 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -39,7 +39,6 @@
 #include "oops/klass.inline.hpp"
 #include "prims/methodHandles.hpp"
 #include "registerSaver_s390.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/icache.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
@@ -3128,194 +3127,7 @@ void MacroAssembler::increment_counter_eq(address counter_address, Register tmp1
   bind(l);
 }
 
-// Semantics are dependent on the slow_case label:
-//   If the slow_case label is not NULL, failure to biased-lock the object
-//   transfers control to the location of the slow_case label. If the
-//   object could be biased-locked, control is transferred to the done label.
-//   The condition code is unpredictable.
-//
-//   If the slow_case label is NULL, failure to biased-lock the object results
-//   in a transfer of control to the done label with a condition code of not_equal.
-//   If the biased-lock could be successfully obtained, control is transfered to
-//   the done label with a condition code of equal.
-//   It is mandatory to react on the condition code At the done label.
-//
-void MacroAssembler::biased_locking_enter(Register  obj_reg,
-                                          Register  mark_reg,
-                                          Register  temp_reg,
-                                          Register  temp2_reg,    // May be Z_RO!
-                                          Label    &done,
-                                          Label    *slow_case) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-  assert_different_registers(obj_reg, mark_reg, temp_reg, temp2_reg);
-
-  Label cas_label; // Try, if implemented, CAS locking. Fall thru to slow path otherwise.
-
-  BLOCK_COMMENT("biased_locking_enter {");
-
-  // Biased locking
-  // See whether the lock is currently biased toward our thread and
-  // whether the epoch is still valid.
-  // Note that the runtime guarantees sufficient alignment of JavaThread
-  // pointers to allow age to be placed into low bits.
-  assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits,
-         "biased locking makes assumptions about bit layout");
-  z_lr(temp_reg, mark_reg);
-  z_nilf(temp_reg, markWord::biased_lock_mask_in_place);
-  z_chi(temp_reg, markWord::biased_lock_pattern);
-  z_brne(cas_label);  // Try cas if object is not biased, i.e. cannot be biased locked.
-
-  load_prototype_header(temp_reg, obj_reg);
-  load_const_optimized(temp2_reg, ~((int) markWord::age_mask_in_place));
-
-  z_ogr(temp_reg, Z_thread);
-  z_xgr(temp_reg, mark_reg);
-  z_ngr(temp_reg, temp2_reg);
-  if (PrintBiasedLockingStatistics) {
-    increment_counter_eq((address) BiasedLocking::biased_lock_entry_count_addr(), mark_reg, temp2_reg);
-    // Restore mark_reg.
-    z_lg(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg);
-  }
-  branch_optimized(Assembler::bcondEqual, done);  // Biased lock obtained, return success.
-
-  Label try_revoke_bias;
-  Label try_rebias;
-  Address mark_addr = Address(obj_reg, oopDesc::mark_offset_in_bytes());
-
-  //----------------------------------------------------------------------------
-  // At this point we know that the header has the bias pattern and
-  // that we are not the bias owner in the current epoch. We need to
-  // figure out more details about the state of the header in order to
-  // know what operations can be legally performed on the object's
-  // header.
-
-  // If the low three bits in the xor result aren't clear, that means
-  // the prototype header is no longer biased and we have to revoke
-  // the bias on this object.
-  z_tmll(temp_reg, markWord::biased_lock_mask_in_place);
-  z_brnaz(try_revoke_bias);
-
-  // Biasing is still enabled for this data type. See whether the
-  // epoch of the current bias is still valid, meaning that the epoch
-  // bits of the mark word are equal to the epoch bits of the
-  // prototype header. (Note that the prototype header's epoch bits
-  // only change at a safepoint.) If not, attempt to rebias the object
-  // toward the current thread. Note that we must be absolutely sure
-  // that the current epoch is invalid in order to do this because
-  // otherwise the manipulations it performs on the mark word are
-  // illegal.
-  z_tmll(temp_reg, markWord::epoch_mask_in_place);
-  z_brnaz(try_rebias);
-
-  //----------------------------------------------------------------------------
-  // The epoch of the current bias is still valid but we know nothing
-  // about the owner; it might be set or it might be clear. Try to
-  // acquire the bias of the object using an atomic operation. If this
-  // fails we will go in to the runtime to revoke the object's bias.
-  // Note that we first construct the presumed unbiased header so we
-  // don't accidentally blow away another thread's valid bias.
-  z_nilf(mark_reg, markWord::biased_lock_mask_in_place | markWord::age_mask_in_place |
-         markWord::epoch_mask_in_place);
-  z_lgr(temp_reg, Z_thread);
-  z_llgfr(mark_reg, mark_reg);
-  z_ogr(temp_reg, mark_reg);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  z_csg(mark_reg, temp_reg, 0, obj_reg);
-
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-
-  if (PrintBiasedLockingStatistics) {
-    increment_counter_eq((address) BiasedLocking::anonymously_biased_lock_entry_count_addr(),
-                         temp_reg, temp2_reg);
-  }
-  if (slow_case != NULL) {
-    branch_optimized(Assembler::bcondNotEqual, *slow_case); // Biased lock not obtained, need to go the long way.
-  }
-  branch_optimized(Assembler::bcondAlways, done);           // Biased lock status given in condition code.
-
-  //----------------------------------------------------------------------------
-  bind(try_rebias);
-  // At this point we know the epoch has expired, meaning that the
-  // current "bias owner", if any, is actually invalid. Under these
-  // circumstances _only_, we are allowed to use the current header's
-  // value as the comparison value when doing the cas to acquire the
-  // bias in the current epoch. In other words, we allow transfer of
-  // the bias from one thread to another directly in this situation.
-
-  z_nilf(mark_reg, markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place);
-  load_prototype_header(temp_reg, obj_reg);
-  z_llgfr(mark_reg, mark_reg);
-
-  z_ogr(temp_reg, Z_thread);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  z_csg(mark_reg, temp_reg, 0, obj_reg);
-
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-
-  if (PrintBiasedLockingStatistics) {
-    increment_counter_eq((address) BiasedLocking::rebiased_lock_entry_count_addr(), temp_reg, temp2_reg);
-  }
-  if (slow_case != NULL) {
-    branch_optimized(Assembler::bcondNotEqual, *slow_case);  // Biased lock not obtained, need to go the long way.
-  }
-  z_bru(done);           // Biased lock status given in condition code.
-
-  //----------------------------------------------------------------------------
-  bind(try_revoke_bias);
-  // The prototype mark in the klass doesn't have the bias bit set any
-  // more, indicating that objects of this data type are not supposed
-  // to be biased any more. We are going to try to reset the mark of
-  // this object to the prototype value and fall through to the
-  // CAS-based locking scheme. Note that if our CAS fails, it means
-  // that another thread raced us for the privilege of revoking the
-  // bias of this particular object, so it's okay to continue in the
-  // normal locking code.
-  load_prototype_header(temp_reg, obj_reg);
-
-  assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-  z_csg(mark_reg, temp_reg, 0, obj_reg);
-
-  // Fall through to the normal CAS-based lock, because no matter what
-  // the result of the above CAS, some thread must have succeeded in
-  // removing the bias bit from the object's header.
-  if (PrintBiasedLockingStatistics) {
-    // z_cgr(mark_reg, temp2_reg);
-    increment_counter_eq((address) BiasedLocking::revoked_lock_entry_count_addr(), temp_reg, temp2_reg);
-  }
-
-  bind(cas_label);
-  BLOCK_COMMENT("} biased_locking_enter");
-}
-
-void MacroAssembler::biased_locking_exit(Register mark_addr, Register temp_reg, Label& done) {
-  // Check for biased locking unlock case, which is a no-op
-  // Note: we do not have to check the thread ID for two reasons.
-  // First, the interpreter checks for IllegalMonitorStateException at
-  // a higher level. Second, if the bias was revoked while we held the
-  // lock, the object could not be rebiased toward another thread, so
-  // the bias bit would be clear.
-  BLOCK_COMMENT("biased_locking_exit {");
-
-  z_lg(temp_reg, 0, mark_addr);
-  z_nilf(temp_reg, markWord::biased_lock_mask_in_place);
-
-  z_chi(temp_reg, markWord::biased_lock_pattern);
-  z_bre(done);
-  BLOCK_COMMENT("} biased_locking_exit");
-}
-
-void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2, bool try_bias) {
+void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2) {
   Register displacedHeader = temp1;
   Register currentHeader = temp1;
   Register temp = temp2;
@@ -3334,10 +3146,6 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
     z_brne(done);
   }
 
-  if (try_bias) {
-    biased_locking_enter(oop, displacedHeader, temp, Z_R0, done);
-  }
-
   // Handle existing monitor.
   // The object has an existing monitor iff (mark & monitor_value) != 0.
   guarantee(Immediate::is_uimm16(markWord::monitor_value), "must be half-word");
@@ -3402,7 +3210,7 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
   // _complete_monitor_locking_Java.
 }
 
-void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2, bool try_bias) {
+void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2) {
   Register displacedHeader = temp1;
   Register currentHeader = temp2;
   Register temp = temp1;
@@ -3412,10 +3220,6 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
 
   BLOCK_COMMENT("compiler_fast_unlock_object {");
 
-  if (try_bias) {
-    biased_locking_exit(oop, currentHeader, done);
-  }
-
   // Find the lock address and load the displaced header from the stack.
   // if the displaced header is zero, we have a recursive unlock.
   load_and_test_long(displacedHeader, Address(box, BasicLock::displaced_header_offset_in_bytes()));
@@ -3833,12 +3637,6 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) {
   }
 }
 
-void MacroAssembler::load_prototype_header(Register Rheader, Register Rsrc_oop) {
-  assert_different_registers(Rheader, Rsrc_oop);
-  load_klass(Rheader, Rsrc_oop);
-  z_lg(Rheader, Address(Rheader, Klass::prototype_header_offset()));
-}
-
 void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
   if (UseCompressedClassPointers) {
     assert_different_registers(dst_oop, klass, Z_R0);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index 113a1a3db2a..72cfbe02355 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016, 2019 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -718,26 +718,9 @@ class MacroAssembler: public Assembler {
   // Increment a counter at counter_address when the eq condition code is set.
   // Kills registers tmp1_reg and tmp2_reg and preserves the condition code.
   void increment_counter_eq(address counter_address, Register tmp1_reg, Register tmp2_reg);
-  // Biased locking support
-  // Upon entry,obj_reg must contain the target object, and mark_reg
-  // must contain the target object's header.
-  // Destroys mark_reg if an attempt is made to bias an anonymously
-  // biased lock. In this case a failure will go either to the slow
-  // case or fall through with the notEqual condition code set with
-  // the expectation that the slow case in the runtime will be called.
-  // In the fall-through case where the CAS-based lock is done,
-  // mark_reg is not destroyed.
-  void biased_locking_enter(Register obj_reg, Register mark_reg, Register temp_reg,
-                            Register temp2_reg, Label& done, Label* slow_case = NULL);
-  // Upon entry, the base register of mark_addr must contain the oop.
-  // Destroys temp_reg.
-  // If allow_delay_slot_filling is set to true, the next instruction
-  // emitted after this one will go in an annulled delay slot if the
-  // biased locking exit case failed.
-  void biased_locking_exit(Register mark_addr, Register temp_reg, Label& done);
-
-  void compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2, bool try_bias = UseBiasedLocking);
-  void compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2, bool try_bias = UseBiasedLocking);
+
+  void compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2);
+  void compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2);
 
   void resolve_jobject(Register value, Register tmp1, Register tmp2);
 
@@ -782,7 +765,6 @@ class MacroAssembler: public Assembler {
   void decode_klass_not_null(Register dst);
   void load_klass(Register klass, Address mem);
   void load_klass(Register klass, Register src_oop);
-  void load_prototype_header(Register Rheader, Register Rsrc_oop);
   void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
   void store_klass_gap(Register s, Register dst_oop);
 
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index acd601b4929..cd22b795886 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1554,10 +1554,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 //----------SUPERWORD HELPERS----------------------------------------
 
 // Vector width in bytes.
@@ -1609,7 +1605,7 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd,
   return NULL;
 }
 
-bool Matcher::is_generic_reg2reg_move(MachNode* m) {
+bool Matcher::is_reg2reg_move(MachNode* m) {
   ShouldNotReachHere();  // generic vector operands not supported
   return false;
 }
@@ -1665,6 +1661,17 @@ bool Matcher::is_spillable_arg(int reg) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  // Medium size register set, 6 special purpose regs, 3 SOE regs.
+  return (INTPRESSURE == -1) ? 10 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }
@@ -9809,8 +9816,7 @@ instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRe
   ins_cost(100);
   // TODO: s390 port size(VARIABLE_SIZE); // Uses load_const_optimized.
   format %{ "FASTLOCK  $oop, $box; KILL Z_ARG4, Z_ARG5" %}
-  ins_encode %{ __ compiler_fast_lock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register,
-                                             UseBiasedLocking && !UseOptoBiasInlining); %}
+  ins_encode %{ __ compiler_fast_lock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); %}
   ins_pipe(pipe_class_dummy);
 %}
 
@@ -9818,10 +9824,9 @@ instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, i
   match(Set pcc (FastUnlock oop box));
   effect(TEMP tmp1, TEMP tmp2);
   ins_cost(100);
-  // TODO: s390 port size(FIXED_SIZE);  // emitted code depends on UseBiasedLocking being on/off.
+  // TODO: s390 port size(FIXED_SIZE);
   format %{ "FASTUNLOCK  $oop, $box; KILL Z_ARG4, Z_ARG5" %}
-  ins_encode %{ __ compiler_fast_unlock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register,
-                                               UseBiasedLocking && !UseOptoBiasInlining); %}
+  ins_encode %{ __ compiler_fast_unlock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); %}
   ins_pipe(pipe_class_dummy);
 %}
 
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index 95facb3a2ef..949f3d206e6 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -1873,13 +1873,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
     // Get the lock box slot's address.
     __ add2reg(r_box, lock_offset, Z_SP);
 
-#ifdef ASSERT
-    if (UseBiasedLocking)
-      // Making the box point to itself will make it clear it went unused
-      // but also be obviously invalid.
-      __ z_stg(r_box, 0, r_box);
-#endif // ASSERT
-
     // Try fastpath for locking.
     // Fast_lock kills r_temp_1, r_temp_2. (Don't use R1 as temp, won't work!)
     __ compiler_fast_lock_object(r_oop, r_box, r_tmp1, r_tmp2);
diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp
index e28481ddd86..49eab73a198 100644
--- a/src/hotspot/cpu/s390/templateTable_s390.cpp
+++ b/src/hotspot/cpu/s390/templateTable_s390.cpp
@@ -3768,9 +3768,8 @@ void TemplateTable::_new() {
 
   // Get instance_size in InstanceKlass (scaled to a count of bytes).
   Register Rsize = offset;
-  const int mask = 1 << Klass::_lh_instance_slow_path_bit;
   __ z_llgf(Rsize, Address(iklass, Klass::layout_helper_offset()));
-  __ z_tmll(Rsize, mask);
+  __ z_tmll(Rsize, Klass::_lh_instance_slow_path_bit);
   __ z_btrue(slow_case);
 
   // Allocate the instance
@@ -3813,14 +3812,8 @@ void TemplateTable::_new() {
 
     // Initialize object header only.
     __ bind(initialize_header);
-    if (UseBiasedLocking) {
-      Register prototype = RobjectFields;
-      __ z_lg(prototype, Address(iklass, Klass::prototype_header_offset()));
-      __ z_stg(prototype, Address(RallocatedObject, oopDesc::mark_offset_in_bytes()));
-    } else {
-      __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
-                     (long)markWord::prototype().value());
-    }
+    __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
+                   (long)markWord::prototype().value());
 
     __ store_klass_gap(Rzero, RallocatedObject);  // Zero klass gap for compressed oops.
     __ store_klass(iklass, RallocatedObject);     // Store klass last.
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 7c6bbc37eec..aee326133e0 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -30,7 +30,6 @@
 #include "interpreter/interpreter.hpp"
 #include "memory/resourceArea.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -3758,6 +3757,15 @@ void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int ve
   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 }
 
+void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+  assert(VM_Version::supports_avx512_vbmi(), "");
+  InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  attributes.set_is_evex_instruction();
+  vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int8((unsigned char)0x8D);
+  emit_operand(dst, src);
+}
+
 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
@@ -3830,6 +3838,22 @@ void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int
   emit_int16(0x76, (0xC0 | encode));
 }
 
+void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+  assert(VM_Version::supports_avx512_vbmi(), "");
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  attributes.set_is_evex_instruction();
+  int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int16(0x7D, (0xC0 | encode));
+}
+
+void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
+  assert(VM_Version::supports_avx512_vbmi(), "");
+  InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  attributes.set_is_evex_instruction();
+  int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
+}
+
 void Assembler::pause() {
   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 }
@@ -4128,6 +4152,15 @@ void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 }
 
+void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+  assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
+  InstructionMark im(this);
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
+  vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int8((unsigned char)0x8C);
+  emit_operand(dst, src);
+}
+
 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
   assert(VM_Version::supports_sse4_1(), "");
   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -4549,6 +4582,15 @@ void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int
   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 }
 
+void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
+assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
+       vector_len == AVX_256bit? VM_Version::supports_avx2() :
+       vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
+  InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+  int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int16(0x04, (0xC0 | encode));
+}
+
 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
   assert(VM_Version::supports_evex(), "");
   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
@@ -4857,6 +4899,15 @@ void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
   emit_int16(0x17, (0xC0 | encode));
 }
 
+void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+  assert(VM_Version::supports_avx512vlbw(), "");
+  // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  attributes.set_is_evex_instruction();
+  int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+  emit_int16((unsigned char)0x26, (0xC0 | encode));
+}
+
 void Assembler::punpcklbw(XMMRegister dst, Address src) {
   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
@@ -6539,6 +6590,13 @@ void Assembler::psubq(XMMRegister dst, XMMRegister src) {
   emit_int8((0xC0 | encode));
 }
 
+void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+  assert(UseAVX > 0, "requires some form of AVX");
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+  int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+  emit_int16((unsigned char)0xD8, (0xC0 | encode));
+}
+
 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
   assert(UseAVX > 0, "requires some form of AVX");
   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -6630,6 +6688,15 @@ void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 }
 
+void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+  assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
+         (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
+         (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+  int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+  emit_int16((unsigned char)0xE4, (0xC0 | encode));
+}
+
 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
   assert(UseAVX > 0, "requires some form of AVX");
   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -7402,6 +7469,10 @@ void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_
   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 }
 
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
   assert(VM_Version::supports_evex(), "requires EVEX support");
   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
@@ -7419,6 +7490,7 @@ void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_
   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 }
+PRAGMA_DIAG_POP
 
 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
   assert(VM_Version::supports_evex(), "requires EVEX support");
@@ -9404,6 +9476,13 @@ void Assembler::shlxq(Register dst, Register src1, Register src2) {
   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
 
+void Assembler::shrxl(Register dst, Register src1, Register src2) {
+  assert(VM_Version::supports_bmi2(), "");
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
+  emit_int16((unsigned char)0xF7, (0xC0 | encode));
+}
+
 void Assembler::shrxq(Register dst, Register src1, Register src2) {
   assert(VM_Version::supports_bmi2(), "");
   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 09849fc49f8..5976597019b 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -28,8 +28,6 @@
 #include "asm/register.hpp"
 #include "utilities/powerOfTwo.hpp"
 
-class BiasedLockingCounters;
-
 // Contains all the definitions needed for x86 assembly code generation.
 
 // Calling convention
@@ -1690,6 +1688,7 @@ class Assembler : public AbstractAssembler  {
   void vpermq(XMMRegister dst, XMMRegister src, int imm8);
   void vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
   void vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+  void vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
   void vpermw(XMMRegister dst,  XMMRegister nds, XMMRegister src, int vector_len);
   void vpermd(XMMRegister dst,  XMMRegister nds, Address src, int vector_len);
   void vpermd(XMMRegister dst,  XMMRegister nds, XMMRegister src, int vector_len);
@@ -1699,6 +1698,8 @@ class Assembler : public AbstractAssembler  {
   void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
   void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
   void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+  void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+  void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);
 
   void pause();
 
@@ -1747,6 +1748,7 @@ class Assembler : public AbstractAssembler  {
 
   void pmovmskb(Register dst, XMMRegister src);
   void vpmovmskb(Register dst, XMMRegister src, int vec_enc);
+  void vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
 
   // SSE 4.1 extract
   void pextrd(Register dst, XMMRegister src, int imm8);
@@ -1812,6 +1814,8 @@ class Assembler : public AbstractAssembler  {
   // Multiply add
   void pmaddwd(XMMRegister dst, XMMRegister src);
   void vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+  void vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+
   // Multiply add accumulate
   void evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
 
@@ -1879,6 +1883,8 @@ class Assembler : public AbstractAssembler  {
   void vptest(XMMRegister dst, XMMRegister src);
   void vptest(XMMRegister dst, Address src);
 
+  void evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+
   // Vector compare
   void vptest(XMMRegister dst, XMMRegister src, int vector_len);
 
@@ -2140,6 +2146,7 @@ class Assembler : public AbstractAssembler  {
 
   void shlxl(Register dst, Register src1, Register src2);
   void shlxq(Register dst, Register src1, Register src2);
+  void shrxl(Register dst, Register src1, Register src2);
   void shrxq(Register dst, Register src1, Register src2);
 
   void bzhiq(Register dst, Register src1, Register src2);
@@ -2244,6 +2251,7 @@ class Assembler : public AbstractAssembler  {
   void psubw(XMMRegister dst, XMMRegister src);
   void psubd(XMMRegister dst, XMMRegister src);
   void psubq(XMMRegister dst, XMMRegister src);
+  void vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
   void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
   void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
   void vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
@@ -2264,6 +2272,7 @@ class Assembler : public AbstractAssembler  {
   void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
   void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
   void vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+  void vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
 
   // Minimum of packed integers
   void pminsb(XMMRegister dst, XMMRegister src);
diff --git a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
index 96df588b409..a989b486e2c 100644
--- a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
@@ -638,7 +638,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->add_call_info_here(info());
 
 #ifndef PRODUCT
-  __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+  if (PrintC1Statistics) {
+    __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+  }
 #endif
 
   __ jmp(_continuation);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index b637db12c55..38bb2d912ea 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -980,7 +980,7 @@ void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool po
 }
 
 
-void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
+void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
   LIR_Address* to_addr = dest->as_address_ptr();
   PatchingStub* patch = NULL;
   Register compressed_src = rscratch1;
@@ -1206,7 +1206,7 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 }
 
 
-void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
   assert(src->is_address(), "should not call otherwise");
   assert(dest->is_register(), "should not call otherwise");
 
@@ -3722,13 +3722,9 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
   if (!UseFastLocking) {
     __ jmp(*op->stub()->entry());
   } else if (op->code() == lir_lock) {
-    Register scratch = noreg;
-    if (UseBiasedLocking) {
-      scratch = op->scratch_opr()->as_register();
-    }
     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
     // add debug info for NullPointerException only if one is possible
-    int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
+    int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
     if (op->info() != NULL) {
       add_debug_info_for_null_check(null_check_offset, op->info());
     }
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index 1647d1a52e2..13492a06f0b 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -302,9 +302,9 @@ void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
 
   // "lock" stores the address of the monitor stack slot, so this is not an oop
   LIR_Opr lock = new_register(T_INT);
-  // Need a scratch register for biased locking on x86
+  // Need a scratch register for inline types on x86
   LIR_Opr scratch = LIR_OprFact::illegalOpr;
-  if (UseBiasedLocking || x->maybe_inlinetype()) {
+  if (EnableValhalla && x->maybe_inlinetype()) {
     scratch = new_register(T_INT);
   }
 
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 5b82cadbc25..7e1f4dac1e0 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -33,13 +33,12 @@
 #include "oops/arrayOop.hpp"
 #include "oops/markWord.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/os.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
   const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
   const int aligned_mask = BytesPerWord -1;
   const int hdr_offset = oopDesc::mark_offset_in_bytes();
@@ -62,17 +61,11 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
     jcc(Assembler::notZero, slow_case);
   }
 
-  if (UseBiasedLocking) {
-    assert(scratch != noreg, "should have scratch register at this point");
-    biased_locking_enter(disp_hdr, obj, hdr, scratch, rklass_decode_tmp, false, done, &slow_case);
-  }
-
   // Load object header
   movptr(hdr, Address(obj, hdr_offset));
   // and mark it as unlocked
   orptr(hdr, markWord::unlocked_value);
   if (EnableValhalla) {
-    assert(!UseBiasedLocking, "Not compatible with biased-locking");
     // Mask inline_type bit such that we go to the slow path if object is an inline type
     andptr(hdr, ~((int) markWord::inline_type_bit_in_place));
   }
@@ -84,10 +77,6 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
   MacroAssembler::lock(); // must be immediately before cmpxchg!
   cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
   // if the object header was the same, we're done
-  if (PrintBiasedLockingStatistics) {
-    cond_inc32(Assembler::equal,
-               ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
-  }
   jcc(Assembler::equal, done);
   // if the object header was not the same, it is now in the hdr register
   // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
@@ -122,22 +111,15 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
   Label done;
 
-  if (UseBiasedLocking) {
-    // load object
-    movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-    biased_locking_exit(obj, hdr, done);
-  }
-
   // load displaced header
   movptr(hdr, Address(disp_hdr, 0));
   // if the loaded hdr is NULL we had recursive locking
   testptr(hdr, hdr);
   // if we had recursive locking, we are done
   jcc(Assembler::zero, done);
-  if (!UseBiasedLocking) {
-    // load object
-    movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
-  }
+  // load object
+  movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
+
   verify_oop(obj);
   // test if object header is pointing to the displaced header, and if so, restore
   // the displaced header in the object - if the object header is not pointing to
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
index 77d5fc6eccd..374b1ac3be3 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
@@ -49,9 +49,8 @@
   // hdr     : must be rax, contents destroyed
   // obj     : must point to the object to lock, contents preserved
   // disp_hdr: must point to the displaced header location, contents preserved
-  // scratch : scratch register, contents destroyed
   // returns code offset at which to add null check debug information
-  int lock_object  (Register swap, Register obj, Register disp_hdr, Register scratch, Label& slow_case);
+  int lock_object  (Register swap, Register obj, Register disp_hdr, Label& slow_case);
 
   // unlocking
   // hdr     : contents destroyed
diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
index 3b069c2b25b..f39c86d6181 100644
--- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
@@ -318,7 +318,11 @@ enum reg_save_layout {
 // expensive.  The deopt blob is the only thing which needs to
 // describe FPU registers.  In all other cases it should be sufficient
 // to simply save their current value.
-
+//
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for rax. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
 static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
                                 bool save_fpu_registers = true) {
 
@@ -418,6 +422,7 @@ static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
 
   return map;
 }
+PRAGMA_DIAG_POP
 
 #define __ this->
 
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index bcb526b3bd8..77c6c55a553 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -30,7 +30,6 @@
 #include "opto/intrinsicnode.hpp"
 #include "opto/opcodes.hpp"
 #include "opto/subnode.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/stubRoutines.hpp"
 
@@ -234,7 +233,6 @@ void C2_MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Regi
                                          Metadata* method_data, bool profile_rtm,
                                          Label& DONE_LABEL, Label& IsInflated) {
   assert(UseRTMForStackLocks, "why call this otherwise?");
-  assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
   assert(tmpReg == rax, "");
   assert(scrReg == rdx, "");
   Label L_rtm_retry, L_decrement_retry, L_on_abort;
@@ -244,7 +242,7 @@ void C2_MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Regi
     bind(L_rtm_retry);
   }
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
-  testptr(tmpReg, markWord::monitor_value);  // inflated vs stack-locked|neutral|biased
+  testptr(tmpReg, markWord::monitor_value);  // inflated vs stack-locked|neutral
   jcc(Assembler::notZero, IsInflated);
 
   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
@@ -259,8 +257,8 @@ void C2_MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Regi
   }
   xbegin(L_on_abort);
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
-  andptr(tmpReg, markWord::biased_lock_mask_in_place); // look at 3 lock bits
-  cmpptr(tmpReg, markWord::unlocked_value);            // bits = 001 unlocked
+  andptr(tmpReg, markWord::lock_mask_in_place);     // look at 2 lock bits
+  cmpptr(tmpReg, markWord::unlocked_value);         // bits = 01 unlocked
   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
 
   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
@@ -447,7 +445,6 @@ void C2_MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, R
 // scr: tmp -- KILLED
 void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
                                  Register scrReg, Register cx1Reg, Register cx2Reg,
-                                 BiasedLockingCounters* counters,
                                  RTMLockingCounters* rtm_counters,
                                  RTMLockingCounters* stack_rtm_counters,
                                  Metadata* method_data,
@@ -462,10 +459,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
   }
 
-  if (counters != NULL) {
-    atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
-  }
-
   // Possible cases that we'll encounter in fast_lock
   // ------------------------------------------------
   // * Inflated
@@ -473,9 +466,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
   //    -- Locked
   //       = by self
   //       = by other
-  // * biased
-  //    -- by Self
-  //    -- by other
   // * neutral
   // * stack-locked
   //    -- by self
@@ -493,16 +483,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
     jcc(Assembler::notZero, DONE_LABEL);
   }
 
-  // it's stack-locked, biased or neutral
-  // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
-  // order to reduce the number of conditional branches in the most common cases.
-  // Beware -- there's a subtle invariant that fetch of the markword
-  // at [FETCH], below, will never observe a biased encoding (*101b).
-  // If this invariant is not held we risk exclusion (safety) failure.
-  if (UseBiasedLocking && !UseOptoBiasInlining) {
-    biased_locking_enter(boxReg, objReg, tmpReg, scrReg, cx1Reg, false, DONE_LABEL, NULL, counters);
-  }
-
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
     rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
@@ -512,23 +492,18 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
 #endif // INCLUDE_RTM_OPT
 
   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
-  testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
+  testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral
   jccb(Assembler::notZero, IsInflated);
 
   // Attempt stack-locking ...
   orptr (tmpReg, markWord::unlocked_value);
   if (EnableValhalla) {
-    assert(!UseBiasedLocking, "Not compatible with biased-locking");
     // Mask inline_type bit such that we go to the slow path if object is an inline type
     andptr(tmpReg, ~((int) markWord::inline_type_bit_in_place));
   }
   movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
   lock();
   cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
-  if (counters != NULL) {
-    cond_inc32(Assembler::equal,
-               ExternalAddress((address)counters->fast_path_entry_count_addr()));
-  }
   jcc(Assembler::equal, DONE_LABEL);           // Success
 
   // Recursive locking.
@@ -538,10 +513,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
   // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
   andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
   movptr(Address(boxReg, 0), tmpReg);
-  if (counters != NULL) {
-    cond_inc32(Assembler::equal,
-               ExternalAddress((address)counters->fast_path_entry_count_addr()));
-  }
   jmp(DONE_LABEL);
 
   bind(IsInflated);
@@ -664,19 +635,12 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
 
   Label DONE_LABEL, Stacked, CheckSucc;
 
-  // Critically, the biased locking test must have precedence over
-  // and appear before the (box->dhw == 0) recursive stack-lock test.
-  if (UseBiasedLocking && !UseOptoBiasInlining) {
-    biased_locking_exit(objReg, tmpReg, DONE_LABEL);
-  }
-
 #if INCLUDE_RTM_OPT
   if (UseRTMForStackLocks && use_rtm) {
-    assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
     Label L_regular_unlock;
     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // fetch markword
-    andptr(tmpReg, markWord::biased_lock_mask_in_place);              // look at 3 lock bits
-    cmpptr(tmpReg, markWord::unlocked_value);                         // bits = 001 unlocked
+    andptr(tmpReg, markWord::lock_mask_in_place);                     // look at 2 lock bits
+    cmpptr(tmpReg, markWord::unlocked_value);                         // bits = 01 unlocked
     jccb(Assembler::notEqual, L_regular_unlock);                      // if !HLE RegularLock
     xend();                                                           // otherwise end...
     jmp(DONE_LABEL);                                                  // ... and we're done
@@ -743,7 +707,7 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   jmpb  (DONE_LABEL);
 
   bind (Stacked);
-  // It's not inflated and it's not recursively stack-locked and it's not biased.
+  // It's not inflated and it's not recursively stack-locked.
   // It must be stack-locked.
   // Try to reset the header to displaced header.
   // The "box" value on the stack is stable, so we can reload
@@ -1467,7 +1431,7 @@ void C2_MacroAssembler::evscatter(BasicType typ, Register base, XMMRegister idx,
   }
 }
 
-void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt) {
+void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy) {
   if (vlen_in_bytes <= 16) {
     pxor (dst, dst);
     psubb(dst, src);
@@ -1482,10 +1446,12 @@ void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int v
       default: assert(false, "%s", type2name(elem_bt));
     }
   } else {
+    assert(!is_legacy || !is_subword_type(elem_bt) || vlen_in_bytes < 64, "");
     int vlen_enc = vector_length_encoding(vlen_in_bytes);
 
     vpxor (dst, dst, dst, vlen_enc);
-    vpsubb(dst, dst, src, vlen_enc);
+    vpsubb(dst, dst, src, is_legacy ? AVX_256bit : vlen_enc);
+
     switch (elem_bt) {
       case T_BYTE:   /* nothing to do */            break;
       case T_SHORT:  vpmovsxbw(dst, dst, vlen_enc); break;
@@ -1501,7 +1467,11 @@ void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int v
 
 void C2_MacroAssembler::load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes) {
   ExternalAddress addr(StubRoutines::x86::vector_iota_indices());
-  if (vlen_in_bytes <= 16) {
+  if (vlen_in_bytes == 4) {
+    movdl(dst, addr);
+  } else if (vlen_in_bytes == 8) {
+    movq(dst, addr);
+  } else if (vlen_in_bytes == 16) {
     movdqu(dst, addr, scratch);
   } else if (vlen_in_bytes == 32) {
     vmovdqu(dst, addr, scratch);
@@ -1510,6 +1480,7 @@ void C2_MacroAssembler::load_iota_indices(XMMRegister dst, Register scratch, int
     evmovdqub(dst, k0, addr, false /*merge*/, Assembler::AVX_512bit, scratch);
   }
 }
+
 // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
 
 void C2_MacroAssembler::reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src) {
@@ -3892,6 +3863,9 @@ void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister
   vpxor(xtmp, xtmp, xtmp, vec_enc);
   vpsubb(xtmp, xtmp, mask, vec_enc);
   vpmovmskb(tmp, xtmp, vec_enc);
+  if (masklen < 64) {
+    andq(tmp, (((jlong)1 << masklen) - 1));
+  }
   switch(opc) {
     case Op_VectorMaskTrueCount:
       popcntq(dst, tmp);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 0a61be37bd1..2ef8e27cadc 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -38,7 +38,6 @@
   // See full desription in macroAssembler_x86.cpp.
   void fast_lock(Register obj, Register box, Register tmp,
                  Register scr, Register cx1, Register cx2,
-                 BiasedLockingCounters* counters,
                  RTMLockingCounters* rtm_counters,
                  RTMLockingCounters* stack_rtm_counters,
                  Metadata* method_data,
@@ -142,7 +141,7 @@
   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
   void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
 
-  void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt);
+  void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
   void load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes);
 
   // vector compare
diff --git a/src/hotspot/cpu/x86/c2_globals_x86.hpp b/src/hotspot/cpu/x86/c2_globals_x86.hpp
index 776caa30cf9..7e5128d7f2d 100644
--- a/src/hotspot/cpu/x86/c2_globals_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_globals_x86.hpp
@@ -46,8 +46,6 @@ define_pd_global(intx, FreqInlineSize,               325);
 define_pd_global(intx, MinJumpTableSize,             10);
 define_pd_global(intx, LoopPercentProfileLimit,      30);
 #ifdef AMD64
-define_pd_global(intx,  INTPRESSURE,                 13);
-define_pd_global(intx,  FLOATPRESSURE,               14);
 define_pd_global(intx,  InteriorEntryAlignment,      16);
 define_pd_global(size_t, NewSizeThreadIncrease,     ScaleForWordSize(4*K));
 define_pd_global(intx,  LoopUnrollLimit,             60);
@@ -58,8 +56,6 @@ define_pd_global(uintx, CodeCacheExpansionSize,      64*K);
 // Ergonomics related flags
 define_pd_global(uint64_t, MaxRAM,                   128ULL*G);
 #else
-define_pd_global(intx,  INTPRESSURE,                 6);
-define_pd_global(intx,  FLOATPRESSURE,               6);
 define_pd_global(intx,  InteriorEntryAlignment,      4);
 define_pd_global(size_t, NewSizeThreadIncrease,      4*K);
 define_pd_global(intx,  LoopUnrollLimit,             50);     // Design center runs on 1.3.1
diff --git a/src/hotspot/cpu/x86/frame_x86.cpp b/src/hotspot/cpu/x86/frame_x86.cpp
index 670fd769368..d957509bef7 100644
--- a/src/hotspot/cpu/x86/frame_x86.cpp
+++ b/src/hotspot/cpu/x86/frame_x86.cpp
@@ -356,9 +356,18 @@ frame frame::sender_for_entry_frame(RegisterMap* map) const {
   return fr;
 }
 
-JavaFrameAnchor* OptimizedEntryBlob::jfa_for_frame(const frame& frame) const {
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
+  assert(frame.is_optimized_entry_frame(), "wrong frame");
   // need unextended_sp here, since normal sp is wrong for interpreter callees
-  return reinterpret_cast<JavaFrameAnchor*>(reinterpret_cast<char*>(frame.unextended_sp()) + in_bytes(jfa_sp_offset()));
+  return reinterpret_cast<OptimizedEntryBlob::FrameData*>(
+    reinterpret_cast<char*>(frame.unextended_sp()) + in_bytes(_frame_data_offset));
+}
+
+bool frame::optimized_entry_frame_is_first() const {
+  assert(is_optimized_entry_frame(), "must be optimzed entry frame");
+  OptimizedEntryBlob* blob = _cb->as_optimized_entry_blob();
+  JavaFrameAnchor* jfa = blob->jfa_for_frame(*this);
+  return jfa->last_Java_sp() == NULL;
 }
 
 frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const {
@@ -367,6 +376,7 @@ frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const {
   // Java frame called from C; skip all C frames and return top C
   // frame of that chunk as the sender
   JavaFrameAnchor* jfa = blob->jfa_for_frame(*this);
+  assert(!optimized_entry_frame_is_first(), "must have a frame anchor to go back to");
   assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack");
   // Since we are walking the stack now this nested anchor is obviously walkable
   // even if it wasn't when it was stacked.
diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp
index 60847fbf391..733a357d5fe 100644
--- a/src/hotspot/cpu/x86/frame_x86.inline.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp
@@ -227,6 +227,10 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const {
 
 // Compiled frames
 
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for rax. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
 inline oop frame::saved_oop_result(RegisterMap* map) const {
   oop* result_adr = (oop *)map->location(rax->as_VMReg());
   guarantee(result_adr != NULL, "bad register save location");
@@ -240,5 +244,6 @@ inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) {
 
   *result_adr = obj;
 }
+PRAGMA_DIAG_POP
 
 #endif // CPU_X86_FRAME_X86_INLINE_HPP
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index f20cbc172b4..bad897f0555 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -446,7 +446,7 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
 
   __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
index d96ea768914..9da848f65bc 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -132,7 +132,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
     ResourceMark mark;
     log_trace(nmethod, barrier)("deoptimize(nmethod: %p, return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p",
                                nm, (address *) return_address_ptr, nm->is_osr_method(), jth,
-                               jth->get_thread_name(), callers_rsp, nm->verified_entry_point());
+                               jth->name(), callers_rsp, nm->verified_entry_point());
   }
 
   assert(nm->frame_size() >= 3, "invariant");
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
index 8a4f1d0e35d..df71d023cd3 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -844,7 +844,7 @@ void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, Shen
   Register pre_val_reg = stub->pre_val()->as_register();
 
   if (stub->do_load()) {
-    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
   }
 
   __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 22ed5ad4c21..595b29e603c 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -467,6 +467,10 @@ class ZSaveLiveRegisters {
     _spill_offset += 8;
   }
 
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for rax. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
   void initialize(ZLoadBarrierStubC2* stub) {
     // Create mask of caller saved registers that need to
     // be saved/restored if live
@@ -542,6 +546,7 @@ class ZSaveLiveRegisters {
     // Stack pointer must be 16 bytes aligned for the call
     _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
   }
+PRAGMA_DIAG_POP
 
 public:
   ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
diff --git a/src/hotspot/cpu/x86/globals_x86.hpp b/src/hotspot/cpu/x86/globals_x86.hpp
index 014bd1cd381..32e57e5024f 100644
--- a/src/hotspot/cpu/x86/globals_x86.hpp
+++ b/src/hotspot/cpu/x86/globals_x86.hpp
@@ -112,6 +112,9 @@ define_pd_global(bool, InlineTypeReturnedAsFields, LP64_ONLY(true) NOT_LP64(fals
           "Highest supported AVX instructions set on x86/x64")              \
           range(0, 99)                                                      \
                                                                             \
+  product(bool, UseKNLSetting, false, DIAGNOSTIC,                           \
+          "Control whether Knights platform setting should be used")        \
+                                                                            \
   product(bool, UseCLMUL, false,                                            \
           "Control whether CLMUL instructions can be used on x86/x64")      \
                                                                             \
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index 9a162d20086..dbb1466da8c 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -36,7 +36,6 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/basicLock.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -1349,8 +1348,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
     Label done;
 
     const Register swap_reg = rax; // Must use rax for cmpxchg instruction
-    const Register tmp_reg = rbx; // Will be passed to biased_locking_enter to avoid a
-                                  // problematic case where tmp_reg = no_reg.
+    const Register tmp_reg = rbx;
     const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
     const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
 
@@ -1371,17 +1369,12 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
       jcc(Assembler::notZero, slow_case);
     }
 
-    if (UseBiasedLocking) {
-      biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, rklass_decode_tmp, false, done, &slow_case);
-    }
-
     // Load immediate 1 into swap_reg %rax
     movl(swap_reg, (int32_t)1);
 
     // Load (object->mark() | 1) into swap_reg %rax
     orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
     if (EnableValhalla) {
-      assert(!UseBiasedLocking, "Not compatible with biased-locking");
       // Mask inline_type bit such that we go to the slow path if object is an inline type
       andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
     }
@@ -1394,10 +1387,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
 
     lock();
     cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
-    if (PrintBiasedLockingStatistics) {
-      cond_inc32(Assembler::zero,
-                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
-    }
     jcc(Assembler::zero, done);
 
     const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
@@ -1434,11 +1423,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
 
     // Save the test result, for recursive case, the result is zero
     movptr(Address(lock_reg, mark_offset), swap_reg);
-
-    if (PrintBiasedLockingStatistics) {
-      cond_inc32(Assembler::zero,
-                 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
-    }
     jcc(Assembler::zero, done);
 
     bind(slow_case);
@@ -1490,10 +1474,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
     // Free entry
     movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
 
-    if (UseBiasedLocking) {
-      biased_locking_exit(obj_reg, header_reg, done);
-    }
-
     // Load the old header from BasicLock structure
     movptr(header_reg, Address(swap_reg,
                                BasicLock::displaced_header_offset_in_bytes()));
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index f02f18d23db..7548a974a14 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -41,7 +41,6 @@
 #include "oops/compressedOops.inline.hpp"
 #include "oops/klass.inline.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/flags/flagSetting.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.hpp"
@@ -1286,200 +1285,6 @@ void MacroAssembler::reserved_stack_check() {
     bind(no_reserved_zone_enabling);
 }
 
-void MacroAssembler::biased_locking_enter(Register lock_reg,
-                                          Register obj_reg,
-                                          Register swap_reg,
-                                          Register tmp_reg,
-                                          Register tmp_reg2,
-                                          bool swap_reg_contains_mark,
-                                          Label& done,
-                                          Label* slow_case,
-                                          BiasedLockingCounters* counters) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-  assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
-  assert(tmp_reg != noreg, "tmp_reg must be supplied");
-  assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
-  assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout");
-  Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
-  NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
-
-  if (PrintBiasedLockingStatistics && counters == NULL) {
-    counters = BiasedLocking::counters();
-  }
-  // Biased locking
-  // See whether the lock is currently biased toward our thread and
-  // whether the epoch is still valid
-  // Note that the runtime guarantees sufficient alignment of JavaThread
-  // pointers to allow age to be placed into low bits
-  // First check to see whether biasing is even enabled for this object
-  Label cas_label;
-  if (!swap_reg_contains_mark) {
-    movptr(swap_reg, mark_addr);
-  }
-  movptr(tmp_reg, swap_reg);
-  andptr(tmp_reg, markWord::biased_lock_mask_in_place);
-  cmpptr(tmp_reg, markWord::biased_lock_pattern);
-  jcc(Assembler::notEqual, cas_label);
-  // The bias pattern is present in the object's header. Need to check
-  // whether the bias owner and the epoch are both still current.
-#ifndef _LP64
-  // Note that because there is no current thread register on x86_32 we
-  // need to store off the mark word we read out of the object to
-  // avoid reloading it and needing to recheck invariants below. This
-  // store is unfortunate but it makes the overall code shorter and
-  // simpler.
-  movptr(saved_mark_addr, swap_reg);
-#endif
-  load_prototype_header(tmp_reg, obj_reg, tmp_reg2);
-#ifdef _LP64
-  orptr(tmp_reg, r15_thread);
-  xorptr(tmp_reg, swap_reg);
-  Register header_reg = tmp_reg;
-#else
-  xorptr(tmp_reg, swap_reg);
-  get_thread(swap_reg);
-  xorptr(swap_reg, tmp_reg);
-  Register header_reg = swap_reg;
-#endif
-  andptr(header_reg, ~((int) markWord::age_mask_in_place));
-  if (counters != NULL) {
-    cond_inc32(Assembler::zero,
-               ExternalAddress((address) counters->biased_lock_entry_count_addr()));
-  }
-  jcc(Assembler::equal, done);
-
-  Label try_revoke_bias;
-  Label try_rebias;
-
-  // At this point we know that the header has the bias pattern and
-  // that we are not the bias owner in the current epoch. We need to
-  // figure out more details about the state of the header in order to
-  // know what operations can be legally performed on the object's
-  // header.
-
-  // If the low three bits in the xor result aren't clear, that means
-  // the prototype header is no longer biased and we have to revoke
-  // the bias on this object.
-  testptr(header_reg, markWord::biased_lock_mask_in_place);
-  jcc(Assembler::notZero, try_revoke_bias);
-
-  // Biasing is still enabled for this data type. See whether the
-  // epoch of the current bias is still valid, meaning that the epoch
-  // bits of the mark word are equal to the epoch bits of the
-  // prototype header. (Note that the prototype header's epoch bits
-  // only change at a safepoint.) If not, attempt to rebias the object
-  // toward the current thread. Note that we must be absolutely sure
-  // that the current epoch is invalid in order to do this because
-  // otherwise the manipulations it performs on the mark word are
-  // illegal.
-  testptr(header_reg, markWord::epoch_mask_in_place);
-  jccb(Assembler::notZero, try_rebias);
-
-  // The epoch of the current bias is still valid but we know nothing
-  // about the owner; it might be set or it might be clear. Try to
-  // acquire the bias of the object using an atomic operation. If this
-  // fails we will go in to the runtime to revoke the object's bias.
-  // Note that we first construct the presumed unbiased header so we
-  // don't accidentally blow away another thread's valid bias.
-  NOT_LP64( movptr(swap_reg, saved_mark_addr); )
-  andptr(swap_reg,
-         markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place);
-#ifdef _LP64
-  movptr(tmp_reg, swap_reg);
-  orptr(tmp_reg, r15_thread);
-#else
-  get_thread(tmp_reg);
-  orptr(tmp_reg, swap_reg);
-#endif
-  lock();
-  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
-  // If the biasing toward our thread failed, this means that
-  // another thread succeeded in biasing it toward itself and we
-  // need to revoke that bias. The revocation will occur in the
-  // interpreter runtime in the slow case.
-  if (counters != NULL) {
-    cond_inc32(Assembler::zero,
-               ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
-  }
-  if (slow_case != NULL) {
-    jcc(Assembler::notZero, *slow_case);
-  }
-  jmp(done);
-
-  bind(try_rebias);
-  // At this point we know the epoch has expired, meaning that the
-  // current "bias owner", if any, is actually invalid. Under these
-  // circumstances _only_, we are allowed to use the current header's
-  // value as the comparison value when doing the cas to acquire the
-  // bias in the current epoch. In other words, we allow transfer of
-  // the bias from one thread to another directly in this situation.
-  //
-  // FIXME: due to a lack of registers we currently blow away the age
-  // bits in this situation. Should attempt to preserve them.
-  load_prototype_header(tmp_reg, obj_reg, tmp_reg2);
-#ifdef _LP64
-  orptr(tmp_reg, r15_thread);
-#else
-  get_thread(swap_reg);
-  orptr(tmp_reg, swap_reg);
-  movptr(swap_reg, saved_mark_addr);
-#endif
-  lock();
-  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
-  // If the biasing toward our thread failed, then another thread
-  // succeeded in biasing it toward itself and we need to revoke that
-  // bias. The revocation will occur in the runtime in the slow case.
-  if (counters != NULL) {
-    cond_inc32(Assembler::zero,
-               ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
-  }
-  if (slow_case != NULL) {
-    jcc(Assembler::notZero, *slow_case);
-  }
-  jmp(done);
-
-  bind(try_revoke_bias);
-  // The prototype mark in the klass doesn't have the bias bit set any
-  // more, indicating that objects of this data type are not supposed
-  // to be biased any more. We are going to try to reset the mark of
-  // this object to the prototype value and fall through to the
-  // CAS-based locking scheme. Note that if our CAS fails, it means
-  // that another thread raced us for the privilege of revoking the
-  // bias of this particular object, so it's okay to continue in the
-  // normal locking code.
-  //
-  // FIXME: due to a lack of registers we currently blow away the age
-  // bits in this situation. Should attempt to preserve them.
-  NOT_LP64( movptr(swap_reg, saved_mark_addr); )
-  load_prototype_header(tmp_reg, obj_reg, tmp_reg2);
-  lock();
-  cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
-  // Fall through to the normal CAS-based lock, because no matter what
-  // the result of the above CAS, some thread must have succeeded in
-  // removing the bias bit from the object's header.
-  if (counters != NULL) {
-    cond_inc32(Assembler::zero,
-               ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
-  }
-
-  bind(cas_label);
-}
-
-void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
-  assert(UseBiasedLocking, "why call this otherwise?");
-
-  // Check for biased locking unlock case, which is a no-op
-  // Note: we do not have to check the thread ID for two reasons.
-  // First, the interpreter checks for IllegalMonitorStateException at
-  // a higher level. Second, if the bias was revoked while we held the
-  // lock, the object could not be rebiased toward another thread, so
-  // the bias bit would be clear.
-  movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
-  andptr(temp_reg, markWord::biased_lock_mask_in_place);
-  cmpptr(temp_reg, markWord::biased_lock_pattern);
-  jcc(Assembler::equal, done);
-}
-
 void MacroAssembler::c2bool(Register x) {
   // implements x == 0 ? 0 : 1
   // note: must only look at least-significant byte of x
@@ -5738,6 +5543,7 @@ int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from
 
 // Move a value between registers/stack slots and update the reg_state
 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
+  assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
   if (reg_state[to->value()] == reg_written) {
     return true; // Already written
   }
@@ -5818,7 +5624,7 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
                                           RegState reg_state[]) {
   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
-  assert(from->is_valid(), "source must bevalid");
+  assert(from->is_valid(), "source must be valid");
   Register fromReg;
   if (from->is_reg()) {
     fromReg = from->as_Register();
@@ -5834,6 +5640,7 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
   VMReg toReg;
   BasicType bt;
   while (stream.next(toReg, bt)) {
+    assert(toReg->is_valid(), "destination must be valid");
     int off = sig->at(stream.sig_index())._offset;
     assert(off > 0, "offset in object should be positive");
     Address fromAddr = Address(fromReg, off);
@@ -5917,6 +5724,7 @@ bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int&
   VMReg fromReg;
   BasicType bt;
   while (stream.next(fromReg, bt)) {
+    assert(fromReg->is_valid(), "source must be valid");
     int off = sig->at(stream.sig_index())._offset;
     assert(off > 0, "offset in object should be positive");
     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 81523a4b3e7..047a0486c11 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -730,21 +730,6 @@ class MacroAssembler: public Assembler {
 
   void verify_tlab();
 
-  // Biased locking support
-  // lock_reg and obj_reg must be loaded up with the appropriate values.
-  // swap_reg must be rax, and is killed.
-  // tmp_reg is optional. If it is supplied (i.e., != noreg) it will
-  // be killed; if not supplied, push/pop will be used internally to
-  // allocate a temporary (inefficient, avoid if possible).
-  // Optional slow case is for implementations (interpreter and C1) which branch to
-  // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
-  void biased_locking_enter(Register lock_reg, Register obj_reg,
-                            Register swap_reg, Register tmp_reg,
-                            Register tmp_reg2, bool swap_reg_contains_mark,
-                            Label& done, Label* slow_case = NULL,
-                            BiasedLockingCounters* counters = NULL);
-  void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
-
   Condition negate_condition(Condition cond);
 
   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
index d067820fdc1..8a0e8e6cf24 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -62,12 +62,6 @@ void SharedRuntime::inline_check_hashcode_from_object_header(MacroAssembler* mas
   __ testptr(result, markWord::unlocked_value);
   __ jcc(Assembler::zero, slowCase);
 
-  if (UseBiasedLocking) {
-    // Check if biased and fall through to runtime if so
-    __ testptr(result, markWord::biased_lock_bit_in_place);
-    __ jcc(Assembler::notZero, slowCase);
-  }
-
   // get hash
 #ifdef _LP64
   // Read the header and build a mask to get its hash field.
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
index 3237e637d6a..0339a58d866 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
@@ -1841,11 +1841,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (UseBiasedLocking) {
-      // Note that oop_handle_reg is trashed during this call
-      __ biased_locking_enter(lock_reg, obj_reg, swap_reg, oop_handle_reg, noreg, false, lock_done, &slow_path_lock);
-    }
-
     // Load immediate 1 into swap_reg %rax,
     __ movptr(swap_reg, 1);
 
@@ -1878,11 +1873,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ jcc(Assembler::notEqual, slow_path_lock);
     // Slow path will re-enter here
     __ bind(lock_done);
-
-    if (UseBiasedLocking) {
-      // Re-fetch oop_handle_reg as we trashed it above
-      __ movptr(oop_handle_reg, Address(rsp, wordSize));
-    }
   }
 
 
@@ -2012,10 +2002,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Get locked oop from the handle we passed to jni
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (UseBiasedLocking) {
-      __ biased_locking_exit(obj_reg, rbx, done);
-    }
-
     // Simple recursive lock?
 
     __ cmpptr(Address(rbp, lock_slot_rbp_offset), (int32_t)NULL_WORD);
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index 03a17035a7d..e37e60ea3d0 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -169,6 +169,10 @@ class RegisterSaver {
   static void restore_result_registers(MacroAssembler* masm);
 };
 
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for rax. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors) {
   int off = 0;
   int num_xmm_regs = XMMRegisterImpl::number_of_registers;
@@ -361,6 +365,7 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_
 
   return map;
 }
+PRAGMA_DIAG_POP
 
 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {
   int num_xmm_regs = XMMRegisterImpl::number_of_registers;
@@ -1196,7 +1201,7 @@ static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
 }
 
 // ---------------------------------------------------------------
-AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
+AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
                                                             int comp_args_on_stack,
                                                             const GrowableArray<SigEntry>* sig,
                                                             const VMRegPair* regs,
@@ -1205,7 +1210,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
                                                             const GrowableArray<SigEntry>* sig_cc_ro,
                                                             const VMRegPair* regs_cc_ro,
                                                             AdapterFingerPrint* fingerprint,
-                                                            AdapterBlob*& new_adapter) {
+                                                            AdapterBlob*& new_adapter,
+                                                            bool allocate_code_blob) {
   address i2c_entry = __ pc();
   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
 
@@ -1282,8 +1288,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
 
   // The c2i adapters might safepoint and trigger a GC. The caller must make sure that
   // the GC knows about the location of oop argument locations passed to the c2i adapter.
-  bool caller_must_gc_arguments = (regs != regs_cc);
-  new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
+  if (allocate_code_blob) {
+    bool caller_must_gc_arguments = (regs != regs_cc);
+    new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
+  }
 
   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
 }
@@ -2326,17 +2334,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // Load the oop from the handle
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
-    if (UseBiasedLocking) {
-      __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, rscratch2, false, lock_done, &slow_path_lock);
-    }
-
     // Load immediate 1 into swap_reg %rax
     __ movl(swap_reg, 1);
 
     // Load (object->mark() | 1) into swap_reg %rax
     __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
     if (EnableValhalla) {
-      assert(!UseBiasedLocking, "Not compatible with biased-locking");
       // Mask inline_type bit such that we go to the slow path if object is an inline type
       __ andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
     }
@@ -2487,11 +2490,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ movptr(obj_reg, Address(oop_handle_reg, 0));
 
     Label done;
-
-    if (UseBiasedLocking) {
-      __ biased_locking_exit(obj_reg, old_hdr, done);
-    }
-
     // Simple recursive lock?
 
     __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), (int32_t)NULL_WORD);
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index 4066acbd2ac..ee38a82dab1 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -39,6 +39,7 @@
 #include "oops/objArrayKlass.hpp"
 #include "oops/oop.inline.hpp"
 #include "prims/methodHandles.hpp"
+#include "runtime/arguments.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -5325,164 +5326,691 @@ address generate_avx_ghash_processBlocks() {
     return start;
   }
 
-  //base64 character set
-  address base64_charset_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "base64_charset");
+  address base64_shuffle_addr()
+  {
+    __ align(64, (unsigned long long)__ pc());
+    StubCodeMark mark(this, "StubRoutines", "shuffle_base64");
     address start = __ pc();
-    __ emit_data64(0x0000004200000041, relocInfo::none);
-    __ emit_data64(0x0000004400000043, relocInfo::none);
-    __ emit_data64(0x0000004600000045, relocInfo::none);
-    __ emit_data64(0x0000004800000047, relocInfo::none);
-    __ emit_data64(0x0000004a00000049, relocInfo::none);
-    __ emit_data64(0x0000004c0000004b, relocInfo::none);
-    __ emit_data64(0x0000004e0000004d, relocInfo::none);
-    __ emit_data64(0x000000500000004f, relocInfo::none);
-    __ emit_data64(0x0000005200000051, relocInfo::none);
-    __ emit_data64(0x0000005400000053, relocInfo::none);
-    __ emit_data64(0x0000005600000055, relocInfo::none);
-    __ emit_data64(0x0000005800000057, relocInfo::none);
-    __ emit_data64(0x0000005a00000059, relocInfo::none);
-    __ emit_data64(0x0000006200000061, relocInfo::none);
-    __ emit_data64(0x0000006400000063, relocInfo::none);
-    __ emit_data64(0x0000006600000065, relocInfo::none);
-    __ emit_data64(0x0000006800000067, relocInfo::none);
-    __ emit_data64(0x0000006a00000069, relocInfo::none);
-    __ emit_data64(0x0000006c0000006b, relocInfo::none);
-    __ emit_data64(0x0000006e0000006d, relocInfo::none);
-    __ emit_data64(0x000000700000006f, relocInfo::none);
-    __ emit_data64(0x0000007200000071, relocInfo::none);
-    __ emit_data64(0x0000007400000073, relocInfo::none);
-    __ emit_data64(0x0000007600000075, relocInfo::none);
-    __ emit_data64(0x0000007800000077, relocInfo::none);
-    __ emit_data64(0x0000007a00000079, relocInfo::none);
-    __ emit_data64(0x0000003100000030, relocInfo::none);
-    __ emit_data64(0x0000003300000032, relocInfo::none);
-    __ emit_data64(0x0000003500000034, relocInfo::none);
-    __ emit_data64(0x0000003700000036, relocInfo::none);
-    __ emit_data64(0x0000003900000038, relocInfo::none);
-    __ emit_data64(0x0000002f0000002b, relocInfo::none);
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x0405030401020001, relocInfo::none);
+    __ emit_data64(0x0a0b090a07080607, relocInfo::none);
+    __ emit_data64(0x10110f100d0e0c0d, relocInfo::none);
+    __ emit_data64(0x1617151613141213, relocInfo::none);
+    __ emit_data64(0x1c1d1b1c191a1819, relocInfo::none);
+    __ emit_data64(0x222321221f201e1f, relocInfo::none);
+    __ emit_data64(0x2829272825262425, relocInfo::none);
+    __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none);
     return start;
   }
 
-  //base64 url character set
-  address base64url_charset_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "base64url_charset");
+  address base64_avx2_shuffle_addr()
+  {
+    __ align(32);
+    StubCodeMark mark(this, "StubRoutines", "avx2_shuffle_base64");
     address start = __ pc();
-    __ emit_data64(0x0000004200000041, relocInfo::none);
-    __ emit_data64(0x0000004400000043, relocInfo::none);
-    __ emit_data64(0x0000004600000045, relocInfo::none);
-    __ emit_data64(0x0000004800000047, relocInfo::none);
-    __ emit_data64(0x0000004a00000049, relocInfo::none);
-    __ emit_data64(0x0000004c0000004b, relocInfo::none);
-    __ emit_data64(0x0000004e0000004d, relocInfo::none);
-    __ emit_data64(0x000000500000004f, relocInfo::none);
-    __ emit_data64(0x0000005200000051, relocInfo::none);
-    __ emit_data64(0x0000005400000053, relocInfo::none);
-    __ emit_data64(0x0000005600000055, relocInfo::none);
-    __ emit_data64(0x0000005800000057, relocInfo::none);
-    __ emit_data64(0x0000005a00000059, relocInfo::none);
-    __ emit_data64(0x0000006200000061, relocInfo::none);
-    __ emit_data64(0x0000006400000063, relocInfo::none);
-    __ emit_data64(0x0000006600000065, relocInfo::none);
-    __ emit_data64(0x0000006800000067, relocInfo::none);
-    __ emit_data64(0x0000006a00000069, relocInfo::none);
-    __ emit_data64(0x0000006c0000006b, relocInfo::none);
-    __ emit_data64(0x0000006e0000006d, relocInfo::none);
-    __ emit_data64(0x000000700000006f, relocInfo::none);
-    __ emit_data64(0x0000007200000071, relocInfo::none);
-    __ emit_data64(0x0000007400000073, relocInfo::none);
-    __ emit_data64(0x0000007600000075, relocInfo::none);
-    __ emit_data64(0x0000007800000077, relocInfo::none);
-    __ emit_data64(0x0000007a00000079, relocInfo::none);
-    __ emit_data64(0x0000003100000030, relocInfo::none);
-    __ emit_data64(0x0000003300000032, relocInfo::none);
-    __ emit_data64(0x0000003500000034, relocInfo::none);
-    __ emit_data64(0x0000003700000036, relocInfo::none);
-    __ emit_data64(0x0000003900000038, relocInfo::none);
-    __ emit_data64(0x0000005f0000002d, relocInfo::none);
+    __ emit_data64(0x0809070805060405, relocInfo::none);
+    __ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none);
+    __ emit_data64(0x0405030401020001, relocInfo::none);
+    __ emit_data64(0x0a0b090a07080607, relocInfo::none);
+    return start;
+  }
 
+  address base64_avx2_input_mask_addr()
+  {
+    __ align(32);
+    StubCodeMark mark(this, "StubRoutines", "avx2_input_mask_base64");
+    address start = __ pc();
+    __ emit_data64(0x8000000000000000, relocInfo::none);
+    __ emit_data64(0x8000000080000000, relocInfo::none);
+    __ emit_data64(0x8000000080000000, relocInfo::none);
+    __ emit_data64(0x8000000080000000, relocInfo::none);
     return start;
   }
 
-  address base64_bswap_mask_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "bswap_mask_base64");
+  address base64_avx2_lut_addr()
+  {
+    __ align(32);
+    StubCodeMark mark(this, "StubRoutines", "avx2_lut_base64");
     address start = __ pc();
-    __ emit_data64(0x0504038002010080, relocInfo::none);
-    __ emit_data64(0x0b0a098008070680, relocInfo::none);
-    __ emit_data64(0x0908078006050480, relocInfo::none);
-    __ emit_data64(0x0f0e0d800c0b0a80, relocInfo::none);
-    __ emit_data64(0x0605048003020180, relocInfo::none);
-    __ emit_data64(0x0c0b0a8009080780, relocInfo::none);
-    __ emit_data64(0x0504038002010080, relocInfo::none);
-    __ emit_data64(0x0b0a098008070680, relocInfo::none);
+    __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
+    __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
+    __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
+    __ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
+
+    // URL LUT
+    __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
+    __ emit_data64(0x000020effcfcfcfc, relocInfo::none);
+    __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
+    __ emit_data64(0x000020effcfcfcfc, relocInfo::none);
+    return start;
+  }
 
+  address base64_encoding_table_addr()
+  {
+    __ align(64, (unsigned long long)__ pc());
+    StubCodeMark mark(this, "StubRoutines", "encoding_table_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x4847464544434241, relocInfo::none);
+    __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none);
+    __ emit_data64(0x5857565554535251, relocInfo::none);
+    __ emit_data64(0x6665646362615a59, relocInfo::none);
+    __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none);
+    __ emit_data64(0x767574737271706f, relocInfo::none);
+    __ emit_data64(0x333231307a797877, relocInfo::none);
+    __ emit_data64(0x2f2b393837363534, relocInfo::none);
+
+    // URL table
+    __ emit_data64(0x4847464544434241, relocInfo::none);
+    __ emit_data64(0x504f4e4d4c4b4a49, relocInfo::none);
+    __ emit_data64(0x5857565554535251, relocInfo::none);
+    __ emit_data64(0x6665646362615a59, relocInfo::none);
+    __ emit_data64(0x6e6d6c6b6a696867, relocInfo::none);
+    __ emit_data64(0x767574737271706f, relocInfo::none);
+    __ emit_data64(0x333231307a797877, relocInfo::none);
+    __ emit_data64(0x5f2d393837363534, relocInfo::none);
     return start;
   }
 
-  address base64_right_shift_mask_addr() {
+  // Code for generating Base64 encoding.
+  // Intrinsic function prototype in Base64.java:
+  // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp,
+  // boolean isURL) {
+  address generate_base64_encodeBlock()
+  {
     __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "right_shift_mask");
+    StubCodeMark mark(this, "StubRoutines", "implEncode");
     address start = __ pc();
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
-    __ emit_data64(0x0006000400020000, relocInfo::none);
+    __ enter();
+
+    // Save callee-saved registers before using them
+    __ push(r12);
+    __ push(r13);
+    __ push(r14);
+    __ push(r15);
+
+    // arguments
+    const Register source = c_rarg0;       // Source Array
+    const Register start_offset = c_rarg1; // start offset
+    const Register end_offset = c_rarg2;   // end offset
+    const Register dest = c_rarg3;   // destination array
+
+#ifndef _WIN64
+    const Register dp = c_rarg4;    // Position for writing to dest array
+    const Register isURL = c_rarg5; // Base64 or URL character set
+#else
+    const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64
+    const Address isURL_mem(rbp, 7 * wordSize);
+    const Register isURL = r10; // pick the volatile windows register
+    const Register dp = r12;
+    __ movl(dp, dp_mem);
+    __ movl(isURL, isURL_mem);
+#endif
+
+    const Register length = r14;
+    const Register encode_table = r13;
+    Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop;
+
+    // calculate length from offsets
+    __ movl(length, end_offset);
+    __ subl(length, start_offset);
+    __ cmpl(length, 0);
+    __ jcc(Assembler::lessEqual, L_exit);
+
+    // Code for 512-bit VBMI encoding.  Encodes 48 input bytes into 64
+    // output bytes. We read 64 input bytes and ignore the last 16, so be
+    // sure not to read past the end of the input buffer.
+    if (VM_Version::supports_avx512_vbmi()) {
+      __ cmpl(length, 64); // Do not overrun input buffer.
+      __ jcc(Assembler::below, L_not512);
+
+      __ shll(isURL, 6); // index into decode table based on isURL
+      __ lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr()));
+      __ addptr(encode_table, isURL);
+      __ shrl(isURL, 6); // restore isURL
+
+      __ mov64(rax, 0x3036242a1016040aull); // Shifts
+      __ evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15);
+      __ evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit);
+      __ evpbroadcastq(xmm1, rax, Assembler::AVX_512bit);
+
+      __ align(32);
+      __ BIND(L_vbmiLoop);
+
+      __ vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit);
+      __ subl(length, 48);
+
+      // Put the input bytes into the proper lanes for writing, then
+      // encode them.
+      __ evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit);
+      __ vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit);
+
+      // Write to destination
+      __ evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit);
+
+      __ addptr(dest, 64);
+      __ addptr(source, 48);
+      __ cmpl(length, 64);
+      __ jcc(Assembler::aboveEqual, L_vbmiLoop);
+
+      __ vzeroupper();
+    }
+
+    __ BIND(L_not512);
+    if (VM_Version::supports_avx2()
+        && VM_Version::supports_avx512vlbw()) {
+      /*
+      ** This AVX2 encoder is based off the paper at:
+      **      https://dl.acm.org/doi/10.1145/3132709
+      **
+      ** We use AVX2 SIMD instructions to encode 24 bytes into 32
+      ** output bytes.
+      **
+      */
+      // Lengths under 32 bytes are done with scalar routine
+      __ cmpl(length, 31);
+      __ jcc(Assembler::belowEqual, L_process3);
+
+      // Set up supporting constant table data
+      __ vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax);
+      // 6-bit mask for 2nd and 4th (and multiples) 6-bit values
+      __ movl(rax, 0x0fc0fc00);
+      __ vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax);
+      __ evpbroadcastd(xmm8, rax, Assembler::AVX_256bit);
+
+      // Multiplication constant for "shifting" right by 6 and 10
+      // bits
+      __ movl(rax, 0x04000040);
+
+      __ subl(length, 24);
+      __ evpbroadcastd(xmm7, rax, Assembler::AVX_256bit);
+
+      // For the first load, we mask off reading of the first 4
+      // bytes into the register. This is so we can get 4 3-byte
+      // chunks into each lane of the register, avoiding having to
+      // handle end conditions.  We then shuffle these bytes into a
+      // specific order so that manipulation is easier.
+      //
+      // The initial read loads the XMM register like this:
+      //
+      // Lower 128-bit lane:
+      // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
+      // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1
+      // | C2 | D0 | D1 | D2 |
+      // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
+      //
+      // Upper 128-bit lane:
+      // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
+      // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2
+      // | XX | XX | XX | XX |
+      // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
+      //
+      // Where A0 is the first input byte, B0 is the fourth, etc.
+      // The alphabetical significance denotes the 3 bytes to be
+      // consumed and encoded into 4 bytes.
+      //
+      // We then shuffle the register so each 32-bit word contains
+      // the sequence:
+      //    A1 A0 A2 A1, B1, B0, B2, B1, etc.
+      // Each of these byte sequences are then manipulated into 4
+      // 6-bit values ready for encoding.
+      //
+      // If we focus on one set of 3-byte chunks, changing the
+      // nomenclature such that A0 => a, A1 => b, and A2 => c, we
+      // shuffle such that each 24-bit chunk contains:
+      //
+      // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6
+      // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0
+      // Explain this step.
+      // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4
+      // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2
+      //
+      // W first and off all but bits 4-9 and 16-21 (c5..c0 and
+      // a5..a0) and shift them using a vector multiplication
+      // operation (vpmulhuw) which effectively shifts c right by 6
+      // bits and a right by 10 bits.  We similarly mask bits 10-15
+      // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4
+      // bits respecively.  This is done using vpmullw.  We end up
+      // with 4 6-bit values, thus splitting the 3 input bytes,
+      // ready for encoding:
+      //    0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0
+      //
+      // For translation, we recognize that there are 5 distinct
+      // ranges of legal Base64 characters as below:
+      //
+      //   +-------------+-------------+------------+
+      //   | 6-bit value | ASCII range |   offset   |
+      //   +-------------+-------------+------------+
+      //   |    0..25    |    A..Z     |     65     |
+      //   |   26..51    |    a..z     |     71     |
+      //   |   52..61    |    0..9     |     -4     |
+      //   |     62      |   + or -    | -19 or -17 |
+      //   |     63      |   / or _    | -16 or 32  |
+      //   +-------------+-------------+------------+
+      //
+      // We note that vpshufb does a parallel lookup in a
+      // destination register using the lower 4 bits of bytes from a
+      // source register.  If we use a saturated subtraction and
+      // subtract 51 from each 6-bit value, bytes from [0,51]
+      // saturate to 0, and [52,63] map to a range of [1,12].  We
+      // distinguish the [0,25] and [26,51] ranges by assigning a
+      // value of 13 for all 6-bit values less than 26.  We end up
+      // with:
+      //
+      //   +-------------+-------------+------------+
+      //   | 6-bit value |   Reduced   |   offset   |
+      //   +-------------+-------------+------------+
+      //   |    0..25    |     13      |     65     |
+      //   |   26..51    |      0      |     71     |
+      //   |   52..61    |    0..9     |     -4     |
+      //   |     62      |     11      | -19 or -17 |
+      //   |     63      |     12      | -16 or 32  |
+      //   +-------------+-------------+------------+
+      //
+      // We then use a final vpshufb to add the appropriate offset,
+      // translating the bytes.
+      //
+      // Load input bytes - only 28 bytes.  Mask the first load to
+      // not load into the full register.
+      __ vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit);
+
+      // Move 3-byte chunks of input (12 bytes) into 16 bytes,
+      // ordering by:
+      //   1, 0, 2, 1; 4, 3, 5, 4; etc.  This groups 6-bit chunks
+      //   for easy masking
+      __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit);
+
+      __ addl(start_offset, 24);
+
+      // Load masking register for first and third (and multiples)
+      // 6-bit values.
+      __ movl(rax, 0x003f03f0);
+      __ evpbroadcastd(xmm6, rax, Assembler::AVX_256bit);
+      // Multiplication constant for "shifting" left by 4 and 8 bits
+      __ movl(rax, 0x01000010);
+      __ evpbroadcastd(xmm5, rax, Assembler::AVX_256bit);
+
+      // Isolate 6-bit chunks of interest
+      __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit);
+
+      // Load constants for encoding
+      __ movl(rax, 0x19191919);
+      __ evpbroadcastd(xmm3, rax, Assembler::AVX_256bit);
+      __ movl(rax, 0x33333333);
+      __ evpbroadcastd(xmm4, rax, Assembler::AVX_256bit);
+
+      // Shift output bytes 0 and 2 into proper lanes
+      __ vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit);
+
+      // Mask and shift output bytes 1 and 3 into proper lanes and
+      // combine
+      __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit);
+      __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit);
+      __ vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit);
+
+      // Find out which are 0..25.  This indicates which input
+      // values fall in the range of 'A'-'Z', which require an
+      // additional offset (see comments above)
+      __ vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit);
+      __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit);
+      __ vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit);
+
+      // Load the proper lookup table
+      __ lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr()));
+      __ movl(r15, isURL);
+      __ shll(r15, 5);
+      __ vmovdqu(xmm2, Address(r11, r15));
+
+      // Shuffle the offsets based on the range calculation done
+      // above. This allows us to add the correct offset to the
+      // 6-bit value corresponding to the range documented above.
+      __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit);
+      __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit);
+
+      // Store the encoded bytes
+      __ vmovdqu(Address(dest, dp), xmm0);
+      __ addl(dp, 32);
+
+      __ cmpl(length, 31);
+      __ jcc(Assembler::belowEqual, L_process3);
+
+      __ align(32);
+      __ BIND(L_32byteLoop);
+
+      // Get next 32 bytes
+      __ vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4));
+
+      __ subl(length, 24);
+      __ addl(start_offset, 24);
+
+      // This logic is identical to the above, with only constant
+      // register loads removed.  Shuffle the input, mask off 6-bit
+      // chunks, shift them into place, then add the offset to
+      // encode.
+      __ vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit);
+
+      __ vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit);
+      __ vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit);
+      __ vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit);
+      __ vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit);
+      __ vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit);
+      __ vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit);
+      __ vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit);
+      __ vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit);
+      __ vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit);
+      __ vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit);
+
+      // Store the encoded bytes
+      __ vmovdqu(Address(dest, dp), xmm0);
+      __ addl(dp, 32);
+
+      __ cmpl(length, 31);
+      __ jcc(Assembler::above, L_32byteLoop);
+
+      __ BIND(L_process3);
+      __ vzeroupper();
+    } else {
+      __ BIND(L_process3);
+    }
+
+    __ cmpl(length, 3);
+    __ jcc(Assembler::below, L_exit);
+
+    // Load the encoding table based on isURL
+    __ lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr()));
+    __ movl(r15, isURL);
+    __ shll(r15, 6);
+    __ addptr(r11, r15);
+
+    __ BIND(L_processdata);
+
+    // Load 3 bytes
+    __ load_unsigned_byte(r15, Address(source, start_offset));
+    __ load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1));
+    __ load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2));
+
+    // Build a 32-bit word with bytes 1, 2, 0, 1
+    __ movl(rax, r10);
+    __ shll(r10, 24);
+    __ orl(rax, r10);
+
+    __ subl(length, 3);
+
+    __ shll(r15, 8);
+    __ shll(r13, 16);
+    __ orl(rax, r15);
+
+    __ addl(start_offset, 3);
+
+    __ orl(rax, r13);
+    // At this point, rax contains | byte1 | byte2 | byte0 | byte1
+    // r13 has byte2 << 16 - need low-order 6 bits to translate.
+    // This translated byte is the fourth output byte.
+    __ shrl(r13, 16);
+    __ andl(r13, 0x3f);
+
+    // The high-order 6 bits of r15 (byte0) is translated.
+    // The translated byte is the first output byte.
+    __ shrl(r15, 10);
+
+    __ load_unsigned_byte(r13, Address(r11, r13));
+    __ load_unsigned_byte(r15, Address(r11, r15));
+
+    __ movb(Address(dest, dp, Address::times_1, 3), r13);
+
+    // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0.
+    // This translated byte is the second output byte.
+    __ shrl(rax, 4);
+    __ movl(r10, rax);
+    __ andl(rax, 0x3f);
+
+    __ movb(Address(dest, dp, Address::times_1, 0), r15);
+
+    __ load_unsigned_byte(rax, Address(r11, rax));
+
+    // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2.
+    // This translated byte is the third output byte.
+    __ shrl(r10, 18);
+    __ andl(r10, 0x3f);
+
+    __ load_unsigned_byte(r10, Address(r11, r10));
+
+    __ movb(Address(dest, dp, Address::times_1, 1), rax);
+    __ movb(Address(dest, dp, Address::times_1, 2), r10);
+
+    __ addl(dp, 4);
+    __ cmpl(length, 3);
+    __ jcc(Assembler::aboveEqual, L_processdata);
 
+    __ BIND(L_exit);
+    __ pop(r15);
+    __ pop(r14);
+    __ pop(r13);
+    __ pop(r12);
+    __ leave();
+    __ ret(0);
     return start;
   }
 
-  address base64_left_shift_mask_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "left_shift_mask");
+  // base64 AVX512vbmi tables
+  address base64_vbmi_lookup_lo_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64");
     address start = __ pc();
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
-    __ emit_data64(0x0000000200040000, relocInfo::none);
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x3f8080803e808080, relocInfo::none);
+    __ emit_data64(0x3b3a393837363534, relocInfo::none);
+    __ emit_data64(0x8080808080803d3c, relocInfo::none);
+    return start;
+  }
 
+  address base64_vbmi_lookup_hi_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x0605040302010080, relocInfo::none);
+    __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
+    __ emit_data64(0x161514131211100f, relocInfo::none);
+    __ emit_data64(0x8080808080191817, relocInfo::none);
+    __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none);
+    __ emit_data64(0x2827262524232221, relocInfo::none);
+    __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
+    __ emit_data64(0x8080808080333231, relocInfo::none);
+    return start;
+  }
+  address base64_vbmi_lookup_lo_url_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64url");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x8080808080808080, relocInfo::none);
+    __ emit_data64(0x80803e8080808080, relocInfo::none);
+    __ emit_data64(0x3b3a393837363534, relocInfo::none);
+    __ emit_data64(0x8080808080803d3c, relocInfo::none);
     return start;
   }
 
-  address base64_and_mask_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "and_mask");
+  address base64_vbmi_lookup_hi_url_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64url");
     address start = __ pc();
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
-    __ emit_data64(0x3f003f003f000000, relocInfo::none);
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x0605040302010080, relocInfo::none);
+    __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
+    __ emit_data64(0x161514131211100f, relocInfo::none);
+    __ emit_data64(0x3f80808080191817, relocInfo::none);
+    __ emit_data64(0x201f1e1d1c1b1a80, relocInfo::none);
+    __ emit_data64(0x2827262524232221, relocInfo::none);
+    __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
+    __ emit_data64(0x8080808080333231, relocInfo::none);
     return start;
   }
 
-  address base64_gather_mask_addr() {
-    __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "gather_mask");
+  address base64_vbmi_pack_vec_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "pack_vec_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x090a040506000102, relocInfo::none);
+    __ emit_data64(0x161011120c0d0e08, relocInfo::none);
+    __ emit_data64(0x1c1d1e18191a1415, relocInfo::none);
+    __ emit_data64(0x292a242526202122, relocInfo::none);
+    __ emit_data64(0x363031322c2d2e28, relocInfo::none);
+    __ emit_data64(0x3c3d3e38393a3435, relocInfo::none);
+    __ emit_data64(0x0000000000000000, relocInfo::none);
+    __ emit_data64(0x0000000000000000, relocInfo::none);
+    return start;
+  }
+
+  address base64_vbmi_join_0_1_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "join_0_1_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x090a040506000102, relocInfo::none);
+    __ emit_data64(0x161011120c0d0e08, relocInfo::none);
+    __ emit_data64(0x1c1d1e18191a1415, relocInfo::none);
+    __ emit_data64(0x292a242526202122, relocInfo::none);
+    __ emit_data64(0x363031322c2d2e28, relocInfo::none);
+    __ emit_data64(0x3c3d3e38393a3435, relocInfo::none);
+    __ emit_data64(0x494a444546404142, relocInfo::none);
+    __ emit_data64(0x565051524c4d4e48, relocInfo::none);
+    return start;
+  }
+
+  address base64_vbmi_join_1_2_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "join_1_2_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x1c1d1e18191a1415, relocInfo::none);
+    __ emit_data64(0x292a242526202122, relocInfo::none);
+    __ emit_data64(0x363031322c2d2e28, relocInfo::none);
+    __ emit_data64(0x3c3d3e38393a3435, relocInfo::none);
+    __ emit_data64(0x494a444546404142, relocInfo::none);
+    __ emit_data64(0x565051524c4d4e48, relocInfo::none);
+    __ emit_data64(0x5c5d5e58595a5455, relocInfo::none);
+    __ emit_data64(0x696a646566606162, relocInfo::none);
+    return start;
+  }
+
+  address base64_vbmi_join_2_3_addr() {
+    __ align(64, (unsigned long long) __ pc());
+    StubCodeMark mark(this, "StubRoutines", "join_2_3_base64");
+    address start = __ pc();
+    assert(((unsigned long long)start & 0x3f) == 0,
+           "Alignment problem (0x%08llx)", (unsigned long long)start);
+    __ emit_data64(0x363031322c2d2e28, relocInfo::none);
+    __ emit_data64(0x3c3d3e38393a3435, relocInfo::none);
+    __ emit_data64(0x494a444546404142, relocInfo::none);
+    __ emit_data64(0x565051524c4d4e48, relocInfo::none);
+    __ emit_data64(0x5c5d5e58595a5455, relocInfo::none);
+    __ emit_data64(0x696a646566606162, relocInfo::none);
+    __ emit_data64(0x767071726c6d6e68, relocInfo::none);
+    __ emit_data64(0x7c7d7e78797a7475, relocInfo::none);
+    return start;
+  }
+
+  address base64_decoding_table_addr() {
+    StubCodeMark mark(this, "StubRoutines", "decoding_table_base64");
     address start = __ pc();
     __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0x3fffffff3effffff, relocInfo::none);
+    __ emit_data64(0x3b3a393837363534, relocInfo::none);
+    __ emit_data64(0xffffffffffff3d3c, relocInfo::none);
+    __ emit_data64(0x06050403020100ff, relocInfo::none);
+    __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
+    __ emit_data64(0x161514131211100f, relocInfo::none);
+    __ emit_data64(0xffffffffff191817, relocInfo::none);
+    __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none);
+    __ emit_data64(0x2827262524232221, relocInfo::none);
+    __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
+    __ emit_data64(0xffffffffff333231, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+
+    // URL table
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffff3effffffffff, relocInfo::none);
+    __ emit_data64(0x3b3a393837363534, relocInfo::none);
+    __ emit_data64(0xffffffffffff3d3c, relocInfo::none);
+    __ emit_data64(0x06050403020100ff, relocInfo::none);
+    __ emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
+    __ emit_data64(0x161514131211100f, relocInfo::none);
+    __ emit_data64(0x3fffffffff191817, relocInfo::none);
+    __ emit_data64(0x201f1e1d1c1b1aff, relocInfo::none);
+    __ emit_data64(0x2827262524232221, relocInfo::none);
+    __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
+    __ emit_data64(0xffffffffff333231, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
+    __ emit_data64(0xffffffffffffffff, relocInfo::none);
     return start;
   }
 
-// Code for generating Base64 encoding.
+
+// Code for generating Base64 decoding.
+//
+// Based on the article (and associated code) from https://arxiv.org/abs/1910.05109.
+//
 // Intrinsic function prototype in Base64.java:
-// private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL) {
-  address generate_base64_encodeBlock() {
+// private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) {
+  address generate_base64_decodeBlock() {
     __ align(CodeEntryAlignment);
-    StubCodeMark mark(this, "StubRoutines", "implEncode");
+    StubCodeMark mark(this, "StubRoutines", "implDecode");
     address start = __ pc();
     __ enter();
 
@@ -5491,16 +6019,19 @@ address generate_avx_ghash_processBlocks() {
     __ push(r13);
     __ push(r14);
     __ push(r15);
+    __ push(rbx);
 
     // arguments
     const Register source = c_rarg0; // Source Array
     const Register start_offset = c_rarg1; // start offset
     const Register end_offset = c_rarg2; // end offset
     const Register dest = c_rarg3; // destination array
+    const Register isMIME = rbx;
 
 #ifndef _WIN64
     const Register dp = c_rarg4;  // Position for writing to dest array
     const Register isURL = c_rarg5;// Base64 or URL character set
+    __ movl(isMIME, Address(rbp, 2 * wordSize));
 #else
     const Address  dp_mem(rbp, 6 * wordSize);  // length is on stack on Win64
     const Address isURL_mem(rbp, 7 * wordSize);
@@ -5508,219 +6039,404 @@ address generate_avx_ghash_processBlocks() {
     const Register dp = r12;
     __ movl(dp, dp_mem);
     __ movl(isURL, isURL_mem);
+    __ movl(isMIME, Address(rbp, 8 * wordSize));
 #endif
 
+    const XMMRegister lookup_lo = xmm5;
+    const XMMRegister lookup_hi = xmm6;
+    const XMMRegister errorvec = xmm7;
+    const XMMRegister pack16_op = xmm9;
+    const XMMRegister pack32_op = xmm8;
+    const XMMRegister input0 = xmm3;
+    const XMMRegister input1 = xmm20;
+    const XMMRegister input2 = xmm21;
+    const XMMRegister input3 = xmm19;
+    const XMMRegister join01 = xmm12;
+    const XMMRegister join12 = xmm11;
+    const XMMRegister join23 = xmm10;
+    const XMMRegister translated0 = xmm2;
+    const XMMRegister translated1 = xmm1;
+    const XMMRegister translated2 = xmm0;
+    const XMMRegister translated3 = xmm4;
+
+    const XMMRegister merged0 = xmm2;
+    const XMMRegister merged1 = xmm1;
+    const XMMRegister merged2 = xmm0;
+    const XMMRegister merged3 = xmm4;
+    const XMMRegister merge_ab_bc0 = xmm2;
+    const XMMRegister merge_ab_bc1 = xmm1;
+    const XMMRegister merge_ab_bc2 = xmm0;
+    const XMMRegister merge_ab_bc3 = xmm4;
+
+    const XMMRegister pack24bits = xmm4;
+
     const Register length = r14;
-    Label L_process80, L_process32, L_process3, L_exit, L_processdata;
+    const Register output_size = r13;
+    const Register output_mask = r15;
+    const KRegister input_mask = k1;
+
+    const XMMRegister input_initial_valid_b64 = xmm0;
+    const XMMRegister tmp = xmm10;
+    const XMMRegister mask = xmm0;
+    const XMMRegister invalid_b64 = xmm1;
+
+    Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL;
+    Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce;
+    Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero;
 
     // calculate length from offsets
     __ movl(length, end_offset);
     __ subl(length, start_offset);
+    __ push(dest);          // Save for return value calc
+
+    // If AVX512 VBMI not supported, just compile non-AVX code
+    if(VM_Version::supports_avx512_vbmi() &&
+       VM_Version::supports_avx512bw()) {
+      __ cmpl(length, 128);     // 128-bytes is break-even for AVX-512
+      __ jcc(Assembler::lessEqual, L_bruteForce);
+
+      __ cmpl(isMIME, 0);
+      __ jcc(Assembler::notEqual, L_bruteForce);
+
+      // Load lookup tables based on isURL
+      __ cmpl(isURL, 0);
+      __ jcc(Assembler::notZero, L_loadURL);
+
+      __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13);
+      __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13);
+
+      __ BIND(L_continue);
+
+      __ movl(r15, 0x01400140);
+      __ evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit);
+
+      __ movl(r15, 0x00011000);
+      __ evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit);
+
+      __ cmpl(length, 0xff);
+      __ jcc(Assembler::lessEqual, L_process64);
+
+      // load masks required for decoding data
+      __ BIND(L_processdata);
+      __ evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13);
+      __ evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13);
+      __ evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13);
+
+      __ align(32);
+      __ BIND(L_process256);
+      // Grab input data
+      __ evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit);
+      __ evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit);
+      __ evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit);
+      __ evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit);
+
+      // Copy the low part of the lookup table into the destination of the permutation
+      __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit);
+      __ evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit);
+      __ evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit);
+      __ evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit);
+
+      // Translate the base64 input into "decoded" bytes
+      __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit);
+      __ evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit);
+      __ evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit);
+      __ evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit);
+
+      // OR all of the translations together to check for errors (high-order bit of byte set)
+      __ vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit);
+
+      __ vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit);
+      __ vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit);
+      __ vpor(errorvec, input3, input0, Assembler::AVX_512bit);
+
+      // Check if there was an error - if so, try 64-byte chunks
+      __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit);
+      __ kortestql(k3, k3);
+      __ jcc(Assembler::notZero, L_process64);
+
+      // The merging and shuffling happens here
+      // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa]
+      // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd]
+      // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40
+      __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit);
+      __ vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit);
+      __ vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit);
+      __ vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit);
+
+      // Now do the same with packed 16-bit values.
+      // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb]
+      // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12
+      // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd]
+      __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit);
+      __ vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit);
+      __ vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit);
+      __ vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit);
+
+      // The join vectors specify which byte from which vector goes into the outputs
+      // One of every 4 bytes in the extended vector is zero, so we pack them into their
+      // final positions in the register for storing (256 bytes in, 192 bytes out)
+      __ evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit);
+      __ evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit);
+      __ evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit);
+
+      // Store result
+      __ evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit);
+      __ evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit);
+      __ evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit);
+
+      __ addptr(source, 0x100);
+      __ addptr(dest, 0xc0);
+      __ subl(length, 0x100);
+      __ cmpl(length, 64 * 4);
+      __ jcc(Assembler::greaterEqual, L_process256);
+
+      // At this point, we've decoded 64 * 4 * n bytes.
+      // The remaining length will be <= 64 * 4 - 1.
+      // UNLESS there was an error decoding the first 256-byte chunk.  In this
+      // case, the length will be arbitrarily long.
+      //
+      // Note that this will be the path for MIME-encoded strings.
+
+      __ BIND(L_process64);
+
+      __ evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13);
+
+      __ cmpl(length, 63);
+      __ jcc(Assembler::lessEqual, L_finalBit);
+
+      __ align(32);
+      __ BIND(L_process64Loop);
+
+      // Handle first 64-byte block
+
+      __ evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit);
+      __ evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit);
+      __ evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit);
+
+      __ vpor(errorvec, translated0, input0, Assembler::AVX_512bit);
+
+      // Check for error and bomb out before updating dest
+      __ evpmovb2m(k3, errorvec, Assembler::AVX_512bit);
+      __ kortestql(k3, k3);
+      __ jcc(Assembler::notZero, L_exit);
+
+      // Pack output register, selecting correct byte ordering
+      __ vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit);
+      __ vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit);
+      __ vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit);
+
+      __ evmovdquq(Address(dest, dp), merged0, Assembler::AVX_512bit);
+
+      __ subl(length, 64);
+      __ addptr(source, 64);
+      __ addptr(dest, 48);
+
+      __ cmpl(length, 64);
+      __ jcc(Assembler::greaterEqual, L_process64Loop);
+
+      __ cmpl(length, 0);
+      __ jcc(Assembler::lessEqual, L_exit);
+
+      __ BIND(L_finalBit);
+      // Now have 1 to 63 bytes left to decode
+
+      // I was going to let Java take care of the final fragment
+      // however it will repeatedly call this routine for every 4 bytes
+      // of input data, so handle the rest here.
+      __ movq(rax, -1);
+      __ bzhiq(rax, rax, length);    // Input mask in rax
+
+      __ movl(output_size, length);
+      __ shrl(output_size, 2);   // Find (len / 4) * 3 (output length)
+      __ lea(output_size, Address(output_size, output_size, Address::times_2, 0));
+      // output_size in r13
+
+      // Strip pad characters, if any, and adjust length and mask
+      __ cmpb(Address(source, length, Address::times_1, -1), '=');
+      __ jcc(Assembler::equal, L_padding);
+
+      __ BIND(L_donePadding);
+
+      // Output size is (64 - output_size), output mask is (all 1s >> output_size).
+      __ kmovql(input_mask, rax);
+      __ movq(output_mask, -1);
+      __ bzhiq(output_mask, output_mask, output_size);
+
+      // Load initial input with all valid base64 characters.  Will be used
+      // in merging source bytes to avoid masking when determining if an error occurred.
+      __ movl(rax, 0x61616161);
+      __ evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit);
+
+      // A register containing all invalid base64 decoded values
+      __ movl(rax, 0x80808080);
+      __ evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit);
+
+      // input_mask is in k1
+      // output_size is in r13
+      // output_mask is in r15
+      // zmm0 - free
+      // zmm1 - 0x00011000
+      // zmm2 - 0x01400140
+      // zmm3 - errorvec
+      // zmm4 - pack vector
+      // zmm5 - lookup_lo
+      // zmm6 - lookup_hi
+      // zmm7 - errorvec
+      // zmm8 - 0x61616161
+      // zmm9 - 0x80808080
+
+      // Load only the bytes from source, merging into our "fully-valid" register
+      __ evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit);
+
+      // Decode all bytes within our merged input
+      __ evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit);
+      __ evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit);
+      __ vporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit);
+
+      // Check for error.  Compare (decoded | initial) to all invalid.
+      // If any bytes have their high-order bit set, then we have an error.
+      __ evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit);
+      __ kortestql(k2, k2);
+
+      // If we have an error, use the brute force loop to decode what we can (4-byte chunks).
+      __ jcc(Assembler::notZero, L_bruteForce);
+
+      // Shuffle output bytes
+      __ vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit);
+      __ vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit);
+
+      __ vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit);
+      __ kmovql(k1, output_mask);
+      __ evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit);
+
+      __ addptr(dest, output_size);
+
+      __ BIND(L_exit);
+      __ vzeroupper();
+      __ pop(rax);             // Get original dest value
+      __ subptr(dest, rax);      // Number of bytes converted
+      __ movptr(rax, dest);
+      __ pop(rbx);
+      __ pop(r15);
+      __ pop(r14);
+      __ pop(r13);
+      __ pop(r12);
+      __ leave();
+      __ ret(0);
+
+      __ BIND(L_loadURL);
+      __ evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13);
+      __ evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13);
+      __ jmp(L_continue);
+
+      __ BIND(L_padding);
+      __ decrementq(output_size, 1);
+      __ shrq(rax, 1);
+
+      __ cmpb(Address(source, length, Address::times_1, -2), '=');
+      __ jcc(Assembler::notEqual, L_donePadding);
+
+      __ decrementq(output_size, 1);
+      __ shrq(rax, 1);
+      __ jmp(L_donePadding);
+
+      __ align(32);
+      __ BIND(L_bruteForce);
+    }   // End of if(avx512_vbmi)
+
+    // Use non-AVX code to decode 4-byte chunks into 3 bytes of output
+
+    // Register state (Linux):
+    // r12-15 - saved on stack
+    // rdi - src
+    // rsi - sp
+    // rdx - sl
+    // rcx - dst
+    // r8 - dp
+    // r9 - isURL
+
+    // Register state (Windows):
+    // r12-15 - saved on stack
+    // rcx - src
+    // rdx - sp
+    // r8 - sl
+    // r9 - dst
+    // r12 - dp
+    // r10 - isURL
+
+    // Registers (common):
+    // length (r14) - bytes in src
+
+    const Register decode_table = r11;
+    const Register out_byte_count = rbx;
+    const Register byte1 = r13;
+    const Register byte2 = r15;
+    const Register byte3 = WINDOWS_ONLY(r8) NOT_WINDOWS(rdx);
+    const Register byte4 = WINDOWS_ONLY(r10) NOT_WINDOWS(r9);
+
+    __ shrl(length, 2);    // Multiple of 4 bytes only - length is # 4-byte chunks
     __ cmpl(length, 0);
-    __ jcc(Assembler::lessEqual, L_exit);
+    __ jcc(Assembler::lessEqual, L_exit_no_vzero);
 
-    __ lea(r11, ExternalAddress(StubRoutines::x86::base64_charset_addr()));
-    // check if base64 charset(isURL=0) or base64 url charset(isURL=1) needs to be loaded
-    __ cmpl(isURL, 0);
-    __ jcc(Assembler::equal, L_processdata);
-    __ lea(r11, ExternalAddress(StubRoutines::x86::base64url_charset_addr()));
+    __ shll(isURL, 8);    // index into decode table based on isURL
+    __ lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr()));
+    __ addptr(decode_table, isURL);
 
-    // load masks required for encoding data
-    __ BIND(L_processdata);
-    __ movdqu(xmm16, ExternalAddress(StubRoutines::x86::base64_gather_mask_addr()));
-    // Set 64 bits of K register.
-    __ evpcmpeqb(k3, xmm16, xmm16, Assembler::AVX_512bit);
-    __ evmovdquq(xmm12, ExternalAddress(StubRoutines::x86::base64_bswap_mask_addr()), Assembler::AVX_256bit, r13);
-    __ evmovdquq(xmm13, ExternalAddress(StubRoutines::x86::base64_right_shift_mask_addr()), Assembler::AVX_512bit, r13);
-    __ evmovdquq(xmm14, ExternalAddress(StubRoutines::x86::base64_left_shift_mask_addr()), Assembler::AVX_512bit, r13);
-    __ evmovdquq(xmm15, ExternalAddress(StubRoutines::x86::base64_and_mask_addr()), Assembler::AVX_512bit, r13);
-
-    // Vector Base64 implementation, producing 96 bytes of encoded data
-    __ BIND(L_process80);
-    __ cmpl(length, 80);
-    __ jcc(Assembler::below, L_process32);
-    __ evmovdquq(xmm0, Address(source, start_offset, Address::times_1, 0), Assembler::AVX_256bit);
-    __ evmovdquq(xmm1, Address(source, start_offset, Address::times_1, 24), Assembler::AVX_256bit);
-    __ evmovdquq(xmm2, Address(source, start_offset, Address::times_1, 48), Assembler::AVX_256bit);
-
-    //permute the input data in such a manner that we have continuity of the source
-    __ vpermq(xmm3, xmm0, 148, Assembler::AVX_256bit);
-    __ vpermq(xmm4, xmm1, 148, Assembler::AVX_256bit);
-    __ vpermq(xmm5, xmm2, 148, Assembler::AVX_256bit);
-
-    //shuffle input and group 3 bytes of data and to it add 0 as the 4th byte.
-    //we can deal with 12 bytes at a time in a 128 bit register
-    __ vpshufb(xmm3, xmm3, xmm12, Assembler::AVX_256bit);
-    __ vpshufb(xmm4, xmm4, xmm12, Assembler::AVX_256bit);
-    __ vpshufb(xmm5, xmm5, xmm12, Assembler::AVX_256bit);
-
-    //convert byte to word. Each 128 bit register will have 6 bytes for processing
-    __ vpmovzxbw(xmm3, xmm3, Assembler::AVX_512bit);
-    __ vpmovzxbw(xmm4, xmm4, Assembler::AVX_512bit);
-    __ vpmovzxbw(xmm5, xmm5, Assembler::AVX_512bit);
-
-    // Extract bits in the following pattern 6, 4+2, 2+4, 6 to convert 3, 8 bit numbers to 4, 6 bit numbers
-    __ evpsrlvw(xmm0, xmm3, xmm13,  Assembler::AVX_512bit);
-    __ evpsrlvw(xmm1, xmm4, xmm13, Assembler::AVX_512bit);
-    __ evpsrlvw(xmm2, xmm5, xmm13, Assembler::AVX_512bit);
-
-    __ evpsllvw(xmm3, xmm3, xmm14, Assembler::AVX_512bit);
-    __ evpsllvw(xmm4, xmm4, xmm14, Assembler::AVX_512bit);
-    __ evpsllvw(xmm5, xmm5, xmm14, Assembler::AVX_512bit);
-
-    __ vpsrlq(xmm0, xmm0, 8, Assembler::AVX_512bit);
-    __ vpsrlq(xmm1, xmm1, 8, Assembler::AVX_512bit);
-    __ vpsrlq(xmm2, xmm2, 8, Assembler::AVX_512bit);
-
-    __ vpsllq(xmm3, xmm3, 8, Assembler::AVX_512bit);
-    __ vpsllq(xmm4, xmm4, 8, Assembler::AVX_512bit);
-    __ vpsllq(xmm5, xmm5, 8, Assembler::AVX_512bit);
-
-    __ vpandq(xmm3, xmm3, xmm15, Assembler::AVX_512bit);
-    __ vpandq(xmm4, xmm4, xmm15, Assembler::AVX_512bit);
-    __ vpandq(xmm5, xmm5, xmm15, Assembler::AVX_512bit);
-
-    // Get the final 4*6 bits base64 encoding
-    __ vporq(xmm3, xmm3, xmm0, Assembler::AVX_512bit);
-    __ vporq(xmm4, xmm4, xmm1, Assembler::AVX_512bit);
-    __ vporq(xmm5, xmm5, xmm2, Assembler::AVX_512bit);
-
-    // Shift
-    __ vpsrlq(xmm3, xmm3, 8, Assembler::AVX_512bit);
-    __ vpsrlq(xmm4, xmm4, 8, Assembler::AVX_512bit);
-    __ vpsrlq(xmm5, xmm5, 8, Assembler::AVX_512bit);
-
-    // look up 6 bits in the base64 character set to fetch the encoding
-    // we are converting word to dword as gather instructions need dword indices for looking up encoding
-    __ vextracti64x4(xmm6, xmm3, 0);
-    __ vpmovzxwd(xmm0, xmm6, Assembler::AVX_512bit);
-    __ vextracti64x4(xmm6, xmm3, 1);
-    __ vpmovzxwd(xmm1, xmm6, Assembler::AVX_512bit);
-
-    __ vextracti64x4(xmm6, xmm4, 0);
-    __ vpmovzxwd(xmm2, xmm6, Assembler::AVX_512bit);
-    __ vextracti64x4(xmm6, xmm4, 1);
-    __ vpmovzxwd(xmm3, xmm6, Assembler::AVX_512bit);
-
-    __ vextracti64x4(xmm4, xmm5, 0);
-    __ vpmovzxwd(xmm6, xmm4, Assembler::AVX_512bit);
-
-    __ vextracti64x4(xmm4, xmm5, 1);
-    __ vpmovzxwd(xmm7, xmm4, Assembler::AVX_512bit);
-
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm4, k2, Address(r11, xmm0, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm5, k2, Address(r11, xmm1, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm8, k2, Address(r11, xmm2, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm9, k2, Address(r11, xmm3, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm10, k2, Address(r11, xmm6, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm11, k2, Address(r11, xmm7, Address::times_4, 0), Assembler::AVX_512bit);
-
-    //Down convert dword to byte. Final output is 16*6 = 96 bytes long
-    __ evpmovdb(Address(dest, dp, Address::times_1, 0), xmm4, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 16), xmm5, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 32), xmm8, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 48), xmm9, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 64), xmm10, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 80), xmm11, Assembler::AVX_512bit);
-
-    __ addq(dest, 96);
-    __ addq(source, 72);
-    __ subq(length, 72);
-    __ jmp(L_process80);
-
-    // Vector Base64 implementation generating 32 bytes of encoded data
-    __ BIND(L_process32);
-    __ cmpl(length, 32);
-    __ jcc(Assembler::below, L_process3);
-    __ evmovdquq(xmm0, Address(source, start_offset), Assembler::AVX_256bit);
-    __ vpermq(xmm0, xmm0, 148, Assembler::AVX_256bit);
-    __ vpshufb(xmm6, xmm0, xmm12, Assembler::AVX_256bit);
-    __ vpmovzxbw(xmm6, xmm6, Assembler::AVX_512bit);
-    __ evpsrlvw(xmm2, xmm6, xmm13, Assembler::AVX_512bit);
-    __ evpsllvw(xmm3, xmm6, xmm14, Assembler::AVX_512bit);
-
-    __ vpsrlq(xmm2, xmm2, 8, Assembler::AVX_512bit);
-    __ vpsllq(xmm3, xmm3, 8, Assembler::AVX_512bit);
-    __ vpandq(xmm3, xmm3, xmm15, Assembler::AVX_512bit);
-    __ vporq(xmm1, xmm2, xmm3, Assembler::AVX_512bit);
-    __ vpsrlq(xmm1, xmm1, 8, Assembler::AVX_512bit);
-    __ vextracti64x4(xmm9, xmm1, 0);
-    __ vpmovzxwd(xmm6, xmm9, Assembler::AVX_512bit);
-    __ vextracti64x4(xmm9, xmm1, 1);
-    __ vpmovzxwd(xmm5, xmm9,  Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm8, k2, Address(r11, xmm6, Address::times_4, 0), Assembler::AVX_512bit);
-    __ kmovql(k2, k3);
-    __ evpgatherdd(xmm10, k2, Address(r11, xmm5, Address::times_4, 0), Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 0), xmm8, Assembler::AVX_512bit);
-    __ evpmovdb(Address(dest, dp, Address::times_1, 16), xmm10, Assembler::AVX_512bit);
-    __ subq(length, 24);
-    __ addq(dest, 32);
-    __ addq(source, 24);
-    __ jmp(L_process32);
-
-    // Scalar data processing takes 3 bytes at a time and produces 4 bytes of encoded data
-    /* This code corresponds to the scalar version of the following snippet in Base64.java
-    ** int bits = (src[sp0++] & 0xff) << 16 |(src[sp0++] & 0xff) << 8 |(src[sp0++] & 0xff);
-    ** dst[dp0++] = (byte)base64[(bits >> > 18) & 0x3f];
-    ** dst[dp0++] = (byte)base64[(bits >> > 12) & 0x3f];
-    ** dst[dp0++] = (byte)base64[(bits >> > 6) & 0x3f];
-    ** dst[dp0++] = (byte)base64[bits & 0x3f];*/
-    __ BIND(L_process3);
-    __ cmpl(length, 3);
-    __ jcc(Assembler::below, L_exit);
-    // Read 1 byte at a time
-    __ movzbl(rax, Address(source, start_offset));
-    __ shll(rax, 0x10);
-    __ movl(r15, rax);
-    __ movzbl(rax, Address(source, start_offset, Address::times_1, 1));
-    __ shll(rax, 0x8);
-    __ movzwl(rax, rax);
-    __ orl(r15, rax);
-    __ movzbl(rax, Address(source, start_offset, Address::times_1, 2));
-    __ orl(rax, r15);
-    // Save 3 bytes read in r15
-    __ movl(r15, rax);
-    __ shrl(rax, 0x12);
-    __ andl(rax, 0x3f);
-    // rax contains the index, r11 contains base64 lookup table
-    __ movb(rax, Address(r11, rax, Address::times_4));
-    // Write the encoded byte to destination
-    __ movb(Address(dest, dp, Address::times_1, 0), rax);
-    __ movl(rax, r15);
-    __ shrl(rax, 0xc);
-    __ andl(rax, 0x3f);
-    __ movb(rax, Address(r11, rax, Address::times_4));
-    __ movb(Address(dest, dp, Address::times_1, 1), rax);
-    __ movl(rax, r15);
-    __ shrl(rax, 0x6);
-    __ andl(rax, 0x3f);
-    __ movb(rax, Address(r11, rax, Address::times_4));
-    __ movb(Address(dest, dp, Address::times_1, 2), rax);
-    __ movl(rax, r15);
-    __ andl(rax, 0x3f);
-    __ movb(rax, Address(r11, rax, Address::times_4));
-    __ movb(Address(dest, dp, Address::times_1, 3), rax);
-    __ subl(length, 3);
-    __ addq(dest, 4);
-    __ addq(source, 3);
-    __ jmp(L_process3);
-    __ BIND(L_exit);
+    __ jmp(L_bottomLoop);
+
+    __ align(32);
+    __ BIND(L_forceLoop);
+    __ shll(byte1, 18);
+    __ shll(byte2, 12);
+    __ shll(byte3, 6);
+    __ orl(byte1, byte2);
+    __ orl(byte1, byte3);
+    __ orl(byte1, byte4);
+
+    __ addptr(source, 4);
+
+    __ movb(Address(dest, dp, Address::times_1, 2), byte1);
+    __ shrl(byte1, 8);
+    __ movb(Address(dest, dp, Address::times_1, 1), byte1);
+    __ shrl(byte1, 8);
+    __ movb(Address(dest, dp, Address::times_1, 0), byte1);
+
+    __ addptr(dest, 3);
+    __ decrementl(length, 1);
+    __ jcc(Assembler::zero, L_exit_no_vzero);
+
+    __ BIND(L_bottomLoop);
+    __ load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00));
+    __ load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01));
+    __ load_signed_byte(byte1, Address(decode_table, byte1));
+    __ load_signed_byte(byte2, Address(decode_table, byte2));
+    __ load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02));
+    __ load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03));
+    __ load_signed_byte(byte3, Address(decode_table, byte3));
+    __ load_signed_byte(byte4, Address(decode_table, byte4));
+
+    __ mov(rax, byte1);
+    __ orl(rax, byte2);
+    __ orl(rax, byte3);
+    __ orl(rax, byte4);
+    __ jcc(Assembler::positive, L_forceLoop);
+
+    __ BIND(L_exit_no_vzero);
+    __ pop(rax);             // Get original dest value
+    __ subptr(dest, rax);      // Number of bytes converted
+    __ movptr(rax, dest);
+    __ pop(rbx);
     __ pop(r15);
     __ pop(r14);
     __ pop(r13);
     __ pop(r12);
     __ leave();
     __ ret(0);
+
     return start;
   }
 
+
   /**
    *  Arguments:
    *
@@ -7144,15 +7860,30 @@ address generate_avx_ghash_processBlocks() {
       }
     }
 
+
     if (UseBASE64Intrinsics) {
-      StubRoutines::x86::_and_mask = base64_and_mask_addr();
-      StubRoutines::x86::_bswap_mask = base64_bswap_mask_addr();
-      StubRoutines::x86::_base64_charset = base64_charset_addr();
-      StubRoutines::x86::_url_charset = base64url_charset_addr();
-      StubRoutines::x86::_gather_mask = base64_gather_mask_addr();
-      StubRoutines::x86::_left_shift_mask = base64_left_shift_mask_addr();
-      StubRoutines::x86::_right_shift_mask = base64_right_shift_mask_addr();
+      if(VM_Version::supports_avx2() &&
+         VM_Version::supports_avx512bw() &&
+         VM_Version::supports_avx512vl()) {
+        StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr();
+        StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr();
+        StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr();
+      }
+      StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr();
+      if (VM_Version::supports_avx512_vbmi()) {
+        StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr();
+        StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr();
+        StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr();
+        StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr();
+        StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr();
+        StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr();
+        StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr();
+        StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr();
+        StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr();
+      }
+      StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr();
       StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
+      StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
     }
 
     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
@@ -7185,7 +7916,10 @@ address generate_avx_ghash_processBlocks() {
     // Get svml stub routine addresses
     void *libsvml = NULL;
     char ebuf[1024];
-    libsvml = os::dll_load(JNI_LIB_PREFIX "svml" JNI_LIB_SUFFIX, ebuf, sizeof ebuf);
+    char dll_name[JVM_MAXPATHLEN];
+    if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "svml")) {
+      libsvml = os::dll_load(dll_name, ebuf, sizeof ebuf);
+    }
     if (libsvml != NULL) {
       // SVML method naming convention
       //   All the methods are named as __svml_op<T><N>_ha_<VV>
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index 062c5032c33..a306f4a0637 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -65,14 +65,21 @@ address StubRoutines::x86::_k256_W_adr = NULL;
 address StubRoutines::x86::_k512_W_addr = NULL;
 address StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = NULL;
 // Base64 masks
-address StubRoutines::x86::_bswap_mask = NULL;
-address StubRoutines::x86::_base64_charset = NULL;
-address StubRoutines::x86::_gather_mask = NULL;
-address StubRoutines::x86::_right_shift_mask = NULL;
-address StubRoutines::x86::_left_shift_mask = NULL;
-address StubRoutines::x86::_and_mask = NULL;
-address StubRoutines::x86::_url_charset = NULL;
+address StubRoutines::x86::_encoding_table_base64 = NULL;
+address StubRoutines::x86::_shuffle_base64 = NULL;
+address StubRoutines::x86::_avx2_shuffle_base64 = NULL;
+address StubRoutines::x86::_avx2_input_mask_base64 = NULL;
+address StubRoutines::x86::_avx2_lut_base64 = NULL;
 address StubRoutines::x86::_counter_mask_addr = NULL;
+address StubRoutines::x86::_lookup_lo_base64 = NULL;
+address StubRoutines::x86::_lookup_hi_base64 = NULL;
+address StubRoutines::x86::_lookup_lo_base64url = NULL;
+address StubRoutines::x86::_lookup_hi_base64url = NULL;
+address StubRoutines::x86::_pack_vec_base64 = NULL;
+address StubRoutines::x86::_join_0_1_base64 = NULL;
+address StubRoutines::x86::_join_1_2_base64 = NULL;
+address StubRoutines::x86::_join_2_3_base64 = NULL;
+address StubRoutines::x86::_decoding_table_base64 = NULL;
 #endif
 address StubRoutines::x86::_pshuffle_byte_flip_mask_addr = NULL;
 
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 339a148b223..3f682d7d0b2 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -184,13 +184,20 @@ class x86 {
   static address _pshuffle_byte_flip_mask_addr_sha512;
   static address _counter_mask_addr;
   // Masks for base64
-  static address _base64_charset;
-  static address _bswap_mask;
-  static address _gather_mask;
-  static address _right_shift_mask;
-  static address _left_shift_mask;
-  static address _and_mask;
-  static address _url_charset;
+  static address _encoding_table_base64;
+  static address _shuffle_base64;
+  static address _avx2_shuffle_base64;
+  static address _avx2_input_mask_base64;
+  static address _avx2_lut_base64;
+  static address _lookup_lo_base64;
+  static address _lookup_hi_base64;
+  static address _lookup_lo_base64url;
+  static address _lookup_hi_base64url;
+  static address _pack_vec_base64;
+  static address _join_0_1_base64;
+  static address _join_1_2_base64;
+  static address _join_2_3_base64;
+  static address _decoding_table_base64;
 #endif
   // byte flip mask for sha256
   static address _pshuffle_byte_flip_mask_addr;
@@ -328,14 +335,21 @@ class x86 {
   static address k256_W_addr()    { return _k256_W_adr; }
   static address k512_W_addr()    { return _k512_W_addr; }
   static address pshuffle_byte_flip_mask_addr_sha512() { return _pshuffle_byte_flip_mask_addr_sha512; }
-  static address base64_charset_addr() { return _base64_charset; }
-  static address base64url_charset_addr() { return _url_charset; }
-  static address base64_bswap_mask_addr() { return _bswap_mask; }
-  static address base64_gather_mask_addr() { return _gather_mask; }
-  static address base64_right_shift_mask_addr() { return _right_shift_mask; }
-  static address base64_left_shift_mask_addr() { return _left_shift_mask; }
-  static address base64_and_mask_addr() { return _and_mask; }
+  static address base64_encoding_table_addr() { return _encoding_table_base64; }
+  static address base64_shuffle_addr() { return _shuffle_base64; }
+  static address base64_avx2_shuffle_addr() { return _avx2_shuffle_base64; }
+  static address base64_avx2_input_mask_addr() { return _avx2_input_mask_base64; }
+  static address base64_avx2_lut_addr() { return _avx2_lut_base64; }
   static address counter_mask_addr() { return _counter_mask_addr; }
+  static address base64_vbmi_lookup_lo_addr() { return _lookup_lo_base64; }
+  static address base64_vbmi_lookup_hi_addr() { return _lookup_hi_base64; }
+  static address base64_vbmi_lookup_lo_url_addr() { return _lookup_lo_base64url; }
+  static address base64_vbmi_lookup_hi_url_addr() { return _lookup_hi_base64url; }
+  static address base64_vbmi_pack_vec_addr() { return _pack_vec_base64; }
+  static address base64_vbmi_join_0_1_addr() { return _join_0_1_base64; }
+  static address base64_vbmi_join_1_2_addr() { return _join_1_2_base64; }
+  static address base64_vbmi_join_2_3_addr() { return _join_2_3_base64; }
+  static address base64_decoding_table_addr() { return _decoding_table_base64; }
 #endif
   static address pshuffle_byte_flip_mask_addr() { return _pshuffle_byte_flip_mask_addr; }
   static void generate_CRC32C_table(bool is_pclmulqdq_supported);
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 09c75c67ddf..d0073d27893 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -3181,9 +3181,16 @@ void TemplateTable::withfield() {
 
   resolve_cache_and_index(f2_byte, cache, index, sizeof(u2));
 
-  call_VM(rbx, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), cache);
+  Register cpentry = rbx;
+
+  ByteSize cp_base_offset = ConstantPoolCache::base_offset();
+
+  __ lea(cpentry, Address(cache, index, Address::times_ptr,
+                         in_bytes(cp_base_offset)));
+  __ lea(rax, at_tos());
+  __ call_VM(rbx, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), cpentry, rax);
   // new value type is returned in rbx
-  // stack adjustement is returned in rax
+  // stack adjustment is returned in rax
   __ verify_oop(rbx);
   __ addptr(rsp, rax);
   __ movptr(rax, rbx);
diff --git a/src/hotspot/cpu/x86/universalUpcallHandler_x86_64.cpp b/src/hotspot/cpu/x86/universalUpcallHandler_x86_64.cpp
index bbfaa87bcb8..c54b907f9b5 100644
--- a/src/hotspot/cpu/x86/universalUpcallHandler_x86_64.cpp
+++ b/src/hotspot/cpu/x86/universalUpcallHandler_x86_64.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -316,47 +316,6 @@ static void print_arg_moves(const GrowableArray<ArgMove>& arg_moves, Method* ent
 }
 #endif
 
-void save_java_frame_anchor(MacroAssembler* _masm, ByteSize store_offset, Register thread) {
-  __ block_comment("{ save_java_frame_anchor ");
-  // upcall->jfa._last_Java_fp = _thread->_anchor._last_Java_fp;
-  __ movptr(rscratch1, Address(thread, JavaThread::last_Java_fp_offset()));
-  __ movptr(Address(rsp, store_offset + JavaFrameAnchor::last_Java_fp_offset()), rscratch1);
-
-  // upcall->jfa._last_Java_pc = _thread->_anchor._last_Java_pc;
-  __ movptr(rscratch1, Address(thread, JavaThread::last_Java_pc_offset()));
-  __ movptr(Address(rsp, store_offset + JavaFrameAnchor::last_Java_pc_offset()), rscratch1);
-
-  // upcall->jfa._last_Java_sp = _thread->_anchor._last_Java_sp;
-  __ movptr(rscratch1, Address(thread, JavaThread::last_Java_sp_offset()));
-  __ movptr(Address(rsp, store_offset + JavaFrameAnchor::last_Java_sp_offset()), rscratch1);
-  __ block_comment("} save_java_frame_anchor ");
-}
-
-void restore_java_frame_anchor(MacroAssembler* _masm, ByteSize load_offset, Register thread) {
-  __ block_comment("{ restore_java_frame_anchor ");
-  // thread->_last_Java_sp = NULL
-  __ movptr(Address(thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
-
-  // ThreadStateTransition::transition_from_java(_thread, _thread_in_vm);
-  // __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
-  __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
-
-  //_thread->frame_anchor()->copy(&_anchor);
-//  _thread->_last_Java_fp = upcall->_last_Java_fp;
-//  _thread->_last_Java_pc = upcall->_last_Java_pc;
-//  _thread->_last_Java_sp = upcall->_last_Java_sp;
-
-  __ movptr(rscratch1, Address(rsp, load_offset + JavaFrameAnchor::last_Java_fp_offset()));
-  __ movptr(Address(thread, JavaThread::last_Java_fp_offset()), rscratch1);
-
-  __ movptr(rscratch1, Address(rsp, load_offset + JavaFrameAnchor::last_Java_pc_offset()));
-  __ movptr(Address(thread, JavaThread::last_Java_pc_offset()), rscratch1);
-
-  __ movptr(rscratch1, Address(rsp, load_offset + JavaFrameAnchor::last_Java_sp_offset()));
-  __ movptr(Address(thread, JavaThread::last_Java_sp_offset()), rscratch1);
-  __ block_comment("} restore_java_frame_anchor ");
-}
-
 static void save_native_arguments(MacroAssembler* _masm, const CallRegs& conv, int arg_save_area_offset) {
   __ block_comment("{ save_native_args ");
   int store_offset = arg_save_area_offset;
@@ -442,6 +401,60 @@ static int compute_arg_save_area_size(const CallRegs& conv) {
   return result_size;
 }
 
+static int compute_res_save_area_size(const CallRegs& conv) {
+  int result_size = 0;
+  for (int i = 0; i < conv._rets_length; i++) {
+    VMReg reg = conv._ret_regs[i];
+    if (reg->is_Register()) {
+      result_size += 8;
+    } else if (reg->is_XMMRegister()) {
+      // Java API doesn't support vector args
+      result_size += 16;
+    } else {
+      ShouldNotReachHere(); // unhandled type
+    }
+  }
+  return result_size;
+}
+
+static void save_java_result(MacroAssembler* _masm, const CallRegs& conv, int res_save_area_offset) {
+  int offset = res_save_area_offset;
+  __ block_comment("{ save java result ");
+  for (int i = 0; i < conv._rets_length; i++) {
+    VMReg reg = conv._ret_regs[i];
+    if (reg->is_Register()) {
+      __ movptr(Address(rsp, offset), reg->as_Register());
+      offset += 8;
+    } else if (reg->is_XMMRegister()) {
+      // Java API doesn't support vector args
+      __ movdqu(Address(rsp, offset), reg->as_XMMRegister());
+      offset += 16;
+    } else {
+      ShouldNotReachHere(); // unhandled type
+    }
+  }
+  __ block_comment("} save java result ");
+}
+
+static void restore_java_result(MacroAssembler* _masm, const CallRegs& conv, int res_save_area_offset) {
+  int offset = res_save_area_offset;
+  __ block_comment("{ restore java result ");
+  for (int i = 0; i < conv._rets_length; i++) {
+    VMReg reg = conv._ret_regs[i];
+    if (reg->is_Register()) {
+      __ movptr(reg->as_Register(), Address(rsp, offset));
+      offset += 8;
+    } else if (reg->is_XMMRegister()) {
+      // Java API doesn't support vector args
+      __ movdqu(reg->as_XMMRegister(), Address(rsp, offset));
+      offset += 16;
+    } else {
+      ShouldNotReachHere(); // unhandled type
+    }
+  }
+  __ block_comment("} restore java result ");
+}
+
 constexpr int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
 
 static void preserve_callee_saved_registers(MacroAssembler* _masm, const ABIDescriptor& abi, int reg_save_area_offset) {
@@ -574,18 +587,16 @@ static void shuffle_arguments(MacroAssembler* _masm, const GrowableArray<ArgMove
   }
 }
 
-struct AuxiliarySaves {
-  JavaFrameAnchor jfa;
-  uintptr_t thread;
-  bool should_detach;
-};
-
+// Register is a class, but it would be assigned numerical value.
+// "0" is assigned for rax and for xmm0. Thus we need to ignore -Wnonnull.
+PRAGMA_DIAG_PUSH
+PRAGMA_NONNULL_IGNORED
 address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiver, Method* entry, jobject jabi, jobject jconv) {
   ResourceMark rm;
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs conv = ForeignGlobals::parse_call_regs(jconv);
   assert(conv._rets_length <= 1, "no multi reg returns");
-  CodeBuffer buffer("upcall_stub_linkToNative", /* code_size = */ 1024, /* locs_size = */ 1024);
+  CodeBuffer buffer("upcall_stub_linkToNative", /* code_size = */ 2048, /* locs_size = */ 1024);
 
   int register_size = sizeof(uintptr_t);
   int buffer_alignment = xmm_reg_size;
@@ -604,19 +615,17 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
 
   int reg_save_area_size = compute_reg_save_area_size(abi);
   int arg_save_area_size = compute_arg_save_area_size(conv);
+  int res_save_area_size = compute_res_save_area_size(conv);
   // To spill receiver during deopt
   int deopt_spill_size = 1 * BytesPerWord;
 
   int shuffle_area_offset    = 0;
   int deopt_spill_offset     = shuffle_area_offset    + out_arg_area;
-  int arg_save_area_offset   = deopt_spill_offset     + deopt_spill_size;
+  int res_save_area_offset   = deopt_spill_offset     + deopt_spill_size;
+  int arg_save_area_offset   = res_save_area_offset   + res_save_area_size;
   int reg_save_area_offset   = arg_save_area_offset   + arg_save_area_size;
-  int auxiliary_saves_offset = reg_save_area_offset   + reg_save_area_size;
-  int frame_bottom_offset    = auxiliary_saves_offset + sizeof(AuxiliarySaves);
-
-  ByteSize jfa_offset           = in_ByteSize(auxiliary_saves_offset) + byte_offset_of(AuxiliarySaves, jfa);
-  ByteSize thread_offset        = in_ByteSize(auxiliary_saves_offset) + byte_offset_of(AuxiliarySaves, thread);
-  ByteSize should_detach_offset = in_ByteSize(auxiliary_saves_offset) + byte_offset_of(AuxiliarySaves, should_detach);
+  int frame_data_offset      = reg_save_area_offset   + reg_save_area_size;
+  int frame_bottom_offset    = frame_data_offset      + sizeof(OptimizedEntryBlob::FrameData);
 
   int frame_size = frame_bottom_offset;
   frame_size = align_up(frame_size, StackAlignmentInBytes);
@@ -627,8 +636,8 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
   // FP-> |                     |
   //      |---------------------| = frame_bottom_offset = frame_size
   //      |                     |
-  //      | AuxiliarySaves      |
-  //      |---------------------| = auxiliary_saves_offset
+  //      | FrameData           |
+  //      |---------------------| = frame_data_offset
   //      |                     |
   //      | reg_save_area       |
   //      |---------------------| = reg_save_are_offset
@@ -636,6 +645,9 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
   //      | arg_save_area       |
   //      |---------------------| = arg_save_are_offset
   //      |                     |
+  //      | res_save_area       |
+  //      |---------------------| = res_save_are_offset
+  //      |                     |
   //      | deopt_spill         |
   //      |---------------------| = deopt_spill_offset
   //      |                     |
@@ -646,7 +658,6 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
   //////////////////////////////////////////////////////////////////////////////
 
   MacroAssembler* _masm = new MacroAssembler(&buffer);
-  Label call_return;
   address start = __ pc();
   __ enter(); // set up frame
   if ((abi._stack_alignment_bytes % 16) != 0) {
@@ -662,53 +673,14 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
 
   preserve_callee_saved_registers(_masm, abi, reg_save_area_offset);
 
-  __ block_comment("{ get_thread");
+  __ block_comment("{ on_entry");
   __ vzeroupper();
-  __ lea(c_rarg0, Address(rsp, should_detach_offset));
+  __ lea(c_rarg0, Address(rsp, frame_data_offset));
   // stack already aligned
-  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ProgrammableUpcallHandler::maybe_attach_and_get_thread)));
+  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ProgrammableUpcallHandler::on_entry)));
   __ movptr(r15_thread, rax);
   __ reinit_heapbase();
-  __ movptr(Address(rsp, thread_offset), r15_thread);
-  __ block_comment("} get_thread");
-
-  // TODO:
-  // We expect not to be coming from JNI code, but we might be.
-  // We should figure out what our stance is on supporting that and then maybe add
-  // some more handling here for:
-  //   - handle blocks
-  //   - check for active exceptions (and emit an error)
-
-  __ block_comment("{ safepoint poll");
-  __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
-
-  if (os::is_MP()) {
-    __ membar(Assembler::Membar_mask_bits(
-                Assembler::LoadLoad  | Assembler::StoreLoad |
-                Assembler::LoadStore | Assembler::StoreStore));
-   }
-
-  // check for safepoint operation in progress and/or pending suspend requests
-  Label L_after_safepoint_poll;
-  Label L_safepoint_poll_slow_path;
-
-  __ safepoint_poll(L_safepoint_poll_slow_path, r15_thread, false /* at_return */, false /* in_nmethod */);
-
-  __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
-  __ jcc(Assembler::notEqual, L_safepoint_poll_slow_path);
-
-  __ bind(L_after_safepoint_poll);
-  __ block_comment("} safepoint poll");
-  // change thread state
-  __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
-
-  __ block_comment("{ reguard stack check");
-  Label L_reguard;
-  Label L_after_reguard;
-  __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled);
-  __ jcc(Assembler::equal, L_reguard);
-  __ bind(L_after_reguard);
-  __ block_comment("} reguard stack check");
+  __ block_comment("} on_entry");
 
   __ block_comment("{ argument shuffle");
   // TODO merge these somehow
@@ -724,13 +696,24 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
 
   __ mov_metadata(rbx, entry);
   __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
+
+  __ call(Address(rbx, Method::from_compiled_offset()));
+
+  save_java_result(_masm, conv, res_save_area_offset);
+
+  __ block_comment("{ on_exit");
+  __ vzeroupper();
+  __ lea(c_rarg0, Address(rsp, frame_data_offset));
+  // stack already aligned
+  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ProgrammableUpcallHandler::on_exit)));
   __ reinit_heapbase();
+  __ block_comment("} on_exit");
 
-  save_java_frame_anchor(_masm, jfa_offset, r15_thread);
-  __ reset_last_Java_frame(r15_thread, true);
+  restore_callee_saved_registers(_masm, abi, reg_save_area_offset);
 
-  __ call(Address(rbx, Method::from_compiled_offset()));
+  restore_java_result(_masm, conv, res_save_area_offset);
 
+  // return value shuffle
 #ifdef ASSERT
   if (conv._rets_length == 1) { // 0 or 1
     VMReg j_expected_result_reg;
@@ -757,55 +740,11 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
   }
 #endif
 
-  __ bind(call_return);
-
-  // also sets last Java frame
-  __ movptr(r15_thread, Address(rsp, thread_offset));
-  // TODO corrupted thread pointer causes havoc. Can we verify it here?
-  restore_java_frame_anchor(_masm, jfa_offset, r15_thread); // also transitions to native state
-
-  __ block_comment("{ maybe_detach_thread");
-  Label L_after_detach;
-  __ cmpb(Address(rsp, should_detach_offset), 0);
-  __ jcc(Assembler::equal, L_after_detach);
-  __ vzeroupper();
-  __ mov(c_rarg0, r15_thread);
-  // stack already aligned
-  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ProgrammableUpcallHandler::detach_thread)));
-  __ reinit_heapbase();
-  __ bind(L_after_detach);
-  __ block_comment("} maybe_detach_thread");
-
-  restore_callee_saved_registers(_masm, abi, reg_save_area_offset);
-
   __ leave();
   __ ret(0);
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ block_comment("{ L_safepoint_poll_slow_path");
-  __ bind(L_safepoint_poll_slow_path);
-  __ vzeroupper();
-  __ mov(c_rarg0, r15_thread);
-  // stack already aligned
-  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
-  __ reinit_heapbase();
-  __ jmp(L_after_safepoint_poll);
-  __ block_comment("} L_safepoint_poll_slow_path");
-
-  //////////////////////////////////////////////////////////////////////////////
-
-  __ block_comment("{ L_reguard");
-  __ bind(L_reguard);
-  __ vzeroupper();
-  // stack already aligned
-  __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
-  __ reinit_heapbase();
-  __ jmp(L_after_reguard);
-  __ block_comment("} L_reguard");
-
-  //////////////////////////////////////////////////////////////////////////////
-
   __ block_comment("{ exception handler");
 
   intptr_t exception_handler_offset = __ pc() - start;
@@ -835,7 +774,7 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
   const char* name = "optimized_upcall_stub";
 #endif // PRODUCT
 
-  OptimizedEntryBlob* blob = OptimizedEntryBlob::create(name, &buffer, exception_handler_offset, receiver, jfa_offset);
+  OptimizedEntryBlob* blob = OptimizedEntryBlob::create(name, &buffer, exception_handler_offset, receiver, in_ByteSize(frame_data_offset));
 
   if (TraceOptimizedUpcallStubs) {
     blob->print_on(tty);
@@ -844,6 +783,7 @@ address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject receiv
 
   return blob->code_begin();
 }
+PRAGMA_DIAG_POP
 
 bool ProgrammableUpcallHandler::supports_optimized_upcalls() {
   return true;
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 9d97f5a5d65..0c65ca80c0b 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -769,6 +769,15 @@ void VM_Version::get_processor_features() {
       _features &= ~CPU_VZEROUPPER;
       _features &= ~CPU_AVX512BW;
       _features &= ~CPU_AVX512VL;
+      _features &= ~CPU_AVX512DQ;
+      _features &= ~CPU_AVX512_VNNI;
+      _features &= ~CPU_AVX512_VAES;
+      _features &= ~CPU_AVX512_VPOPCNTDQ;
+      _features &= ~CPU_AVX512_VPCLMULQDQ;
+      _features &= ~CPU_AVX512_VBMI;
+      _features &= ~CPU_AVX512_VBMI2;
+      _features &= ~CPU_CLWB;
+      _features &= ~CPU_FLUSHOPT;
     }
   }
 
@@ -1012,10 +1021,6 @@ void VM_Version::get_processor_features() {
   }
 
   if (!supports_rtm() && UseRTMLocking) {
-    // Can't continue because UseRTMLocking affects UseBiasedLocking flag
-    // setting during arguments processing. See use_biased_locking().
-    // VM_Version_init() is executed after UseBiasedLocking is used
-    // in Thread::allocate().
     vm_exit_during_initialization("RTM instructions are not available on this CPU");
   }
 
@@ -1023,8 +1028,6 @@ void VM_Version::get_processor_features() {
   if (UseRTMLocking) {
     if (!CompilerConfig::is_c2_enabled()) {
       // Only C2 does RTM locking optimization.
-      // Can't continue because UseRTMLocking affects UseBiasedLocking flag
-      // setting during arguments processing. See use_biased_locking().
       vm_exit_during_initialization("RTM locking optimization is not supported in this VM");
     }
     if (is_intel_family_core()) {
@@ -1062,8 +1065,6 @@ void VM_Version::get_processor_features() {
 #else
   if (UseRTMLocking) {
     // Only C2 does RTM locking optimization.
-    // Can't continue because UseRTMLocking affects UseBiasedLocking flag
-    // setting during arguments processing. See use_biased_locking().
     vm_exit_during_initialization("RTM locking optimization is not supported in this VM");
   }
 #endif
@@ -1736,27 +1737,6 @@ void VM_Version::print_platform_virtualization_info(outputStream* st) {
   }
 }
 
-bool VM_Version::use_biased_locking() {
-#if INCLUDE_RTM_OPT
-  // RTM locking is most useful when there is high lock contention and
-  // low data contention.  With high lock contention the lock is usually
-  // inflated and biased locking is not suitable for that case.
-  // RTM locking code requires that biased locking is off.
-  // Note: we can't switch off UseBiasedLocking in get_processor_features()
-  // because it is used by Thread::allocate() which is called before
-  // VM_Version::initialize().
-  if (UseRTMLocking && UseBiasedLocking) {
-    if (FLAG_IS_DEFAULT(UseBiasedLocking)) {
-      FLAG_SET_DEFAULT(UseBiasedLocking, false);
-    } else {
-      warning("Biased locking is not supported with RTM locking; ignoring UseBiasedLocking flag." );
-      UseBiasedLocking = false;
-    }
-  }
-#endif
-  return UseBiasedLocking;
-}
-
 bool VM_Version::compute_has_intel_jcc_erratum() {
   if (!is_intel_family_core()) {
     // Only Intel CPUs are affected.
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index 88334ec8c6f..e6e94ca9a99 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -747,9 +747,6 @@ enum Extended_Family {
   // Override Abstract_VM_Version implementation
   static void print_platform_virtualization_info(outputStream*);
 
-  // Override Abstract_VM_Version implementation
-  static bool use_biased_locking();
-
   // Asserts
   static void assert_is_initialized() {
     assert(_cpuid_info.std_cpuid1_eax.bits.family != 0, "VM_Version not initialized");
@@ -778,7 +775,7 @@ enum Extended_Family {
   static bool is_intel()          { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x756e6547; } // 'uneG'
   static bool is_zx()             { assert_is_initialized(); return (_cpuid_info.std_vendor_name_0 == 0x746e6543) || (_cpuid_info.std_vendor_name_0 == 0x68532020); } // 'tneC'||'hS  '
   static bool is_atom_family()    { return ((cpu_family() == 0x06) && ((extended_cpu_model() == 0x36) || (extended_cpu_model() == 0x37) || (extended_cpu_model() == 0x4D))); } //Silvermont and Centerton
-  static bool is_knights_family() { return ((cpu_family() == 0x06) && ((extended_cpu_model() == 0x57) || (extended_cpu_model() == 0x85))); } // Xeon Phi 3200/5200/7200 and Future Xeon Phi
+  static bool is_knights_family() { return UseKNLSetting || ((cpu_family() == 0x06) && ((extended_cpu_model() == 0x57) || (extended_cpu_model() == 0x85))); } // Xeon Phi 3200/5200/7200 and Future Xeon Phi
 
   static bool supports_processor_topology() {
     return (_cpuid_info.std_max_function >= 0xB) &&
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 2109037e1bb..e449b239f67 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1216,39 +1216,6 @@ class HandlerImpl {
 #endif
 };
 
-
-inline uint vector_length(const Node* n) {
-  const TypeVect* vt = n->bottom_type()->is_vect();
-  return vt->length();
-}
-
-inline uint vector_length(const MachNode* use, MachOper* opnd) {
-  uint def_idx = use->operand_index(opnd);
-  Node* def = use->in(def_idx);
-  return def->bottom_type()->is_vect()->length();
-}
-
-inline uint vector_length_in_bytes(const Node* n) {
-  const TypeVect* vt = n->bottom_type()->is_vect();
-  return vt->length_in_bytes();
-}
-
-inline uint vector_length_in_bytes(const MachNode* use, MachOper* opnd) {
-  uint def_idx = use->operand_index(opnd);
-  Node* def = use->in(def_idx);
-  return def->bottom_type()->is_vect()->length_in_bytes();
-}
-
-inline BasicType vector_element_basic_type(const Node *n) {
-  return n->bottom_type()->is_vect()->element_basic_type();
-}
-
-inline BasicType vector_element_basic_type(const MachNode *use, MachOper* opnd) {
-  uint def_idx = use->operand_index(opnd);
-  Node* def = use->in(def_idx);
-  return def->bottom_type()->is_vect()->element_basic_type();
-}
-
 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
   switch(bytes) {
     case  4: // fall-through
@@ -1265,7 +1232,7 @@ inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
 }
 
 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
-  return vector_length_encoding(vector_length_in_bytes(n));
+  return vector_length_encoding(Matcher::vector_length_in_bytes(n));
 }
 
 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
@@ -1835,6 +1802,11 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
         return false;
       }
       break;
+    case Op_VectorMaskCmp:
+      if (vlen < 2 || size_in_bits < 32) {
+        return false;
+      }
+      break;
   }
   return true;  // Per default match rules are supported.
 }
@@ -1868,10 +1840,18 @@ MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd,
   return NULL;
 }
 
-bool Matcher::is_generic_reg2reg_move(MachNode* m) {
+bool Matcher::is_reg2reg_move(MachNode* m) {
   switch (m->rule()) {
     case MoveVec2Leg_rule:
     case MoveLeg2Vec_rule:
+    case MoveF2VL_rule:
+    case MoveF2LEG_rule:
+    case MoveVL2F_rule:
+    case MoveLEG2F_rule:
+    case MoveD2VL_rule:
+    case MoveD2LEG_rule:
+    case MoveVL2D_rule:
+    case MoveLEG2D_rule:
       return true;
     default:
       return false;
@@ -1898,18 +1878,6 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return new TypeVectMask(TypeInt::BOOL, length);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  int float_pressure_threshold = default_pressure_threshold;
-#ifdef _LP64
-  if (UseAVX > 2) {
-    // Increase pressure threshold on machines with AVX3 which have
-    // 2x more XMM registers.
-    float_pressure_threshold = default_pressure_threshold * 2;
-  }
-#endif
-  return float_pressure_threshold;
-}
-
 // Max vector size in bytes. 0 if not supported.
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   assert(is_java_primitive(bt), "only primitive type vectors");
@@ -3345,7 +3313,7 @@ instruct sqrtD_reg(regD dst) %{
 // ---------------------------------------- VectorReinterpret ------------------------------------
 
 instruct reinterpret(vec dst) %{
-  predicate(vector_length_in_bytes(n) == vector_length_in_bytes(n->in(1))); // dst == src
+  predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
   match(Set dst (VectorReinterpret dst));
   ins_cost(125);
   format %{ "vector_reinterpret $dst\t!" %}
@@ -3357,16 +3325,16 @@ instruct reinterpret(vec dst) %{
 
 instruct reinterpret_expand(vec dst, vec src, rRegP scratch) %{
   predicate(UseAVX == 0 &&
-            (vector_length_in_bytes(n->in(1)) < vector_length_in_bytes(n))); // src < dst
+            (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
   match(Set dst (VectorReinterpret src));
   ins_cost(125);
   effect(TEMP dst, TEMP scratch);
   format %{ "vector_reinterpret_expand $dst,$src\t! using $scratch as TEMP" %}
   ins_encode %{
-    assert(vector_length_in_bytes(this)       <= 16, "required");
-    assert(vector_length_in_bytes(this, $src) <=  8, "required");
+    assert(Matcher::vector_length_in_bytes(this)       <= 16, "required");
+    assert(Matcher::vector_length_in_bytes(this, $src) <=  8, "required");
 
-    int src_vlen_in_bytes = vector_length_in_bytes(this, $src);
+    int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
     if (src_vlen_in_bytes == 4) {
       __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), $scratch$$Register);
     } else {
@@ -3380,8 +3348,8 @@ instruct reinterpret_expand(vec dst, vec src, rRegP scratch) %{
 
 instruct vreinterpret_expand4(legVec dst, vec src, rRegP scratch) %{
   predicate(UseAVX > 0 &&
-            (vector_length_in_bytes(n->in(1)) == 4) && // src
-            (vector_length_in_bytes(n->in(1)) < vector_length_in_bytes(n))); // src < dst
+            (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
+            (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
   match(Set dst (VectorReinterpret src));
   ins_cost(125);
   effect(TEMP scratch);
@@ -3395,13 +3363,13 @@ instruct vreinterpret_expand4(legVec dst, vec src, rRegP scratch) %{
 
 instruct vreinterpret_expand(legVec dst, vec src) %{
   predicate(UseAVX > 0 &&
-            (vector_length_in_bytes(n->in(1)) > 4) && // src
-            (vector_length_in_bytes(n->in(1)) < vector_length_in_bytes(n))); // src < dst
+            (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
+            (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
   match(Set dst (VectorReinterpret src));
   ins_cost(125);
   format %{ "vector_reinterpret_expand $dst,$src\t!" %}
   ins_encode %{
-    switch (vector_length_in_bytes(this, $src)) {
+    switch (Matcher::vector_length_in_bytes(this, $src)) {
       case  8: __ movq   ($dst$$XMMRegister, $src$$XMMRegister); break;
       case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
       case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
@@ -3412,12 +3380,12 @@ instruct vreinterpret_expand(legVec dst, vec src) %{
 %}
 
 instruct reinterpret_shrink(vec dst, legVec src) %{
-  predicate(vector_length_in_bytes(n->in(1)) > vector_length_in_bytes(n)); // src > dst
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
   match(Set dst (VectorReinterpret src));
   ins_cost(125);
   format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
   ins_encode %{
-    switch (vector_length_in_bytes(this)) {
+    switch (Matcher::vector_length_in_bytes(this)) {
       case  4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
       case  8: __ movq   ($dst$$XMMRegister, $src$$XMMRegister); break;
       case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
@@ -3466,7 +3434,7 @@ instruct roundD_imm(legRegD dst, immD con, immU8 rmode, rRegI scratch_reg) %{
 %}
 
 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
-  predicate(vector_length(n) < 8);
+  predicate(Matcher::vector_length(n) < 8);
   match(Set dst (RoundDoubleModeV src rmode));
   format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
   ins_encode %{
@@ -3478,7 +3446,7 @@ instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
 %}
 
 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (RoundDoubleModeV src rmode));
   format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
   ins_encode %{
@@ -3489,7 +3457,7 @@ instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
 %}
 
 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
-  predicate(vector_length(n) < 8);
+  predicate(Matcher::vector_length(n) < 8);
   match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
   format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
   ins_encode %{
@@ -3501,7 +3469,7 @@ instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
 %}
 
 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
   format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
   ins_encode %{
@@ -3579,7 +3547,7 @@ instruct loadV(vec dst, memory mem) %{
   ins_cost(125);
   format %{ "load_vector $dst,$mem" %}
   ins_encode %{
-    switch (vector_length_in_bytes(this)) {
+    switch (Matcher::vector_length_in_bytes(this)) {
       case  4: __ movdl    ($dst$$XMMRegister, $mem$$Address); break;
       case  8: __ movq     ($dst$$XMMRegister, $mem$$Address); break;
       case 16: __ movdqu   ($dst$$XMMRegister, $mem$$Address); break;
@@ -3597,7 +3565,7 @@ instruct storeV(memory mem, vec src) %{
   ins_cost(145);
   format %{ "store_vector $mem,$src\n\t" %}
   ins_encode %{
-    switch (vector_length_in_bytes(this, $src)) {
+    switch (Matcher::vector_length_in_bytes(this, $src)) {
       case  4: __ movdl    ($mem$$Address, $src$$XMMRegister); break;
       case  8: __ movq     ($mem$$Address, $src$$XMMRegister); break;
       case 16: __ movdqu   ($mem$$Address, $src$$XMMRegister); break;
@@ -3614,7 +3582,7 @@ instruct storeV(memory mem, vec src) %{
 // Gather INT, LONG, FLOAT, DOUBLE
 
 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
-  predicate(vector_length_in_bytes(n) <= 32);
+  predicate(Matcher::vector_length_in_bytes(n) <= 32);
   match(Set dst (LoadVectorGather mem idx));
   effect(TEMP dst, TEMP tmp, TEMP mask);
   format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
@@ -3622,9 +3590,9 @@ instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
     assert(UseAVX >= 2, "sanity");
 
     int vlen_enc = vector_length_encoding(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
-    assert(vector_length_in_bytes(this) >= 16, "sanity");
+    assert(Matcher::vector_length_in_bytes(this) >= 16, "sanity");
     assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
 
     if (vlen_enc == Assembler::AVX_128bit) {
@@ -3639,7 +3607,7 @@ instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
 %}
 
 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
-  predicate(vector_length_in_bytes(n) == 64);
+  predicate(Matcher::vector_length_in_bytes(n) == 64);
   match(Set dst (LoadVectorGather mem idx));
   effect(TEMP dst, TEMP tmp, TEMP ktmp);
   format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and k2 as TEMP" %}
@@ -3647,7 +3615,7 @@ instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
     assert(UseAVX > 2, "sanity");
 
     int vlen_enc = vector_length_encoding(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
     assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
 
@@ -3669,9 +3637,9 @@ instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
   format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
   ins_encode %{
     int vlen_enc = vector_length_encoding(this, $src);
-    BasicType elem_bt = vector_element_basic_type(this, $src);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
 
-    assert(vector_length_in_bytes(this, $src) >= 16, "sanity");
+    assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
     assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
 
     __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), $tmp$$Register);
@@ -3688,7 +3656,7 @@ instruct ReplB_reg(vec dst, rRegI src) %{
   match(Set dst (ReplicateB src));
   format %{ "replicateB $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
       assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
       int vlen_enc = vector_length_encoding(this);
@@ -3728,7 +3696,7 @@ instruct ReplB_imm(vec dst, immI con) %{
   match(Set dst (ReplicateB con));
   format %{ "replicateB $dst,$con" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     InternalAddress const_addr = $constantaddress(replicate8_imm($con$$constant, 1));
     if (vlen == 4) {
       __ movdl($dst$$XMMRegister, const_addr);
@@ -3753,7 +3721,7 @@ instruct ReplB_zero(vec dst, immI_0 zero) %{
   match(Set dst (ReplicateB zero));
   format %{ "replicateB $dst,$zero" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 16) {
       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -3771,7 +3739,7 @@ instruct ReplS_reg(vec dst, rRegI src) %{
   match(Set dst (ReplicateS src));
   format %{ "replicateS $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
       assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
       int vlen_enc = vector_length_encoding(this);
@@ -3810,7 +3778,7 @@ instruct ReplS_imm(vec dst, immI con) %{
   match(Set dst (ReplicateS con));
   format %{ "replicateS $dst,$con" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     InternalAddress const_addr = $constantaddress(replicate8_imm($con$$constant, 2));
     if (vlen == 2) {
       __ movdl($dst$$XMMRegister, const_addr);
@@ -3834,7 +3802,7 @@ instruct ReplS_zero(vec dst, immI_0 zero) %{
   match(Set dst (ReplicateS zero));
   format %{ "replicateS $dst,$zero" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 8) {
       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -3851,7 +3819,7 @@ instruct ReplI_reg(vec dst, rRegI src) %{
   match(Set dst (ReplicateI src));
   format %{ "replicateI $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
       int vlen_enc = vector_length_encoding(this);
       __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
@@ -3875,7 +3843,7 @@ instruct ReplI_mem(vec dst, memory mem) %{
   match(Set dst (ReplicateI (LoadI mem)));
   format %{ "replicateI $dst,$mem" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ movdl($dst$$XMMRegister, $mem$$Address);
       __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
@@ -3892,7 +3860,7 @@ instruct ReplI_imm(vec dst, immI con) %{
   match(Set dst (ReplicateI con));
   format %{ "replicateI $dst,$con" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     InternalAddress const_addr = $constantaddress(replicate8_imm($con$$constant, 4));
     if (vlen <= 4) {
       __ movq($dst$$XMMRegister, const_addr);
@@ -3914,7 +3882,7 @@ instruct ReplI_zero(vec dst, immI_0 zero) %{
   match(Set dst (ReplicateI zero));
   format %{ "replicateI $dst,$zero" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -3947,7 +3915,7 @@ instruct ReplL_reg(vec dst, rRegL src) %{
   match(Set dst (ReplicateL src));
   format %{ "replicateL $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ movdq($dst$$XMMRegister, $src$$Register);
       __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
@@ -3971,12 +3939,12 @@ instruct ReplL_reg(vec dst, rRegL src) %{
 #else // _LP64
 // Replicate long (8 byte) scalar to be vector
 instruct ReplL_reg(vec dst, eRegL src, vec tmp) %{
-  predicate(vector_length(n) <= 4);
+  predicate(Matcher::vector_length(n) <= 4);
   match(Set dst (ReplicateL src));
   effect(TEMP dst, USE src, TEMP tmp);
   format %{ "replicateL $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ movdl($dst$$XMMRegister, $src$$Register);
       __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
@@ -4000,7 +3968,7 @@ instruct ReplL_reg(vec dst, eRegL src, vec tmp) %{
 %}
 
 instruct ReplL_reg_leg(legVec dst, eRegL src, legVec tmp) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (ReplicateL src));
   effect(TEMP dst, USE src, TEMP tmp);
   format %{ "replicateL $dst,$src" %}
@@ -4028,7 +3996,7 @@ instruct ReplL_mem(vec dst, memory mem) %{
   match(Set dst (ReplicateL (LoadL mem)));
   format %{ "replicateL $dst,$mem" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ movq($dst$$XMMRegister, $mem$$Address);
       __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
@@ -4046,7 +4014,7 @@ instruct ReplL_imm(vec dst, immL con) %{
   match(Set dst (ReplicateL con));
   format %{ "replicateL $dst,$con" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     InternalAddress const_addr = $constantaddress($con);
     if (vlen == 2) {
       __ movq($dst$$XMMRegister, const_addr);
@@ -4065,7 +4033,7 @@ instruct ReplL_zero(vec dst, immL0 zero) %{
   match(Set dst (ReplicateL zero));
   format %{ "replicateL $dst,$zero" %}
   ins_encode %{
-    int vlen = vector_length(this);
+    int vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -4094,7 +4062,7 @@ instruct ReplF_reg(vec dst, vlRegF src) %{
   match(Set dst (ReplicateF src));
   format %{ "replicateF $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
    } else if (VM_Version::supports_avx2()) {
@@ -4113,7 +4081,7 @@ instruct ReplF_mem(vec dst, memory mem) %{
   match(Set dst (ReplicateF (LoadF mem)));
   format %{ "replicateF $dst,$mem" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ movdl($dst$$XMMRegister, $mem$$Address);
       __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
@@ -4130,7 +4098,7 @@ instruct ReplF_zero(vec dst, immF0 zero) %{
   match(Set dst (ReplicateF zero));
   format %{ "replicateF $dst,$zero" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -4148,7 +4116,7 @@ instruct ReplD_reg(vec dst, vlRegD src) %{
   match(Set dst (ReplicateD src));
   format %{ "replicateD $dst,$src" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
     } else if (VM_Version::supports_avx2()) {
@@ -4167,7 +4135,7 @@ instruct ReplD_mem(vec dst, memory mem) %{
   match(Set dst (ReplicateD (LoadD mem)));
   format %{ "replicateD $dst,$mem" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ movq($dst$$XMMRegister, $mem$$Address);
       __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x44);
@@ -4184,7 +4152,7 @@ instruct ReplD_zero(vec dst, immD0 zero) %{
   match(Set dst (ReplicateD zero));
   format %{ "replicateD $dst,$zero" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
     } else {
@@ -4198,17 +4166,17 @@ instruct ReplD_zero(vec dst, immD0 zero) %{
 // ====================VECTOR INSERT=======================================
 
 instruct insert(vec dst, rRegI val, immU8 idx) %{
-  predicate(vector_length_in_bytes(n) < 32);
+  predicate(Matcher::vector_length_in_bytes(n) < 32);
   match(Set dst (VectorInsert (Binary dst val) idx));
   format %{ "vector_insert $dst,$val,$idx" %}
   ins_encode %{
     assert(UseSSE >= 4, "required");
-    assert(vector_length_in_bytes(this) >= 8, "required");
+    assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
 
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
     assert(is_integral_type(elem_bt), "");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
   %}
@@ -4216,18 +4184,18 @@ instruct insert(vec dst, rRegI val, immU8 idx) %{
 %}
 
 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
-  predicate(vector_length_in_bytes(n) == 32);
+  predicate(Matcher::vector_length_in_bytes(n) == 32);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
     int vlen_enc = Assembler::AVX_256bit;
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     int elem_per_lane = 16/type2aelembytes(elem_bt);
     int log2epr = log2(elem_per_lane);
 
     assert(is_integral_type(elem_bt), "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(log2epr);
     uint y_idx = ($idx$$constant >> log2epr) & 1;
@@ -4239,19 +4207,19 @@ instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
 %}
 
 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
-  predicate(vector_length_in_bytes(n) == 64);
+  predicate(Matcher::vector_length_in_bytes(n) == 64);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
     assert(UseAVX > 2, "sanity");
 
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     int elem_per_lane = 16/type2aelembytes(elem_bt);
     int log2epr = log2(elem_per_lane);
 
     assert(is_integral_type(elem_bt), "");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(log2epr);
     uint y_idx = ($idx$$constant >> log2epr) & 3;
@@ -4264,13 +4232,13 @@ instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
 
 #ifdef _LP64
 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
-  predicate(vector_length(n) == 2);
+  predicate(Matcher::vector_length(n) == 2);
   match(Set dst (VectorInsert (Binary dst val) idx));
   format %{ "vector_insert $dst,$val,$idx" %}
   ins_encode %{
     assert(UseSSE >= 4, "required");
-    assert(vector_element_basic_type(this) == T_LONG, "");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
   %}
@@ -4278,13 +4246,13 @@ instruct insert2L(vec dst, rRegL val, immU8 idx) %{
 %}
 
 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
-  predicate(vector_length(n) == 4);
+  predicate(Matcher::vector_length(n) == 4);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
-    assert(vector_element_basic_type(this) == T_LONG, "");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(1);
     uint y_idx = ($idx$$constant >> 1) & 1;
@@ -4297,13 +4265,13 @@ instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
 %}
 
 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
-    assert(vector_element_basic_type(this) == T_LONG, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(1);
     uint y_idx = ($idx$$constant >> 1) & 3;
@@ -4316,14 +4284,14 @@ instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
 #endif
 
 instruct insertF(vec dst, regF val, immU8 idx) %{
-  predicate(vector_length(n) < 8);
+  predicate(Matcher::vector_length(n) < 8);
   match(Set dst (VectorInsert (Binary dst val) idx));
   format %{ "vector_insert $dst,$val,$idx" %}
   ins_encode %{
     assert(UseSSE >= 4, "sanity");
 
-    assert(vector_element_basic_type(this) == T_FLOAT, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     __ insertps($dst$$XMMRegister, $val$$XMMRegister, $idx$$constant);
   %}
@@ -4331,15 +4299,15 @@ instruct insertF(vec dst, regF val, immU8 idx) %{
 %}
 
 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
-  predicate(vector_length(n) >= 8);
+  predicate(Matcher::vector_length(n) >= 8);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
-    assert(vector_element_basic_type(this) == T_FLOAT, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
-    int vlen = vector_length(this);
+    int vlen = Matcher::vector_length(this);
     uint x_idx = $idx$$constant & right_n_bits(2);
     if (vlen == 8) {
       uint y_idx = ($idx$$constant >> 2) & 1;
@@ -4360,14 +4328,14 @@ instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
 
 #ifdef _LP64
 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
-  predicate(vector_length(n) == 2);
+  predicate(Matcher::vector_length(n) == 2);
   match(Set dst (VectorInsert (Binary dst val) idx));
   effect(TEMP tmp);
   format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
   ins_encode %{
     assert(UseSSE >= 4, "sanity");
-    assert(vector_element_basic_type(this) == T_DOUBLE, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     __ movq($tmp$$Register, $val$$XMMRegister);
     __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
@@ -4376,13 +4344,13 @@ instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
 %}
 
 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
-  predicate(vector_length(n) == 4);
+  predicate(Matcher::vector_length(n) == 4);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP vtmp, TEMP tmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
   ins_encode %{
-    assert(vector_element_basic_type(this) == T_DOUBLE, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(1);
     uint y_idx = ($idx$$constant >> 1) & 1;
@@ -4396,13 +4364,13 @@ instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
 %}
 
 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (VectorInsert (Binary src val) idx));
   effect(TEMP tmp, TEMP vtmp);
   format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
   ins_encode %{
-    assert(vector_element_basic_type(this) == T_DOUBLE, "sanity");
-    assert($idx$$constant < (int)vector_length(this), "out of bounds");
+    assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
+    assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
 
     uint x_idx = $idx$$constant & right_n_bits(1);
     uint y_idx = ($idx$$constant >> 1) & 3;
@@ -4420,7 +4388,7 @@ instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp)
 // =======================Int Reduction==========================================
 
 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_INT); // src2
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
   match(Set dst (AddReductionVI src1 src2));
   match(Set dst (MulReductionVI src1 src2));
   match(Set dst (AndReductionV  src1 src2));
@@ -4432,7 +4400,7 @@ instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
   format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4442,7 +4410,7 @@ instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
 
 #ifdef _LP64
 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
   match(Set dst (AddReductionVL src1 src2));
   match(Set dst (MulReductionVL src1 src2));
   match(Set dst (AndReductionV  src1 src2));
@@ -4454,14 +4422,14 @@ instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtm
   format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
   match(Set dst (AddReductionVL src1 src2));
   match(Set dst (MulReductionVL src1 src2));
   match(Set dst (AndReductionV  src1 src2));
@@ -4473,7 +4441,7 @@ instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtm
   format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4483,42 +4451,42 @@ instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtm
 // =======================Float Reduction==========================================
 
 instruct reductionF128(regF dst, vec src, vec vtmp) %{
-  predicate(vector_length(n->in(2)) <= 4); // src
+  predicate(Matcher::vector_length(n->in(2)) <= 4); // src
   match(Set dst (AddReductionVF dst src));
   match(Set dst (MulReductionVF dst src));
   effect(TEMP dst, TEMP vtmp);
   format %{ "vector_reduction_float  $dst,$src ; using $vtmp as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
-  predicate(vector_length(n->in(2)) == 8); // src
+  predicate(Matcher::vector_length(n->in(2)) == 8); // src
   match(Set dst (AddReductionVF dst src));
   match(Set dst (MulReductionVF dst src));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_length(n->in(2)) == 16); // src
+  predicate(Matcher::vector_length(n->in(2)) == 16); // src
   match(Set dst (AddReductionVF dst src));
   match(Set dst (MulReductionVF dst src));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4527,42 +4495,42 @@ instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
 // =======================Double Reduction==========================================
 
 instruct reduction2D(regD dst, vec src, vec vtmp) %{
-  predicate(vector_length(n->in(2)) == 2); // src
+  predicate(Matcher::vector_length(n->in(2)) == 2); // src
   match(Set dst (AddReductionVD dst src));
   match(Set dst (MulReductionVD dst src));
   effect(TEMP dst, TEMP vtmp);
   format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
 %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
-  predicate(vector_length(n->in(2)) == 4); // src
+  predicate(Matcher::vector_length(n->in(2)) == 4); // src
   match(Set dst (AddReductionVD dst src));
   match(Set dst (MulReductionVD dst src));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_length(n->in(2)) == 8); // src
+  predicate(Matcher::vector_length(n->in(2)) == 8); // src
   match(Set dst (AddReductionVD dst src));
   match(Set dst (MulReductionVD dst src));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4572,7 +4540,7 @@ instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
 
 #ifdef _LP64
 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
   match(Set dst (AddReductionVI src1 src2));
   match(Set dst (AndReductionV  src1 src2));
   match(Set dst ( OrReductionV  src1 src2));
@@ -4583,14 +4551,14 @@ instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
   format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
   match(Set dst (AddReductionVI src1 src2));
   match(Set dst (AndReductionV  src1 src2));
   match(Set dst ( OrReductionV  src1 src2));
@@ -4601,7 +4569,7 @@ instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtm
   format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4611,7 +4579,7 @@ instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtm
 // =======================Short Reduction==========================================
 
 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_SHORT); // src2
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
   match(Set dst (AddReductionVI src1 src2));
   match(Set dst (MulReductionVI src1 src2));
   match(Set dst (AndReductionV  src1 src2));
@@ -4623,7 +4591,7 @@ instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
   format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4632,28 +4600,28 @@ instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
 // =======================Mul Reduction==========================================
 
 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_BYTE &&
-            vector_length(n->in(2)) <= 32); // src2
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
+            Matcher::vector_length(n->in(2)) <= 32); // src2
   match(Set dst (MulReductionVI src1 src2));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_BYTE &&
-            vector_length(n->in(2)) == 64); // src2
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
+            Matcher::vector_length(n->in(2)) == 64); // src2
   match(Set dst (MulReductionVI src1 src2));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
   format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
@@ -4663,10 +4631,10 @@ instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legV
 // Float Min Reduction
 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp,
                             legVec atmp, legVec btmp, legVec xmm_1, rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_FLOAT &&
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
             ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
              (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
-            vector_length(n->in(2)) == 2);
+            Matcher::vector_length(n->in(2)) == 2);
   match(Set dst (MinReductionV src1 src2));
   match(Set dst (MaxReductionV src1 src2));
   effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
@@ -4675,7 +4643,7 @@ instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
                          $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
   %}
@@ -4684,10 +4652,10 @@ instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp,
 
 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
                            legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_FLOAT &&
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
             ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
              (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
-            vector_length(n->in(2)) >= 4);
+            Matcher::vector_length(n->in(2)) >= 4);
   match(Set dst (MinReductionV src1 src2));
   match(Set dst (MaxReductionV src1 src2));
   effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
@@ -4696,7 +4664,7 @@ instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legV
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
                          $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
   %}
@@ -4705,8 +4673,8 @@ instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legV
 
 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp,
                                legVec atmp, legVec btmp, legVec xmm_1, rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_FLOAT &&
-            vector_length(n->in(2)) == 2);
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
+            Matcher::vector_length(n->in(2)) == 2);
   match(Set dst (MinReductionV dst src));
   match(Set dst (MaxReductionV dst src));
   effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
@@ -4715,7 +4683,7 @@ instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
                          $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
   %}
@@ -4725,8 +4693,8 @@ instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp,
 
 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp,
                               legVec atmp, legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_FLOAT &&
-            vector_length(n->in(2)) >= 4);
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
+            Matcher::vector_length(n->in(2)) >= 4);
   match(Set dst (MinReductionV dst src));
   match(Set dst (MaxReductionV dst src));
   effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
@@ -4735,7 +4703,7 @@ instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
                          $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
   %}
@@ -4747,10 +4715,10 @@ instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp,
 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2,
                             legVec tmp1, legVec tmp2, legVec tmp3, legVec tmp4, // TEMPs
                             rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_DOUBLE &&
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
             ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
              (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
-            vector_length(n->in(2)) == 2);
+            Matcher::vector_length(n->in(2)) == 2);
   match(Set dst (MinReductionV src1 src2));
   match(Set dst (MaxReductionV src1 src2));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
@@ -4759,7 +4727,7 @@ instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
                           $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
   %}
@@ -4769,10 +4737,10 @@ instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2,
 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2,
                            legVec tmp1, legVec tmp2, legVec tmp3, legVec tmp4, legVec tmp5, // TEMPs
                            rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_DOUBLE &&
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
             ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
              (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
-            vector_length(n->in(2)) >= 4);
+            Matcher::vector_length(n->in(2)) >= 4);
   match(Set dst (MinReductionV src1 src2));
   match(Set dst (MaxReductionV src1 src2));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
@@ -4781,7 +4749,7 @@ instruct minmax_reductionD(legRegD dst, immD src1, legVec src2,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src2);
+    int vlen = Matcher::vector_length(this, $src2);
     __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
                           $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
   %}
@@ -4792,8 +4760,8 @@ instruct minmax_reductionD(legRegD dst, immD src1, legVec src2,
 instruct minmax_reduction2D_av(legRegD dst, legVec src,
                                legVec tmp1, legVec tmp2, legVec tmp3, legVec tmp4, // TEMPs
                                rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_DOUBLE &&
-            vector_length(n->in(2)) == 2);
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
+            Matcher::vector_length(n->in(2)) == 2);
   match(Set dst (MinReductionV dst src));
   match(Set dst (MaxReductionV dst src));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
@@ -4802,7 +4770,7 @@ instruct minmax_reduction2D_av(legRegD dst, legVec src,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
                           $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
   %}
@@ -4812,8 +4780,8 @@ instruct minmax_reduction2D_av(legRegD dst, legVec src,
 instruct minmax_reductionD_av(legRegD dst, legVec src,
                               legVec tmp1, legVec tmp2, legVec tmp3, legVec tmp4, legVec tmp5, // TEMPs
                               rFlagsReg cr) %{
-  predicate(vector_element_basic_type(n->in(2)) == T_DOUBLE &&
-            vector_length(n->in(2)) >= 4);
+  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
+            Matcher::vector_length(n->in(2)) >= 4);
   match(Set dst (MinReductionV dst src));
   match(Set dst (MaxReductionV dst src));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
@@ -4822,7 +4790,7 @@ instruct minmax_reductionD_av(legRegD dst, legVec src,
     assert(UseAVX > 0, "sanity");
 
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this, $src);
+    int vlen = Matcher::vector_length(this, $src);
     __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
                           $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
   %}
@@ -4856,7 +4824,8 @@ instruct vaddB_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddB_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVB src (LoadVector mem)));
   format %{ "vpaddb  $dst,$src,$mem\t! add packedB" %}
   ins_encode %{
@@ -4889,7 +4858,8 @@ instruct vaddS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVS src (LoadVector mem)));
   format %{ "vpaddw  $dst,$src,$mem\t! add packedS" %}
   ins_encode %{
@@ -4923,7 +4893,8 @@ instruct vaddI_reg(vec dst, vec src1, vec src2) %{
 
 
 instruct vaddI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVI src (LoadVector mem)));
   format %{ "vpaddd  $dst,$src,$mem\t! add packedI" %}
   ins_encode %{
@@ -4956,7 +4927,8 @@ instruct vaddL_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddL_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVL src (LoadVector mem)));
   format %{ "vpaddq  $dst,$src,$mem\t! add packedL" %}
   ins_encode %{
@@ -4989,7 +4961,8 @@ instruct vaddF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVF src (LoadVector mem)));
   format %{ "vaddps  $dst,$src,$mem\t! add packedF" %}
   ins_encode %{
@@ -5022,7 +4995,8 @@ instruct vaddD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVD src (LoadVector mem)));
   format %{ "vaddpd  $dst,$src,$mem\t! add packedD" %}
   ins_encode %{
@@ -5057,7 +5031,8 @@ instruct vsubB_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubB_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVB src (LoadVector mem)));
   format %{ "vpsubb  $dst,$src,$mem\t! sub packedB" %}
   ins_encode %{
@@ -5091,7 +5066,8 @@ instruct vsubS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVS src (LoadVector mem)));
   format %{ "vpsubw  $dst,$src,$mem\t! sub packedS" %}
   ins_encode %{
@@ -5124,7 +5100,8 @@ instruct vsubI_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVI src (LoadVector mem)));
   format %{ "vpsubd  $dst,$src,$mem\t! sub packedI" %}
   ins_encode %{
@@ -5158,7 +5135,8 @@ instruct vsubL_reg(vec dst, vec src1, vec src2) %{
 
 
 instruct vsubL_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVL src (LoadVector mem)));
   format %{ "vpsubq  $dst,$src,$mem\t! sub packedL" %}
   ins_encode %{
@@ -5191,7 +5169,8 @@ instruct vsubF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVF src (LoadVector mem)));
   format %{ "vsubps  $dst,$src,$mem\t! sub packedF" %}
   ins_encode %{
@@ -5224,7 +5203,8 @@ instruct vsubD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVD src (LoadVector mem)));
   format %{ "vsubpd  $dst,$src,$mem\t! sub packedD" %}
   ins_encode %{
@@ -5238,8 +5218,8 @@ instruct vsubD_mem(vec dst, vec src, memory mem) %{
 
 // Byte vector mul
 instruct mulB_reg(vec dst, vec src1, vec src2, vec tmp, rRegI scratch) %{
-  predicate(vector_length(n) == 4 ||
-            vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 4 ||
+            Matcher::vector_length(n) == 8);
   match(Set dst (MulVB src1 src2));
   effect(TEMP dst, TEMP tmp, TEMP scratch);
   format %{"vector_mulB $dst,$src1,$src2" %}
@@ -5256,7 +5236,7 @@ instruct mulB_reg(vec dst, vec src1, vec src2, vec tmp, rRegI scratch) %{
 %}
 
 instruct mul16B_reg(vec dst, vec src1, vec src2, vec tmp1, vec tmp2, rRegI scratch) %{
-  predicate(vector_length(n) == 16 && UseAVX <= 1);
+  predicate(Matcher::vector_length(n) == 16 && UseAVX <= 1);
   match(Set dst (MulVB src1 src2));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP scratch);
   format %{"vector_mulB $dst,$src1,$src2" %}
@@ -5279,7 +5259,7 @@ instruct mul16B_reg(vec dst, vec src1, vec src2, vec tmp1, vec tmp2, rRegI scrat
 %}
 
 instruct vmul16B_reg_avx(vec dst, vec src1, vec src2, vec tmp, rRegI scratch) %{
-  predicate(vector_length(n) == 16 && UseAVX > 1);
+  predicate(Matcher::vector_length(n) == 16 && UseAVX > 1);
   match(Set dst (MulVB src1 src2));
   effect(TEMP dst, TEMP tmp, TEMP scratch);
   format %{"vector_mulB $dst,$src1,$src2" %}
@@ -5297,7 +5277,7 @@ instruct vmul16B_reg_avx(vec dst, vec src1, vec src2, vec tmp, rRegI scratch) %{
 %}
 
 instruct vmul32B_reg_avx(vec dst, vec src1, vec src2, vec tmp1, vec tmp2, rRegI scratch) %{
-  predicate(vector_length(n) == 32);
+  predicate(Matcher::vector_length(n) == 32);
   match(Set dst (MulVB src1 src2));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP scratch);
   format %{"vector_mulB $dst,$src1,$src2" %}
@@ -5323,7 +5303,7 @@ instruct vmul32B_reg_avx(vec dst, vec src1, vec src2, vec tmp1, vec tmp2, rRegI
 %}
 
 instruct vmul64B_reg_avx(vec dst, vec src1, vec src2, vec tmp1, vec tmp2, rRegI scratch) %{
-  predicate(vector_length(n) == 64);
+  predicate(Matcher::vector_length(n) == 64);
   match(Set dst (MulVB src1 src2));
   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP scratch);
   format %{"vector_mulB $dst,$src1,$src2\n\t" %}
@@ -5372,7 +5352,8 @@ instruct vmulS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVS src (LoadVector mem)));
   format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
   ins_encode %{
@@ -5406,7 +5387,8 @@ instruct vmulI_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVI src (LoadVector mem)));
   format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
   ins_encode %{
@@ -5430,7 +5412,8 @@ instruct vmulL_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulL_mem(vec dst, vec src, memory mem) %{
-  predicate(VM_Version::supports_avx512dq());
+  predicate(VM_Version::supports_avx512dq() &&
+              (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVL src (LoadVector mem)));
   format %{ "vpmullq $dst,$src,$mem\t! mul packedL" %}
   ins_encode %{
@@ -5442,7 +5425,7 @@ instruct vmulL_mem(vec dst, vec src, memory mem) %{
 %}
 
 instruct mul2L_reg(vec dst, vec src2, legVec tmp) %{
-  predicate(vector_length(n) == 2 && !VM_Version::supports_avx512dq());
+  predicate(Matcher::vector_length(n) == 2 && !VM_Version::supports_avx512dq());
   match(Set dst (MulVL dst src2));
   effect(TEMP dst, TEMP tmp);
   format %{ "pshufd $tmp,$src2, 177\n\t"
@@ -5468,7 +5451,7 @@ instruct mul2L_reg(vec dst, vec src2, legVec tmp) %{
 %}
 
 instruct vmul4L_reg_avx(vec dst, vec src1, vec src2, legVec tmp, legVec tmp1) %{
-  predicate(vector_length(n) == 4 && !VM_Version::supports_avx512dq());
+  predicate(Matcher::vector_length(n) == 4 && !VM_Version::supports_avx512dq());
   match(Set dst (MulVL src1 src2));
   effect(TEMP tmp1, TEMP tmp);
   format %{ "vpshufd $tmp,$src2\n\t"
@@ -5515,7 +5498,8 @@ instruct vmulF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVF src (LoadVector mem)));
   format %{ "vmulps  $dst,$src,$mem\t! mul packedF" %}
   ins_encode %{
@@ -5548,7 +5532,8 @@ instruct vmulD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVD src (LoadVector mem)));
   format %{ "vmulpd  $dst,$src,$mem\t! mul packedD" %}
   ins_encode %{
@@ -5559,7 +5544,7 @@ instruct vmulD_mem(vec dst, vec src, memory mem) %{
 %}
 
 instruct vcmov8F_reg(legVec dst, legVec src1, legVec src2, immI8 cop, cmpOp_vcmppd copnd) %{
-  predicate(vector_length(n) == 8);
+  predicate(Matcher::vector_length(n) == 8);
   match(Set dst (CMoveVF (Binary copnd cop) (Binary src1 src2)));
   effect(TEMP dst, USE src1, USE src2);
   format %{ "cmpps.$copnd  $dst, $src1, $src2  ! vcmovevf, cond=$cop\n\t"
@@ -5577,7 +5562,7 @@ instruct vcmov8F_reg(legVec dst, legVec src1, legVec src2, immI8 cop, cmpOp_vcmp
 %}
 
 instruct vcmov4D_reg(legVec dst, legVec src1, legVec src2, immI8 cop, cmpOp_vcmppd copnd) %{
-  predicate(vector_length(n) == 4);
+  predicate(Matcher::vector_length(n) == 4);
   match(Set dst (CMoveVD (Binary copnd cop) (Binary src1 src2)));
   effect(TEMP dst, USE src1, USE src2);
   format %{ "cmppd.$copnd  $dst, $src1, $src2  ! vcmovevd, cond=$cop\n\t"
@@ -5619,7 +5604,8 @@ instruct vdivF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vdivF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (DivVF src (LoadVector mem)));
   format %{ "vdivps  $dst,$src,$mem\t! div packedF" %}
   ins_encode %{
@@ -5652,7 +5638,8 @@ instruct vdivD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vdivD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (DivVD src (LoadVector mem)));
   format %{ "vdivpd  $dst,$src,$mem\t! div packedD" %}
   ins_encode %{
@@ -5666,7 +5653,7 @@ instruct vdivD_mem(vec dst, vec src, memory mem) %{
 
 // Byte, Short, Int vector Min/Max
 instruct minmax_reg_sse(vec dst, vec src) %{
-  predicate(is_integral_type(vector_element_basic_type(n)) && vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
+  predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
             UseAVX == 0);
   match(Set dst (MinV dst src));
   match(Set dst (MaxV dst src));
@@ -5675,14 +5662,14 @@ instruct minmax_reg_sse(vec dst, vec src) %{
     assert(UseSSE >= 4, "required");
 
     int opcode = this->ideal_Opcode();
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
-  predicate(is_integral_type(vector_element_basic_type(n)) && vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
+  predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
             UseAVX > 0);
   match(Set dst (MinV src1 src2));
   match(Set dst (MaxV src1 src2));
@@ -5690,7 +5677,7 @@ instruct vminmax_reg(vec dst, vec src1, vec src2) %{
   ins_encode %{
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
     __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
   %}
@@ -5699,7 +5686,7 @@ instruct vminmax_reg(vec dst, vec src1, vec src2) %{
 
 // Long vector Min/Max
 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
-  predicate(vector_length_in_bytes(n) == 16 && vector_element_basic_type(n) == T_LONG &&
+  predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
             UseAVX == 0);
   match(Set dst (MinV dst src));
   match(Set dst (MaxV src dst));
@@ -5709,7 +5696,7 @@ instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
     assert(UseSSE >= 4, "required");
 
     int opcode = this->ideal_Opcode();
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     assert(elem_bt == T_LONG, "sanity");
 
     __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
@@ -5718,7 +5705,7 @@ instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
 %}
 
 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
-  predicate(vector_length_in_bytes(n) <= 32 && vector_element_basic_type(n) == T_LONG &&
+  predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
             UseAVX > 0 && !VM_Version::supports_avx512vl());
   match(Set dst (MinV src1 src2));
   match(Set dst (MaxV src1 src2));
@@ -5727,7 +5714,7 @@ instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
   ins_encode %{
     int vlen_enc = vector_length_encoding(this);
     int opcode = this->ideal_Opcode();
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     assert(elem_bt == T_LONG, "sanity");
 
     __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
@@ -5736,8 +5723,8 @@ instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
 %}
 
 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
-  predicate((vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
-            vector_element_basic_type(n) == T_LONG);
+  predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
+            Matcher::vector_element_basic_type(n) == T_LONG);
   match(Set dst (MinV src1 src2));
   match(Set dst (MaxV src1 src2));
   format %{ "vector_minmaxL  $dst,$src1,src2\t! " %}
@@ -5746,7 +5733,7 @@ instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
 
     int vlen_enc = vector_length_encoding(this);
     int opcode = this->ideal_Opcode();
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
     assert(elem_bt == T_LONG, "sanity");
 
     __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
@@ -5756,8 +5743,8 @@ instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
 
 // Float/Double vector Min/Max
 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
-  predicate(vector_length_in_bytes(n) <= 32 &&
-            is_floating_point_type(vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
+  predicate(Matcher::vector_length_in_bytes(n) <= 32 &&
+            is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
             UseAVX > 0);
   match(Set dst (MinV a b));
   match(Set dst (MaxV a b));
@@ -5768,7 +5755,7 @@ instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, l
 
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
     __ vminmax_fp(opcode, elem_bt,
                   $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
@@ -5778,8 +5765,8 @@ instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, l
 %}
 
 instruct evminmaxFP_reg_eavx(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
-  predicate(vector_length_in_bytes(n) == 64 &&
-            is_floating_point_type(vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
+  predicate(Matcher::vector_length_in_bytes(n) == 64 &&
+            is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
   match(Set dst (MinV a b));
   match(Set dst (MaxV a b));
   effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
@@ -5789,7 +5776,7 @@ instruct evminmaxFP_reg_eavx(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktm
 
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
     __ evminmax_fp(opcode, elem_bt,
                    $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
@@ -5836,6 +5823,7 @@ instruct vsqrtF_reg(vec dst, vec src) %{
 %}
 
 instruct vsqrtF_mem(vec dst, memory mem) %{
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
   match(Set dst (SqrtVF (LoadVector mem)));
   format %{ "vsqrtps  $dst,$mem\t! sqrt packedF" %}
   ins_encode %{
@@ -5859,6 +5847,7 @@ instruct vsqrtD_reg(vec dst, vec src) %{
 %}
 
 instruct vsqrtD_mem(vec dst, memory mem) %{
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
   match(Set dst (SqrtVD (LoadVector mem)));
   format %{ "vsqrtpd  $dst,$mem\t! sqrt packedD" %}
   ins_encode %{
@@ -5885,7 +5874,7 @@ instruct vshiftcnt(vec dst, rRegI cnt) %{
 
 // Byte vector shift
 instruct vshiftB(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
-  predicate(vector_length(n) <= 8 && VectorNode::is_vshift_cnt(n->in(2)));
+  predicate(Matcher::vector_length(n) <= 8 && VectorNode::is_vshift_cnt(n->in(2)));
   match(Set dst ( LShiftVB src shift));
   match(Set dst ( RShiftVB src shift));
   match(Set dst (URShiftVB src shift));
@@ -5905,7 +5894,7 @@ instruct vshiftB(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
 %}
 
 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2, rRegI scratch) %{
-  predicate(vector_length(n) == 16 && VectorNode::is_vshift_cnt(n->in(2)) &&
+  predicate(Matcher::vector_length(n) == 16 && VectorNode::is_vshift_cnt(n->in(2)) &&
             UseAVX <= 1);
   match(Set dst ( LShiftVB src shift));
   match(Set dst ( RShiftVB src shift));
@@ -5930,7 +5919,7 @@ instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2, rRegI scratc
 %}
 
 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
-  predicate(vector_length(n) == 16 && VectorNode::is_vshift_cnt(n->in(2)) &&
+  predicate(Matcher::vector_length(n) == 16 && VectorNode::is_vshift_cnt(n->in(2)) &&
             UseAVX > 1);
   match(Set dst ( LShiftVB src shift));
   match(Set dst ( RShiftVB src shift));
@@ -5951,7 +5940,7 @@ instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
 %}
 
 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
-  predicate(vector_length(n) == 32 && VectorNode::is_vshift_cnt(n->in(2)));
+  predicate(Matcher::vector_length(n) == 32 && VectorNode::is_vshift_cnt(n->in(2)));
   match(Set dst ( LShiftVB src shift));
   match(Set dst ( RShiftVB src shift));
   match(Set dst (URShiftVB src shift));
@@ -5976,7 +5965,7 @@ instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
 %}
 
 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2, rRegI scratch) %{
-  predicate(vector_length(n) == 64 && VectorNode::is_vshift_cnt(n->in(2)));
+  predicate(Matcher::vector_length(n) == 64 && VectorNode::is_vshift_cnt(n->in(2)));
   match(Set dst ( LShiftVB src shift));
   match(Set dst  (RShiftVB src shift));
   match(Set dst (URShiftVB src shift));
@@ -6021,7 +6010,7 @@ instruct vshiftS(vec dst, vec src, vec shift) %{
       int vlen_enc = vector_length_encoding(this);
       __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
     } else {
-      int vlen = vector_length(this);
+      int vlen = Matcher::vector_length(this);
       if (vlen == 2) {
         __ movflt($dst$$XMMRegister, $src$$XMMRegister);
         __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
@@ -6052,7 +6041,7 @@ instruct vshiftI(vec dst, vec src, vec shift) %{
       int vlen_enc = vector_length_encoding(this);
       __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
     } else {
-      int vlen = vector_length(this);
+      int vlen = Matcher::vector_length(this);
       if (vlen == 2) {
         __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
         __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
@@ -6078,7 +6067,7 @@ instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
       int vector_len = vector_length_encoding(this);
       __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
     } else {
-      int vlen = vector_length(this);
+      int vlen = Matcher::vector_length(this);
       if (vlen == 2) {
         __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
         __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
@@ -6105,7 +6094,7 @@ instruct vshiftL(vec dst, vec src, vec shift) %{
       int vlen_enc = vector_length_encoding(this);
       __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
     } else {
-      assert(vector_length(this) == 2, "");
+      assert(Matcher::vector_length(this) == 2, "");
       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
       __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
     }
@@ -6124,7 +6113,7 @@ instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
       int vector_len = vector_length_encoding(this);
       __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
     } else {
-      assert(vector_length(this) == 2, "");
+      assert(Matcher::vector_length(this) == 2, "");
       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
       __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
     }
@@ -6140,7 +6129,7 @@ instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp, rRegI scratch)
   effect(TEMP dst, TEMP tmp, TEMP scratch);
   format %{ "vshiftq $dst,$src,$shift" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       assert(UseSSE >= 2, "required");
       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
@@ -6177,7 +6166,7 @@ instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
 // ------------------- Variable Shift -----------------------------
 // Byte variable shift
 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp, rRegP scratch) %{
-  predicate(vector_length(n) <= 8 &&
+  predicate(Matcher::vector_length(n) <= 8 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             !VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVB src shift));
@@ -6197,7 +6186,7 @@ instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp, rRegP scratch)
 %}
 
 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, rRegP scratch) %{
-  predicate(vector_length(n) == 16 &&
+  predicate(Matcher::vector_length(n) == 16 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             !VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVB src shift));
@@ -6225,7 +6214,7 @@ instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, r
 %}
 
 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4, rRegP scratch) %{
-  predicate(vector_length(n) == 32 &&
+  predicate(Matcher::vector_length(n) == 32 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             !VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVB src shift));
@@ -6261,7 +6250,7 @@ instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, v
 %}
 
 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp, rRegP scratch) %{
-  predicate(vector_length(n) <= 32 &&
+  predicate(Matcher::vector_length(n) <= 32 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVB src shift));
@@ -6280,7 +6269,7 @@ instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp, rRegP scratc
 %}
 
 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, rRegP scratch) %{
-  predicate(vector_length(n) == 64 &&
+  predicate(Matcher::vector_length(n) == 64 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVB src shift));
@@ -6304,7 +6293,7 @@ instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2
 
 // Short variable shift
 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp, rRegP scratch) %{
-  predicate(vector_length(n) <= 8 &&
+  predicate(Matcher::vector_length(n) <= 8 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             !VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVS src shift));
@@ -6329,7 +6318,7 @@ instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp, rRegP scratch)
 %}
 
 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, rRegP scratch) %{
-  predicate(vector_length(n) == 16 &&
+  predicate(Matcher::vector_length(n) == 16 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             !VM_Version::supports_avx512bw());
   match(Set dst ( LShiftVS src shift));
@@ -6419,7 +6408,7 @@ instruct vshiftL_var(vec dst, vec src, vec shift) %{
 
 //Long variable right shift arithmetic
 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
-  predicate(vector_length(n) <= 4 &&
+  predicate(Matcher::vector_length(n) <= 4 &&
             !VectorNode::is_vshift_cnt(n->in(2)) &&
             UseAVX == 2);
   match(Set dst (RShiftVL src shift));
@@ -6471,7 +6460,8 @@ instruct vand_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vand_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AndV src (LoadVector mem)));
   format %{ "vpand   $dst,$src,$mem\t! and vectors" %}
   ins_encode %{
@@ -6505,7 +6495,8 @@ instruct vor_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vor_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (OrV src (LoadVector mem)));
   format %{ "vpor    $dst,$src,$mem\t! or vectors" %}
   ins_encode %{
@@ -6539,7 +6530,8 @@ instruct vxor_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vxor_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (Matcher::vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (XorV src (LoadVector mem)));
   format %{ "vpxor   $dst,$src,$mem\t! xor vectors" %}
   ins_encode %{
@@ -6557,7 +6549,7 @@ instruct vcastBtoX(vec dst, vec src) %{
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    BasicType to_elem_bt = vector_element_basic_type(this);
+    BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
     int vlen_enc = vector_length_encoding(this);
     switch (to_elem_bt) {
       case T_SHORT:
@@ -6585,9 +6577,9 @@ instruct vcastBtoX(vec dst, vec src) %{
 %}
 
 instruct castStoX(vec dst, vec src, rRegP scratch) %{
-  predicate(UseAVX <= 2 &&
-            vector_length(n->in(1)) <= 8 && // src
-            vector_element_basic_type(n) == T_BYTE);
+  predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
+            Matcher::vector_length(n->in(1)) <= 8 && // src
+            Matcher::vector_element_basic_type(n) == T_BYTE);
   effect(TEMP scratch);
   match(Set dst (VectorCastS2X src));
   format %{ "vector_cast_s2x $dst,$src\t! using $scratch as TEMP" %}
@@ -6601,16 +6593,16 @@ instruct castStoX(vec dst, vec src, rRegP scratch) %{
 %}
 
 instruct vcastStoX(vec dst, vec src, vec vtmp, rRegP scratch) %{
-  predicate(UseAVX <= 2 &&
-            vector_length(n->in(1)) == 16 && // src
-            vector_element_basic_type(n) == T_BYTE);
+  predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
+            Matcher::vector_length(n->in(1)) == 16 && // src
+            Matcher::vector_element_basic_type(n) == T_BYTE);
   effect(TEMP dst, TEMP vtmp, TEMP scratch);
   match(Set dst (VectorCastS2X src));
   format %{ "vector_cast_s2x $dst,$src\t! using $vtmp, $scratch as TEMP" %}
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    int vlen_enc = vector_length_encoding(vector_length_in_bytes(this, $src));
+    int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, $scratch$$Register);
     __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
@@ -6619,12 +6611,12 @@ instruct vcastStoX(vec dst, vec src, vec vtmp, rRegP scratch) %{
 %}
 
 instruct vcastStoX_evex(vec dst, vec src) %{
-  predicate(UseAVX > 2 ||
-            (vector_length_in_bytes(n) >= vector_length_in_bytes(n->in(1)))); // dst >= src
+  predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
+            (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
   match(Set dst (VectorCastS2X src));
   format %{ "vector_cast_s2x $dst,$src\t!" %}
   ins_encode %{
-    BasicType to_elem_bt = vector_element_basic_type(this);
+    BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
     int src_vlen_enc = vector_length_encoding(this, $src);
     int vlen_enc = vector_length_encoding(this);
     switch (to_elem_bt) {
@@ -6657,15 +6649,15 @@ instruct vcastStoX_evex(vec dst, vec src) %{
 
 instruct castItoX(vec dst, vec src, rRegP scratch) %{
   predicate(UseAVX <= 2 &&
-            (vector_length_in_bytes(n->in(1)) <= 16) &&
-            (vector_length_in_bytes(n) < vector_length_in_bytes(n->in(1)))); // dst < src
+            (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
+            (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
   match(Set dst (VectorCastI2X src));
   format %{ "vector_cast_i2x $dst,$src\t! using $scratch as TEMP" %}
   effect(TEMP scratch);
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    BasicType to_elem_bt = vector_element_basic_type(this);
+    BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
     int vlen_enc = vector_length_encoding(this, $src);
 
     if (to_elem_bt == T_BYTE) {
@@ -6683,15 +6675,15 @@ instruct castItoX(vec dst, vec src, rRegP scratch) %{
 
 instruct vcastItoX(vec dst, vec src, vec vtmp, rRegP scratch) %{
   predicate(UseAVX <= 2 &&
-            (vector_length_in_bytes(n->in(1)) == 32) &&
-            (vector_length_in_bytes(n) < vector_length_in_bytes(n->in(1)))); // dst < src
+            (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
+            (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
   match(Set dst (VectorCastI2X src));
   format %{ "vector_cast_i2x $dst,$src\t! using $vtmp and $scratch as TEMP" %}
   effect(TEMP dst, TEMP vtmp, TEMP scratch);
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    BasicType to_elem_bt = vector_element_basic_type(this);
+    BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
     int vlen_enc = vector_length_encoding(this, $src);
 
     if (to_elem_bt == T_BYTE) {
@@ -6711,13 +6703,13 @@ instruct vcastItoX(vec dst, vec src, vec vtmp, rRegP scratch) %{
 
 instruct vcastItoX_evex(vec dst, vec src) %{
   predicate(UseAVX > 2 ||
-            (vector_length_in_bytes(n) >= vector_length_in_bytes(n->in(1)))); // dst >= src
+            (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
   match(Set dst (VectorCastI2X src));
   format %{ "vector_cast_i2x $dst,$src\t!" %}
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    BasicType dst_elem_bt = vector_element_basic_type(this);
+    BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
     int src_vlen_enc = vector_length_encoding(this, $src);
     int dst_vlen_enc = vector_length_encoding(this);
     switch (dst_elem_bt) {
@@ -6734,13 +6726,13 @@ instruct vcastItoX_evex(vec dst, vec src) %{
         __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
         break;
       case T_FLOAT:
-        __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
+        __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
         break;
       case T_LONG:
         __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
         break;
       case T_DOUBLE:
-        __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
+        __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
         break;
       default:
         ShouldNotReachHere();
@@ -6750,7 +6742,7 @@ instruct vcastItoX_evex(vec dst, vec src) %{
 %}
 
 instruct vcastLtoBS(vec dst, vec src, rRegP scratch) %{
-  predicate((vector_element_basic_type(n) == T_BYTE || vector_element_basic_type(n) == T_SHORT) &&
+  predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
             UseAVX <= 2);
   match(Set dst (VectorCastL2X src));
   effect(TEMP scratch);
@@ -6758,8 +6750,8 @@ instruct vcastLtoBS(vec dst, vec src, rRegP scratch) %{
   ins_encode %{
     assert(UseAVX > 0, "required");
 
-    int vlen = vector_length_in_bytes(this, $src);
-    BasicType to_elem_bt  = vector_element_basic_type(this);
+    int vlen = Matcher::vector_length_in_bytes(this, $src);
+    BasicType to_elem_bt  = Matcher::vector_element_basic_type(this);
     AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
                                                       : ExternalAddress(vector_int_to_short_mask());
     if (vlen <= 16) {
@@ -6782,14 +6774,14 @@ instruct vcastLtoBS(vec dst, vec src, rRegP scratch) %{
 
 instruct vcastLtoX_evex(vec dst, vec src) %{
   predicate(UseAVX > 2 ||
-            (vector_element_basic_type(n) == T_INT ||
-             vector_element_basic_type(n) == T_FLOAT ||
-             vector_element_basic_type(n) == T_DOUBLE));
+            (Matcher::vector_element_basic_type(n) == T_INT ||
+             Matcher::vector_element_basic_type(n) == T_FLOAT ||
+             Matcher::vector_element_basic_type(n) == T_DOUBLE));
   match(Set dst (VectorCastL2X src));
   format %{ "vector_cast_l2x  $dst,$src\t!" %}
   ins_encode %{
-    BasicType to_elem_bt = vector_element_basic_type(this);
-    int vlen = vector_length_in_bytes(this, $src);
+    BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
+    int vlen = Matcher::vector_length_in_bytes(this, $src);
     int vlen_enc = vector_length_encoding(this, $src);
     switch (to_elem_bt) {
       case T_BYTE:
@@ -6841,7 +6833,7 @@ instruct vcastLtoX_evex(vec dst, vec src) %{
 %}
 
 instruct vcastFtoD_reg(vec dst, vec src) %{
-  predicate(vector_element_basic_type(n) == T_DOUBLE);
+  predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
   match(Set dst (VectorCastF2X src));
   format %{ "vector_cast_f2x  $dst,$src\t!" %}
   ins_encode %{
@@ -6852,7 +6844,7 @@ instruct vcastFtoD_reg(vec dst, vec src) %{
 %}
 
 instruct vcastDtoF_reg(vec dst, vec src) %{
-  predicate(vector_element_basic_type(n) == T_FLOAT);
+  predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
   match(Set dst (VectorCastD2X src));
   format %{ "vector_cast_d2x  $dst,$src\t!" %}
   ins_encode %{
@@ -6865,15 +6857,15 @@ instruct vcastDtoF_reg(vec dst, vec src) %{
 // --------------------------------- VectorMaskCmp --------------------------------------
 
 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
-  predicate(vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
-            vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
-            is_floating_point_type(vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
+            is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
   ins_encode %{
     int vlen_enc = vector_length_encoding(this, $src1);
     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
-    if (vector_element_basic_type(this, $src1) == T_FLOAT) {
+    if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
       __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
     } else {
       __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
@@ -6883,8 +6875,8 @@ instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
 %}
 
 instruct evcmpFD(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg ktmp) %{
-  predicate(vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
-            is_floating_point_type(vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
+            is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   effect(TEMP scratch, TEMP ktmp);
   format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $scratch as TEMP" %}
@@ -6892,7 +6884,7 @@ instruct evcmpFD(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg kt
     int vlen_enc = Assembler::AVX_512bit;
     Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
     KRegister mask = k0; // The comparison itself is not being masked.
-    if (vector_element_basic_type(this, $src1) == T_FLOAT) {
+    if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
       __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
       __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, $scratch$$Register);
     } else {
@@ -6906,16 +6898,16 @@ instruct evcmpFD(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg kt
 instruct vcmp(legVec dst, legVec src1, legVec src2, immI8 cond, rRegP scratch) %{
   predicate((UseAVX <= 2 || !VM_Version::supports_avx512vl()) &&
             !is_unsigned_booltest_pred(n->in(2)->get_int()) &&
-            vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
-            vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
-            is_integral_type(vector_element_basic_type(n->in(1)->in(1)))); // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
+            is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   effect(TEMP scratch);
   format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $scratch as TEMP" %}
   ins_encode %{
     int vlen_enc = vector_length_encoding(this, $src1);
     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
-    Assembler::Width ww = widthForType(vector_element_basic_type(this, $src1));
+    Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
     __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, ww, vlen_enc, $scratch$$Register);
   %}
   ins_pipe( pipe_slow );
@@ -6924,16 +6916,16 @@ instruct vcmp(legVec dst, legVec src1, legVec src2, immI8 cond, rRegP scratch) %
 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec vtmp1, legVec vtmp2, rRegP scratch) %{
   predicate((UseAVX == 2 || !VM_Version::supports_avx512vl()) &&
             is_unsigned_booltest_pred(n->in(2)->get_int()) &&
-            vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
-            vector_length_in_bytes(n->in(1)->in(1)) <= 16 && // src1
-            is_integral_type(vector_element_basic_type(n->in(1)->in(1)))); // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 16 && // src1
+            is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   effect(TEMP vtmp1, TEMP vtmp2, TEMP scratch);
   format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $scratch as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
-    BasicType bt = vector_element_basic_type(this, $src1);
+    BasicType bt = Matcher::vector_element_basic_type(this, $src1);
     __ vpcmpu(bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen, $vtmp1$$XMMRegister,
               $vtmp2$$XMMRegister, $scratch$$Register);
   %}
@@ -6943,15 +6935,15 @@ instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec vtmp1, l
 instruct vcmpu32(legVec dst, legVec src1, legVec src2, immI8 cond, legVec vtmp1, legVec vtmp2, legVec vtmp3, rRegP scratch) %{
   predicate((UseAVX == 2 || !VM_Version::supports_avx512vl()) &&
             is_unsigned_booltest_pred(n->in(2)->get_int()) &&
-            vector_length_in_bytes(n->in(1)->in(1)) == 32 && // src1
-            is_integral_type(vector_element_basic_type(n->in(1)->in(1)))); // src1
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 32 && // src1
+            is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP scratch);
   format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $scratch as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
-    BasicType bt = vector_element_basic_type(this, $src1);
+    BasicType bt = Matcher::vector_element_basic_type(this, $src1);
     __ vpcmpu32(bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen, $vtmp1$$XMMRegister,
                 $vtmp2$$XMMRegister, $vtmp3$$XMMRegister, $scratch$$Register);
   %}
@@ -6961,8 +6953,8 @@ instruct vcmpu32(legVec dst, legVec src1, legVec src2, immI8 cond, legVec vtmp1,
 instruct evcmp(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg ktmp) %{
   predicate(UseAVX > 2 &&
             (VM_Version::supports_avx512vl() ||
-             vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
-             is_integral_type(vector_element_basic_type(n->in(1)->in(1)))); // src1
+             Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
+             is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
   effect(TEMP scratch, TEMP ktmp);
   format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $scratch as TEMP" %}
@@ -6974,7 +6966,7 @@ instruct evcmp(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg ktmp
     bool is_unsigned = is_unsigned_booltest_pred($cond$$constant);
     KRegister mask = k0; // The comparison itself is not being masked.
     bool merge = false;
-    BasicType src1_elem_bt = vector_element_basic_type(this, $src1);
+    BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
 
     switch (src1_elem_bt) {
       case T_BYTE: {
@@ -7006,7 +6998,7 @@ instruct evcmp(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg ktmp
 // Extract
 
 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
-  predicate(vector_length_in_bytes(n->in(1)) <= 16); // src
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
   match(Set dst (ExtractI src idx));
   match(Set dst (ExtractS src idx));
 #ifdef _LP64
@@ -7014,17 +7006,17 @@ instruct extractI(rRegI dst, legVec src, immU8 idx) %{
 #endif
   format %{ "extractI $dst,$src,$idx\t!" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
-    BasicType elem_bt = vector_element_basic_type(this, $src);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
     __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
-  predicate(vector_length_in_bytes(n->in(1)) == 32 || // src
-            vector_length_in_bytes(n->in(1)) == 64);  // src
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
+            Matcher::vector_length_in_bytes(n->in(1)) == 64);  // src
   match(Set dst (ExtractI src idx));
   match(Set dst (ExtractS src idx));
 #ifdef _LP64
@@ -7033,9 +7025,9 @@ instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
   effect(TEMP vtmp);
   format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
-    BasicType elem_bt = vector_element_basic_type(this, $src);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
     XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
     __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
   %}
@@ -7044,12 +7036,12 @@ instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
 
 #ifdef _LP64
 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
-  predicate(vector_length(n->in(1)) <= 2); // src
+  predicate(Matcher::vector_length(n->in(1)) <= 2); // src
   match(Set dst (ExtractL src idx));
   format %{ "extractL $dst,$src,$idx\t!" %}
   ins_encode %{
     assert(UseSSE >= 4, "required");
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
   %}
@@ -7057,13 +7049,13 @@ instruct extractL(rRegL dst, legVec src, immU8 idx) %{
 %}
 
 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
-  predicate(vector_length(n->in(1)) == 4 || // src
-            vector_length(n->in(1)) == 8);  // src
+  predicate(Matcher::vector_length(n->in(1)) == 4 || // src
+            Matcher::vector_length(n->in(1)) == 8);  // src
   match(Set dst (ExtractL src idx));
   effect(TEMP vtmp);
   format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
     __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
@@ -7073,12 +7065,12 @@ instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
 #endif
 
 instruct extractF(legRegF dst, legVec src, immU8 idx, rRegI tmp, legVec vtmp) %{
-  predicate(vector_length(n->in(1)) <= 4);
+  predicate(Matcher::vector_length(n->in(1)) <= 4);
   match(Set dst (ExtractF src idx));
   effect(TEMP dst, TEMP tmp, TEMP vtmp);
   format %{ "extractF $dst,$src,$idx\t! using $tmp, $vtmp as TEMP" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $tmp$$Register, $vtmp$$XMMRegister);
   %}
@@ -7086,13 +7078,13 @@ instruct extractF(legRegF dst, legVec src, immU8 idx, rRegI tmp, legVec vtmp) %{
 %}
 
 instruct vextractF(legRegF dst, legVec src, immU8 idx, rRegI tmp, legVec vtmp) %{
-  predicate(vector_length(n->in(1)/*src*/) == 8 ||
-            vector_length(n->in(1)/*src*/) == 16);
+  predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
+            Matcher::vector_length(n->in(1)/*src*/) == 16);
   match(Set dst (ExtractF src idx));
   effect(TEMP tmp, TEMP vtmp);
   format %{ "vextractF $dst,$src,$idx\t! using $tmp, $vtmp as TEMP" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
     __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant, $tmp$$Register);
@@ -7101,11 +7093,11 @@ instruct vextractF(legRegF dst, legVec src, immU8 idx, rRegI tmp, legVec vtmp) %
 %}
 
 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
-  predicate(vector_length(n->in(1)) == 2); // src
+  predicate(Matcher::vector_length(n->in(1)) == 2); // src
   match(Set dst (ExtractD src idx));
   format %{ "extractD $dst,$src,$idx\t!" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
   %}
@@ -7113,13 +7105,13 @@ instruct extractD(legRegD dst, legVec src, immU8 idx) %{
 %}
 
 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
-  predicate(vector_length(n->in(1)) == 4 || // src
-            vector_length(n->in(1)) == 8);  // src
+  predicate(Matcher::vector_length(n->in(1)) == 4 || // src
+            Matcher::vector_length(n->in(1)) == 8);  // src
   match(Set dst (ExtractD src idx));
   effect(TEMP vtmp);
   format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
   ins_encode %{
-    assert($idx$$constant < (int)vector_length(this, $src), "out of bounds");
+    assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
 
     XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
     __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
@@ -7147,8 +7139,8 @@ instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
 
 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
   predicate(UseAVX > 0 &&
-            vector_length_in_bytes(n) <= 32 &&
-            is_integral_type(vector_element_basic_type(n)));
+            Matcher::vector_length_in_bytes(n) <= 32 &&
+            is_integral_type(Matcher::vector_element_basic_type(n)));
   match(Set dst (VectorBlend (Binary src1 src2) mask));
   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
   ins_encode %{
@@ -7160,8 +7152,8 @@ instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
 
 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
   predicate(UseAVX > 0 &&
-            vector_length_in_bytes(n) <= 32 &&
-            !is_integral_type(vector_element_basic_type(n)));
+            Matcher::vector_length_in_bytes(n) <= 32 &&
+            !is_integral_type(Matcher::vector_element_basic_type(n)));
   match(Set dst (VectorBlend (Binary src1 src2) mask));
   format %{ "vector_blend  $dst,$src1,$src2,$mask\t!" %}
   ins_encode %{
@@ -7172,13 +7164,13 @@ instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
 %}
 
 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, rRegP scratch, kReg ktmp) %{
-  predicate(vector_length_in_bytes(n) == 64);
+  predicate(Matcher::vector_length_in_bytes(n) == 64);
   match(Set dst (VectorBlend (Binary src1 src2) mask));
   format %{ "vector_blend  $dst,$src1,$src2,$mask\t! using $scratch and k2 as TEMP" %}
   effect(TEMP scratch, TEMP ktmp);
   ins_encode %{
      int vlen_enc = Assembler::AVX_512bit;
-     BasicType elem_bt = vector_element_basic_type(this);
+     BasicType elem_bt = Matcher::vector_element_basic_type(this);
     __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, $scratch$$Register);
     __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
   %}
@@ -7191,7 +7183,7 @@ instruct vabsB_reg(vec dst, vec src) %{
   match(Set dst (AbsVB  src));
   format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 16) {
       __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
     } else {
@@ -7206,7 +7198,7 @@ instruct vabsS_reg(vec dst, vec src) %{
   match(Set dst (AbsVS  src));
   format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 8) {
       __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
     } else {
@@ -7221,7 +7213,7 @@ instruct vabsI_reg(vec dst, vec src) %{
   match(Set dst (AbsVI  src));
   format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
   ins_encode %{
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen <= 4) {
       __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
     } else {
@@ -7249,7 +7241,7 @@ instruct vabsL_reg(vec dst, vec src) %{
 // --------------------------------- ABSNEG --------------------------------------
 
 instruct vabsnegF(vec dst, vec src, rRegI scratch) %{
-  predicate(vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
+  predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
   match(Set dst (AbsVF src));
   match(Set dst (NegVF src));
   effect(TEMP scratch);
@@ -7257,7 +7249,7 @@ instruct vabsnegF(vec dst, vec src, rRegI scratch) %{
   ins_cost(150);
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    int vlen = vector_length(this);
+    int vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, $scratch$$Register);
     } else {
@@ -7270,7 +7262,7 @@ instruct vabsnegF(vec dst, vec src, rRegI scratch) %{
 %}
 
 instruct vabsneg4F(vec dst, rRegI scratch) %{
-  predicate(vector_length(n) == 4);
+  predicate(Matcher::vector_length(n) == 4);
   match(Set dst (AbsVF dst));
   match(Set dst (NegVF dst));
   effect(TEMP scratch);
@@ -7290,7 +7282,7 @@ instruct vabsnegD(vec dst, vec src, rRegI scratch) %{
   format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
   ins_encode %{
     int opcode = this->ideal_Opcode();
-    uint vlen = vector_length(this);
+    uint vlen = Matcher::vector_length(this);
     if (vlen == 2) {
       assert(UseSSE >= 2, "required");
       __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $scratch$$Register);
@@ -7306,14 +7298,14 @@ instruct vabsnegD(vec dst, vec src, rRegI scratch) %{
 
 #ifdef _LP64
 instruct vptest_alltrue_lt16(rRegI dst, legVec src1, legVec src2, legVec vtmp1, legVec vtmp2, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) >= 4 &&
-            vector_length_in_bytes(n->in(1)) < 16 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 4 &&
+            Matcher::vector_length_in_bytes(n->in(1)) < 16 &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
   match(Set dst (VectorTest src1 src2 ));
   effect(TEMP vtmp1, TEMP vtmp2, KILL cr);
   format %{ "vector_test $dst,$src1, $src2\t! using $vtmp1, $vtmp2 and $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::overflow, vlen, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
     __ setb(Assembler::carrySet, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7322,14 +7314,14 @@ instruct vptest_alltrue_lt16(rRegI dst, legVec src1, legVec src2, legVec vtmp1,
 %}
 
 instruct vptest_alltrue(rRegI dst, legVec src1, legVec src2, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) >= 16 &&
-            vector_length_in_bytes(n->in(1)) <  64 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16 &&
+            Matcher::vector_length_in_bytes(n->in(1)) <  64 &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
   match(Set dst (VectorTest src1 src2 ));
   effect(KILL cr);
   format %{ "vector_test $dst,$src1, $src2\t! using $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::overflow, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, knoreg);
     __ setb(Assembler::carrySet, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7338,13 +7330,13 @@ instruct vptest_alltrue(rRegI dst, legVec src1, legVec src2, rFlagsReg cr) %{
 %}
 
 instruct vptest_alltrue_evex(rRegI dst, legVec src1, legVec src2, kReg ktmp, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) == 64 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) == 64 &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
   match(Set dst (VectorTest src1 src2 ));
   effect(KILL cr, TEMP ktmp);
   format %{ "vector_test $dst,$src1, $src2\t! using $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::overflow, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, $ktmp$$KRegister);
     __ setb(Assembler::carrySet, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7353,14 +7345,14 @@ instruct vptest_alltrue_evex(rRegI dst, legVec src1, legVec src2, kReg ktmp, rFl
 %}
 
 instruct vptest_anytrue_lt16(rRegI dst, legVec src1, legVec src2, legVec vtmp, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) >= 4 &&
-            vector_length_in_bytes(n->in(1)) < 16 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 4 &&
+            Matcher::vector_length_in_bytes(n->in(1)) < 16 &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
   match(Set dst (VectorTest src1 src2 ));
   effect(TEMP vtmp, KILL cr);
   format %{ "vector_test_any_true $dst,$src1,$src2\t! using $vtmp, $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
     __ setb(Assembler::notZero, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7369,14 +7361,14 @@ instruct vptest_anytrue_lt16(rRegI dst, legVec src1, legVec src2, legVec vtmp, r
 %}
 
 instruct vptest_anytrue(rRegI dst, legVec src1, legVec src2, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) >= 16 &&
-            vector_length_in_bytes(n->in(1)) < 64  &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16 &&
+            Matcher::vector_length_in_bytes(n->in(1)) < 64  &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
   match(Set dst (VectorTest src1 src2 ));
   effect(KILL cr);
   format %{ "vector_test_any_true $dst,$src1,$src2\t! using $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, knoreg);
     __ setb(Assembler::notZero, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7385,13 +7377,13 @@ instruct vptest_anytrue(rRegI dst, legVec src1, legVec src2, rFlagsReg cr) %{
 %}
 
 instruct vptest_anytrue_evex(rRegI dst, legVec src1, legVec src2, kReg ktmp, rFlagsReg cr) %{
-  predicate(vector_length_in_bytes(n->in(1)) == 64 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) == 64 &&
             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
   match(Set dst (VectorTest src1 src2 ));
   effect(KILL cr, TEMP ktmp);
   format %{ "vector_test_any_true $dst,$src1,$src2\t! using $cr as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, $ktmp$$KRegister);
     __ setb(Assembler::notZero, $dst$$Register);
     __ movzbl($dst$$Register, $dst$$Register);
@@ -7400,40 +7392,40 @@ instruct vptest_anytrue_evex(rRegI dst, legVec src1, legVec src2, kReg ktmp, rFl
 %}
 
 instruct cmpvptest_anytrue_lt16(rFlagsReg cr, legVec src1, legVec src2, immI_0 zero, legVec vtmp) %{
-  predicate(vector_length_in_bytes(n->in(1)->in(1)) >= 4 &&
-            vector_length_in_bytes(n->in(1)->in(1)) < 16 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 &&
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) < 16 &&
             static_cast<const VectorTestNode*>(n->in(1))->get_predicate() == BoolTest::ne);
   match(Set cr (CmpI (VectorTest src1 src2) zero));
   effect(TEMP vtmp);
   format %{ "cmp_vector_test_any_true $src1,$src2\t! using $vtmp as TEMP" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct cmpvptest_anytrue(rFlagsReg cr, legVec src1, legVec src2, immI_0 zero) %{
-  predicate(vector_length_in_bytes(n->in(1)->in(1)) >= 16 &&
-            vector_length_in_bytes(n->in(1)->in(1)) <  64 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 16 &&
+            Matcher::vector_length_in_bytes(n->in(1)->in(1)) <  64 &&
             static_cast<const VectorTestNode*>(n->in(1))->get_predicate() == BoolTest::ne);
   match(Set cr (CmpI (VectorTest src1 src2) zero));
   format %{ "cmp_vector_test_any_true $src1,$src2\t!" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, knoreg);
   %}
   ins_pipe( pipe_slow );
 %}
 
 instruct cmpvptest_anytrue_evex(rFlagsReg cr, legVec src1, legVec src2, immI_0 zero, kReg ktmp) %{
-  predicate(vector_length_in_bytes(n->in(1)->in(1)) == 64 &&
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 &&
             static_cast<const VectorTestNode*>(n->in(1))->get_predicate() == BoolTest::ne);
   match(Set cr (CmpI (VectorTest src1 src2) zero));
   effect(TEMP ktmp);
   format %{ "cmp_vector_test_any_true $src1,$src2\t!" %}
   ins_encode %{
-    int vlen = vector_length_in_bytes(this, $src1);
+    int vlen = Matcher::vector_length_in_bytes(this, $src1);
     __ vectortest(BoolTest::ne, vlen, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg, $ktmp$$KRegister);
   %}
   ins_pipe( pipe_slow );
@@ -7442,15 +7434,30 @@ instruct cmpvptest_anytrue_evex(rFlagsReg cr, legVec src1, legVec src2, immI_0 z
 
 //------------------------------------- LoadMask --------------------------------------------
 
-instruct loadMask(vec dst, vec src) %{
+instruct loadMask(legVec dst, legVec src) %{
+  predicate(!VM_Version::supports_avx512vlbw());
+  match(Set dst (VectorLoadMask src));
+  effect(TEMP dst);
+  format %{ "vector_loadmask_byte $dst,$src\n\t" %}
+  ins_encode %{
+    int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
+
+    __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
+  %}
+  ins_pipe( pipe_slow );
+%}
+
+instruct loadMask_evex(vec dst, vec src) %{
+  predicate(VM_Version::supports_avx512vlbw());
   match(Set dst (VectorLoadMask src));
   effect(TEMP dst);
   format %{ "vector_loadmask_byte $dst,$src\n\t" %}
   ins_encode %{
-    int vlen_in_bytes = vector_length_in_bytes(this);
-    BasicType elem_bt = vector_element_basic_type(this);
+    int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this);
 
-    __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt);
+    __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, false);
   %}
   ins_pipe( pipe_slow );
 %}
@@ -7458,12 +7465,12 @@ instruct loadMask(vec dst, vec src) %{
 //------------------------------------- StoreMask --------------------------------------------
 
 instruct storeMask1B(vec dst, vec src, immI_1 size) %{
-  predicate(vector_length(n) < 64 || VM_Version::supports_avx512vlbw());
+  predicate(Matcher::vector_length(n) < 64 || VM_Version::supports_avx512vlbw());
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\t!" %}
   ins_encode %{
     assert(UseSSE >= 3, "required");
-    if (vector_length_in_bytes(this) <= 16) {
+    if (Matcher::vector_length_in_bytes(this) <= 16) {
       __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
     } else {
       assert(UseAVX >= 2, "required");
@@ -7475,7 +7482,7 @@ instruct storeMask1B(vec dst, vec src, immI_1 size) %{
 %}
 
 instruct storeMask2B(vec dst, vec src, immI_2 size) %{
-  predicate(vector_length(n) <= 8);
+  predicate(Matcher::vector_length(n) <= 8);
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\n\t" %}
   ins_encode %{
@@ -7487,7 +7494,7 @@ instruct storeMask2B(vec dst, vec src, immI_2 size) %{
 %}
 
 instruct vstoreMask2B(vec dst, vec src, immI_2 size) %{
-  predicate(vector_length(n) == 16 && !VM_Version::supports_avx512bw());
+  predicate(Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
   match(Set dst (VectorStoreMask src size));
   effect(TEMP dst);
   format %{ "vector_store_mask $dst,$src\t!" %}
@@ -7514,7 +7521,7 @@ instruct vstoreMask2B_evex(vec dst, vec src, immI_2 size) %{
 %}
 
 instruct storeMask4B(vec dst, vec src, immI_4 size) %{
-  predicate (vector_length(n) <= 4 && UseAVX <= 2);
+  predicate(Matcher::vector_length(n) <= 4 && UseAVX <= 2);
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\t!" %}
   ins_encode %{
@@ -7527,7 +7534,7 @@ instruct storeMask4B(vec dst, vec src, immI_4 size) %{
 %}
 
 instruct vstoreMask4B(vec dst, vec src, immI_4 size) %{
-  predicate(vector_length(n) == 8 && UseAVX <= 2);
+  predicate(Matcher::vector_length(n) == 8 && UseAVX <= 2);
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\t!" %}
   effect(TEMP dst);
@@ -7558,7 +7565,7 @@ instruct vstoreMask4B_evex(vec dst, vec src, immI_4 size) %{
 %}
 
 instruct storeMask8B(vec dst, vec src, immI_8 size) %{
-  predicate(vector_length(n) == 2 && UseAVX <= 2);
+  predicate(Matcher::vector_length(n) == 2 && UseAVX <= 2);
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\t!" %}
   ins_encode %{
@@ -7572,7 +7579,7 @@ instruct storeMask8B(vec dst, vec src, immI_8 size) %{
 %}
 
 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, legVec vtmp) %{
-  predicate(vector_length(n) == 4 && UseAVX <= 2);
+  predicate(Matcher::vector_length(n) == 4 && UseAVX <= 2);
   match(Set dst (VectorStoreMask src size));
   format %{ "vector_store_mask $dst,$src\t! using $vtmp as TEMP" %}
   effect(TEMP dst, TEMP vtmp);
@@ -7605,8 +7612,8 @@ instruct vstoreMask8B_evex(vec dst, vec src, immI_8 size) %{
 %}
 
 instruct vmaskcast(vec dst) %{
-  predicate((vector_length(n) == vector_length(n->in(1))) &&
-            (vector_length_in_bytes(n) == vector_length_in_bytes(n->in(1))));
+  predicate((Matcher::vector_length(n) == Matcher::vector_length(n->in(1))) &&
+            (Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))));
   match(Set dst (VectorMaskCast dst));
   ins_cost(0);
   format %{ "vector_mask_cast $dst" %}
@@ -7619,12 +7626,12 @@ instruct vmaskcast(vec dst) %{
 //-------------------------------- Load Iota Indices ----------------------------------
 
 instruct loadIotaIndices(vec dst, immI_0 src, rRegP scratch) %{
-  predicate(vector_element_basic_type(n) == T_BYTE);
+  predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
   match(Set dst (VectorLoadConst src));
   effect(TEMP scratch);
   format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
   ins_encode %{
-     int vlen_in_bytes = vector_length_in_bytes(this);
+     int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
      __ load_iota_indices($dst$$XMMRegister, $scratch$$Register, vlen_in_bytes);
   %}
   ins_pipe( pipe_slow );
@@ -7635,7 +7642,7 @@ instruct loadIotaIndices(vec dst, immI_0 src, rRegP scratch) %{
 // LoadShuffle/Rearrange for Byte
 
 instruct loadShuffleB(vec dst) %{
-  predicate(vector_element_basic_type(n) == T_BYTE);
+  predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
   match(Set dst (VectorLoadShuffle dst));
   format %{ "vector_load_shuffle $dst, $dst" %}
   ins_encode %{
@@ -7645,8 +7652,8 @@ instruct loadShuffleB(vec dst) %{
 %}
 
 instruct rearrangeB(vec dst, vec shuffle) %{
-  predicate(vector_element_basic_type(n) == T_BYTE &&
-            vector_length(n) < 32);
+  predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
+            Matcher::vector_length(n) < 32);
   match(Set dst (VectorRearrange dst shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $dst" %}
   ins_encode %{
@@ -7657,8 +7664,8 @@ instruct rearrangeB(vec dst, vec shuffle) %{
 %}
 
 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2, rRegP scratch) %{
-  predicate(vector_element_basic_type(n) == T_BYTE &&
-            vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
+  predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
+            Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
   match(Set dst (VectorRearrange src shuffle));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP scratch);
   format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2, $scratch as TEMP" %}
@@ -7679,8 +7686,8 @@ instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVe
 %}
 
 instruct rearrangeB_evex(vec dst, vec src, vec shuffle) %{
-  predicate(vector_element_basic_type(n) == T_BYTE &&
-            vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
+  predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
+            Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
   match(Set dst (VectorRearrange src shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $src" %}
   ins_encode %{
@@ -7693,15 +7700,15 @@ instruct rearrangeB_evex(vec dst, vec src, vec shuffle) %{
 // LoadShuffle/Rearrange for Short
 
 instruct loadShuffleS(vec dst, vec src, vec vtmp, rRegP scratch) %{
-  predicate(vector_element_basic_type(n) == T_SHORT &&
-            vector_length(n) <= 16 && !VM_Version::supports_avx512bw()); // NB! aligned with rearrangeS
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
+            Matcher::vector_length(n) <= 16 && !VM_Version::supports_avx512bw()); // NB! aligned with rearrangeS
   match(Set dst (VectorLoadShuffle src));
   effect(TEMP dst, TEMP vtmp, TEMP scratch);
   format %{ "vector_load_shuffle $dst, $src\t! using $vtmp and $scratch as TEMP" %}
   ins_encode %{
     // Create a byte shuffle mask from short shuffle mask
     // only byte shuffle instruction available on these platforms
-    int vlen_in_bytes = vector_length_in_bytes(this);
+    int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
     if (UseAVX == 0) {
       assert(vlen_in_bytes <= 16, "required");
       // Multiply each shuffle by two to get byte index
@@ -7735,8 +7742,8 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp, rRegP scratch) %{
 %}
 
 instruct rearrangeS(vec dst, vec shuffle) %{
-  predicate(vector_element_basic_type(n) == T_SHORT &&
-            vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
+            Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
   match(Set dst (VectorRearrange dst shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $dst" %}
   ins_encode %{
@@ -7747,8 +7754,8 @@ instruct rearrangeS(vec dst, vec shuffle) %{
 %}
 
 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2, rRegP scratch) %{
-  predicate(vector_element_basic_type(n) == T_SHORT &&
-            vector_length(n) == 16 && !VM_Version::supports_avx512bw());
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
+            Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
   match(Set dst (VectorRearrange src shuffle));
   effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP scratch);
   format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2, $scratch as TEMP" %}
@@ -7769,7 +7776,7 @@ instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVe
 %}
 
 instruct loadShuffleS_evex(vec dst, vec src) %{
-  predicate(vector_element_basic_type(n) == T_SHORT &&
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
             VM_Version::supports_avx512bw());
   match(Set dst (VectorLoadShuffle src));
   format %{ "vector_load_shuffle $dst, $src" %}
@@ -7784,7 +7791,7 @@ instruct loadShuffleS_evex(vec dst, vec src) %{
 %}
 
 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
-  predicate(vector_element_basic_type(n) == T_SHORT &&
+  predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
             VM_Version::supports_avx512bw());
   match(Set dst (VectorRearrange src shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $src" %}
@@ -7801,8 +7808,8 @@ instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
 // LoadShuffle/Rearrange for Integer and Float
 
 instruct loadShuffleI(vec dst, vec src, vec vtmp, rRegP scratch) %{
-  predicate((vector_element_basic_type(n) == T_INT || vector_element_basic_type(n) == T_FLOAT) &&
-            vector_length(n) == 4 && UseAVX < 2);
+  predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
+            Matcher::vector_length(n) == 4 && UseAVX < 2);
   match(Set dst (VectorLoadShuffle src));
   effect(TEMP dst, TEMP vtmp, TEMP scratch);
   format %{ "vector_load_shuffle $dst, $src\t! using $vtmp and $scratch as TEMP" %}
@@ -7831,8 +7838,8 @@ instruct loadShuffleI(vec dst, vec src, vec vtmp, rRegP scratch) %{
 %}
 
 instruct rearrangeI(vec dst, vec shuffle) %{
- predicate((vector_element_basic_type(n) == T_INT || vector_element_basic_type(n) == T_FLOAT) &&
-           vector_length(n) == 4 && UseAVX < 2);
+ predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
+           Matcher::vector_length(n) == 4 && UseAVX < 2);
   match(Set dst (VectorRearrange dst shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $dst" %}
   ins_encode %{
@@ -7843,7 +7850,7 @@ instruct rearrangeI(vec dst, vec shuffle) %{
 %}
 
 instruct loadShuffleI_avx(vec dst, vec src) %{
-  predicate((vector_element_basic_type(n) == T_INT || vector_element_basic_type(n) == T_FLOAT) &&
+  predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
             UseAVX >= 2);
   match(Set dst (VectorLoadShuffle src));
   format %{ "vector_load_shuffle $dst, $src" %}
@@ -7855,7 +7862,7 @@ instruct loadShuffleI_avx(vec dst, vec src) %{
 %}
 
 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
-  predicate((vector_element_basic_type(n) == T_INT || vector_element_basic_type(n) == T_FLOAT) &&
+  predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
             UseAVX >= 2);
   match(Set dst (VectorRearrange src shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $src" %}
@@ -7872,8 +7879,8 @@ instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
 // LoadShuffle/Rearrange for Long and Double
 
 instruct loadShuffleL(vec dst, vec src, vec vtmp, rRegP scratch) %{
-  predicate(is_double_word_type(vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
-            vector_length(n) < 8 && !VM_Version::supports_avx512vl());
+  predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
+            Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
   match(Set dst (VectorLoadShuffle src));
   effect(TEMP dst, TEMP vtmp, TEMP scratch);
   format %{ "vector_load_shuffle $dst, $src\t! using $vtmp and $scratch as TEMP" %}
@@ -7899,8 +7906,8 @@ instruct loadShuffleL(vec dst, vec src, vec vtmp, rRegP scratch) %{
 %}
 
 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
-  predicate(is_double_word_type(vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
-            vector_length(n) < 8 && !VM_Version::supports_avx512vl());
+  predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
+            Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
   match(Set dst (VectorRearrange src shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $src" %}
   ins_encode %{
@@ -7913,8 +7920,8 @@ instruct rearrangeL(vec dst, vec src, vec shuffle) %{
 %}
 
 instruct loadShuffleL_evex(vec dst, vec src) %{
-  predicate(is_double_word_type(vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
-            (vector_length(n) == 8 || VM_Version::supports_avx512vl()));
+  predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
+            (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
   match(Set dst (VectorLoadShuffle src));
   format %{ "vector_load_shuffle $dst, $src" %}
   ins_encode %{
@@ -7927,8 +7934,8 @@ instruct loadShuffleL_evex(vec dst, vec src) %{
 %}
 
 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
-  predicate(is_double_word_type(vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
-            (vector_length(n) == 8 || VM_Version::supports_avx512vl()));
+  predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
+            (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
   match(Set dst (VectorRearrange src shuffle));
   format %{ "vector_rearrange $dst, $shuffle, $src" %}
   ins_encode %{
@@ -7959,6 +7966,7 @@ instruct vfmaF_reg(vec a, vec b, vec c) %{
 %}
 
 instruct vfmaF_mem(vec a, memory b, vec c) %{
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
   match(Set c (FmaVF  c (Binary a (LoadVector b))));
   format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
   ins_cost(150);
@@ -7983,6 +7991,7 @@ instruct vfmaD_reg(vec a, vec b, vec c) %{
 %}
 
 instruct vfmaD_mem(vec a, memory b, vec c) %{
+  predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
   match(Set c (FmaVD  c (Binary a (LoadVector b))));
   format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
   ins_cost(150);
@@ -8060,6 +8069,7 @@ instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
 %}
 
 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
+  predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
   match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
   effect(TEMP dst);
   format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
@@ -8106,11 +8116,11 @@ instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kR
   format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
   ins_encode %{
     assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
-    assert(vector_element_basic_type(this, $src1) == vector_element_basic_type(this, $src2), "mismatch");
+    assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
 
     Label DONE;
     int vlen_enc = vector_length_encoding(this, $src1);
-    BasicType elem_bt = vector_element_basic_type(this, $src1);
+    BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
 
     __ knotql($ktmp2$$KRegister, $mask$$KRegister);
     __ mov64($dst$$Register, -1L);
@@ -8178,7 +8188,7 @@ instruct vmask_truecount_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp, vec xtm
   ins_encode %{
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this, $mask);
-    int mask_len = vector_length(this, $mask);
+    int mask_len = Matcher::vector_length(this, $mask);
     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
                              $tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc);
   %}
@@ -8194,7 +8204,7 @@ instruct vmask_first_or_last_true_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp
   ins_encode %{
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this, $mask);
-    int mask_len = vector_length(this, $mask);
+    int mask_len = Matcher::vector_length(this, $mask);
     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
                              $tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc);
   %}
@@ -8209,7 +8219,7 @@ instruct vmask_truecount_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1
   ins_encode %{
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this, $mask);
-    int mask_len = vector_length(this, $mask);
+    int mask_len = Matcher::vector_length(this, $mask);
     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
                              $xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc);
   %}
@@ -8225,7 +8235,7 @@ instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp,
   ins_encode %{
     int opcode = this->ideal_Opcode();
     int vlen_enc = vector_length_encoding(this, $mask);
-    int mask_len = vector_length(this, $mask);
+    int mask_len = Matcher::vector_length(this, $mask);
     __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
                              $xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc);
   %}
diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad
index 6e51926161a..96c0e869576 100644
--- a/src/hotspot/cpu/x86/x86_32.ad
+++ b/src/hotspot/cpu/x86/x86_32.ad
@@ -1437,6 +1437,16 @@ bool Matcher::is_spillable_arg( int reg ) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 6 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 6 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   // Use hardware integer DIV instruction when
   // it is faster than a code which use multiply.
@@ -5008,6 +5018,89 @@ define %{
 //               name must have been defined in an 'enc_class' specification
 //               in the encode section of the architecture description.
 
+// Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
+// Load Float
+instruct MoveF2LEG(legRegF dst, regF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveLEG2F(regF dst, legRegF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveF2VL(vlRegF dst, regF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveVL2F(regF dst, vlRegF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+
+
+// Load Double
+instruct MoveD2LEG(legRegD dst, regD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveLEG2D(regD dst, legRegD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveD2VL(vlRegD dst, regD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveVL2D(regD dst, vlRegD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
 //----------BSWAP-Instruction--------------------------------------------------
 instruct bytes_reverse_int(rRegI dst) %{
   match(Set dst (ReverseBytesI dst));
@@ -5753,46 +5846,6 @@ instruct loadKlass(eRegP dst, memory mem) %{
   ins_pipe( ialu_reg_mem );
 %}
 
-// Load Float
-instruct MoveF2LEG(legRegF dst, regF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveLEG2F(regF dst, legRegF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveD2LEG(legRegD dst, regD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveLEG2D(regD dst, legRegD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
 // Load Double
 instruct loadDPR(regDPR dst, memory mem) %{
   predicate(UseSSE<=1);
@@ -6422,26 +6475,6 @@ instruct storeD(memory mem, regD src) %{
   ins_pipe( pipe_slow );
 %}
 
-// Load Double
-instruct MoveD2VL(vlRegD dst, regD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveVL2D(regD dst, vlRegD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
 // Store XMM register to memory (single-precision floating point)
 // MOVSS instruction
 instruct storeF(memory mem, regF src) %{
@@ -6455,25 +6488,6 @@ instruct storeF(memory mem, regF src) %{
   ins_pipe( pipe_slow );
 %}
 
-// Load Float
-instruct MoveF2VL(vlRegF dst, regF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveVL2F(regF dst, vlRegF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
 
 // Store Float
 instruct storeFPR( memory mem, regFPR1 src) %{
@@ -11467,8 +11481,7 @@ instruct MoveL2D_reg_reg_sse(regD dst, eRegL src, regD tmp) %{
 // fast clearing of an array
 // Small ClearArray non-AVX512.
 instruct rep_stos(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
-  predicate(!((ClearArrayNode*)n)->is_large() &&
-              (UseAVX <= 2 || !VM_Version::supports_avx512vlbw()));
+  predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
   match(Set dummy (ClearArray cnt base));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
 
@@ -11528,10 +11541,9 @@ instruct rep_stos(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Universe du
 
 // Small ClearArray AVX512 non-constant length.
 instruct rep_stos_evex(eCXRegI cnt, eDIRegP base, regD tmp, kReg ktmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
-  predicate(!((ClearArrayNode*)n)->is_large() &&
-               UseAVX > 2 && VM_Version::supports_avx512vlbw() &&
-               !n->in(2)->bottom_type()->is_int()->is_con());
+  predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
   match(Set dummy (ClearArray cnt base));
+  ins_cost(125);
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
 
   format %{ $$template
@@ -11590,7 +11602,7 @@ instruct rep_stos_evex(eCXRegI cnt, eDIRegP base, regD tmp, kReg ktmp, eAXRegI z
 
 // Large ClearArray non-AVX512.
 instruct rep_stos_large(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
-  predicate(UseAVX <= 2 && ((ClearArrayNode*)n)->is_large());
+  predicate((UseAVX <= 2) && ((ClearArrayNode*)n)->is_large());
   match(Set dummy (ClearArray cnt base));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
   format %{ $$template
@@ -11640,7 +11652,7 @@ instruct rep_stos_large(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Unive
 
 // Large ClearArray AVX512.
 instruct rep_stos_large_evex(eCXRegI cnt, eDIRegP base, regD tmp, kReg ktmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
-  predicate(UseAVX > 2 && ((ClearArrayNode*)n)->is_large());
+  predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
   match(Set dummy (ClearArray cnt base));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
   format %{ $$template
@@ -11692,9 +11704,9 @@ instruct rep_stos_large_evex(eCXRegI cnt, eDIRegP base, regD tmp, kReg ktmp, eAX
 instruct rep_stos_im(immI cnt, kReg ktmp, eRegP base, regD tmp, rRegI zero, Universe dummy, eFlagsReg cr)
 %{
   predicate(!((ClearArrayNode*)n)->is_large() &&
-               (UseAVX > 2 && VM_Version::supports_avx512vlbw() &&
-                 n->in(2)->bottom_type()->is_int()->is_con()));
+               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
   match(Set dummy (ClearArray cnt base));
+  ins_cost(100);
   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
   ins_encode %{
@@ -13664,7 +13676,7 @@ instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eD
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
                  $scr$$Register, $cx1$$Register, $cx2$$Register,
-                 _counters, _rtm_counters, _stack_rtm_counters,
+                 _rtm_counters, _stack_rtm_counters,
                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
                  true, ra_->C->profile_rtm());
   %}
@@ -13679,7 +13691,7 @@ instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP
   format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr" %}
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
+                 $scr$$Register, noreg, noreg, NULL, NULL, NULL, false, false);
   %}
   ins_pipe(pipe_slow);
 %}
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 7a10c44a050..3422e836261 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -322,6 +322,7 @@ extern RegMask _LONG_NO_RCX_REG_mask;
 extern RegMask _INT_REG_mask;
 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 extern RegMask _INT_NO_RCX_REG_mask;
+extern RegMask _FLOAT_REG_mask;
 
 extern RegMask _STACK_OR_PTR_REG_mask;
 extern RegMask _STACK_OR_LONG_REG_mask;
@@ -350,6 +351,7 @@ RegMask _LONG_NO_RCX_REG_mask;
 RegMask _INT_REG_mask;
 RegMask _INT_NO_RAX_RDX_REG_mask;
 RegMask _INT_NO_RCX_REG_mask;
+RegMask _FLOAT_REG_mask;
 RegMask _STACK_OR_PTR_REG_mask;
 RegMask _STACK_OR_LONG_REG_mask;
 RegMask _STACK_OR_INT_REG_mask;
@@ -425,6 +427,10 @@ void reg_mask_init() {
   _INT_NO_RCX_REG_mask = _INT_REG_mask;
   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 
+  // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
+  // from the float_reg_legacy/float_reg_evex register class.
+  _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
+
   if (Matcher::has_predicated_vectors()) {
     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
@@ -1757,6 +1763,20 @@ bool Matcher::is_spillable_arg(int reg)
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  // After experiment around with different values, the following default threshold
+  // works best for LCM's register pressure scheduling on x64.
+  uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
+  uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
+  return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   // In 64 bit mode a code which use multiply when
   // devisor is constant is faster than hardware
@@ -4815,6 +4835,87 @@ define
 //               name must have been defined in an 'enc_class' specification
 //               in the encode section of the architecture description.
 
+// Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
+// Load Float
+instruct MoveF2VL(vlRegF dst, regF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveF2LEG(legRegF dst, regF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveVL2F(regF dst, vlRegF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Float
+instruct MoveLEG2F(regF dst, legRegF src) %{
+  match(Set dst src);
+  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveD2VL(vlRegD dst, regD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveD2LEG(legRegD dst, regD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveVL2D(regD dst, vlRegD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
+// Load Double
+instruct MoveLEG2D(regD dst, legRegD src) %{
+  match(Set dst src);
+  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
+  ins_encode %{
+    ShouldNotReachHere();
+  %}
+  ins_pipe( fpu_reg_reg );
+%}
+
 //----------Load/Store/Move Instructions---------------------------------------
 //----------Load Instructions--------------------------------------------------
 
@@ -5228,46 +5329,6 @@ instruct loadF(regF dst, memory mem)
   ins_pipe(pipe_slow); // XXX
 %}
 
-// Load Float
-instruct MoveF2VL(vlRegF dst, regF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveF2LEG(legRegF dst, regF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveVL2F(regF dst, vlRegF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t! load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveLEG2F(regF dst, legRegF src) %{
-  match(Set dst src);
-  format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
-  ins_encode %{
-    __ movflt($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
 // Load Double
 instruct loadD_partial(regD dst, memory mem)
 %{
@@ -5295,45 +5356,6 @@ instruct loadD(regD dst, memory mem)
   ins_pipe(pipe_slow); // XXX
 %}
 
-// Load Double
-instruct MoveD2VL(vlRegD dst, regD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveD2LEG(legRegD dst, regD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveVL2D(regD dst, vlRegD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveLEG2D(regD dst, legRegD src) %{
-  match(Set dst src);
-  format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
-  ins_encode %{
-    __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
-  %}
-  ins_pipe( fpu_reg_reg );
-%}
 
 // Following pseudo code describes the algorithm for max[FD]:
 // Min algorithm is on similar lines
@@ -11052,8 +11074,7 @@ instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
                   Universe dummy, rFlagsReg cr)
 %{
-  predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
-            (UseAVX <= 2 || !VM_Version::supports_avx512vlbw()));
+  predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
 
@@ -11112,8 +11133,7 @@ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
                             Universe dummy, rFlagsReg cr)
 %{
-  predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() &&
-            (UseAVX <= 2 || !VM_Version::supports_avx512vlbw()));
+  predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
 
@@ -11170,9 +11190,9 @@ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, rax_RegL val,
                        Universe dummy, rFlagsReg cr)
 %{
-  predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX > 2 && VM_Version::supports_avx512vlbw() && !n->in(2)->in(1)->bottom_type()->is_long()->is_con());
+  predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
+  ins_cost(125);
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
 
   format %{ $$template
@@ -11230,9 +11250,9 @@ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, rax_Reg
 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, rax_RegL val,
                                  Universe dummy, rFlagsReg cr)
 %{
-  predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX > 2 && VM_Version::supports_avx512vlbw() && !n->in(2)->in(1)->bottom_type()->is_long()->is_con());
+  predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
+  ins_cost(125);
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
 
   format %{ $$template
@@ -11291,8 +11311,7 @@ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktm
 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
                         Universe dummy, rFlagsReg cr)
 %{
-  predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX <= 2);
+  predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
 
@@ -11341,8 +11360,7 @@ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
                                   Universe dummy, rFlagsReg cr)
 %{
-  predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX <= 2);
+  predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
 
@@ -11389,8 +11407,7 @@ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_Reg
 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, rax_RegL val,
                              Universe dummy, rFlagsReg cr)
 %{
-  predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX > 2);
+  predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
 
@@ -11440,8 +11457,7 @@ instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, r
 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, kReg ktmp, rax_RegL val,
                                        Universe dummy, rFlagsReg cr)
 %{
-  predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() &&
-            UseAVX > 2);
+  predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
   match(Set dummy (ClearArray (Binary cnt base) val));
   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
 
@@ -11492,8 +11508,9 @@ instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, kR
 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
 %{
   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
-            (UseAVX > 2 && VM_Version::supports_avx512vlbw() && n->in(2)->in(1)->bottom_type()->is_long()->is_con()));
+            ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
   match(Set dummy (ClearArray (Binary cnt base) val));
+  ins_cost(100);
   effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
   ins_encode %{
@@ -13150,7 +13167,7 @@ instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp,
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
                  $scr$$Register, $cx1$$Register, $cx2$$Register,
-                 _counters, _rtm_counters, _stack_rtm_counters,
+                 _rtm_counters, _stack_rtm_counters,
                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
                  true, ra_->C->profile_rtm());
   %}
@@ -13165,7 +13182,7 @@ instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRe
   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
   ins_encode %{
     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
-                 $scr$$Register, $cx1$$Register, noreg, _counters, NULL, NULL, NULL, false, false);
+                 $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
   %}
   ins_pipe(pipe_slow);
 %}
diff --git a/src/hotspot/cpu/zero/assembler_zero.cpp b/src/hotspot/cpu/zero/assembler_zero.cpp
index 693ff06ca54..fe0f1688856 100644
--- a/src/hotspot/cpu/zero/assembler_zero.cpp
+++ b/src/hotspot/cpu/zero/assembler_zero.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2007, 2008, 2009 Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -30,7 +30,6 @@
 #include "interpreter/interpreter.hpp"
 #include "memory/resourceArea.hpp"
 #include "prims/methodHandles.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/objectMonitor.hpp"
 #include "runtime/os.hpp"
diff --git a/src/hotspot/cpu/zero/frame_zero.cpp b/src/hotspot/cpu/zero/frame_zero.cpp
index 70d6a5e855c..c8586cca998 100644
--- a/src/hotspot/cpu/zero/frame_zero.cpp
+++ b/src/hotspot/cpu/zero/frame_zero.cpp
@@ -61,6 +61,16 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return frame(zeroframe()->next(), sender_sp());
 }
 
+OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const {
+  ShouldNotCallThis();
+  return nullptr;
+}
+
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_nonentry_frame(RegisterMap *map) const {
   assert(zeroframe()->is_interpreter_frame() ||
          zeroframe()->is_fake_stub_frame(), "wrong type of frame");
@@ -95,7 +105,7 @@ void frame::patch_pc(Thread* thread, address pc) {
     // We borrow this call to set the thread pointer in the interpreter
     // state; the hook to set up deoptimized frames isn't supplied it.
     assert(pc == NULL, "should be");
-    get_interpreterState()->set_thread(thread->as_Java_thread());
+    get_interpreterState()->set_thread(JavaThread::cast(thread));
   }
 }
 
diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
index c65f5647b89..d6ee756111b 100644
--- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
+++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
@@ -66,17 +66,20 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
   return NULL;
 }
 
-AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
+AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
                                                             int comp_args_on_stack,
-                                                            const GrowableArray <SigEntry> *sig,
-                                                            const VMRegPair *regs,
-                                                            const GrowableArray <SigEntry> *sig_cc,
-                                                            const VMRegPair *regs_cc,
-                                                            const GrowableArray <SigEntry> *sig_cc_ro,
-                                                            const VMRegPair *regs_cc_ro,
-                                                            AdapterFingerPrint *fingerprint,
-                                                            AdapterBlob *&new_adapter) {
-  new_adapter = AdapterBlob::create(masm->code(), 0, 0, NULL);
+                                                            const GrowableArray <SigEntry>* sig,
+                                                            const VMRegPair* regs,
+                                                            const GrowableArray <SigEntry>* sig_cc,
+                                                            const VMRegPair* regs_cc,
+                                                            const GrowableArray <SigEntry>* sig_cc_ro,
+                                                            const VMRegPair* regs_cc_ro,
+                                                            AdapterFingerPrint* fingerprint,
+                                                            AdapterBlob*& new_adapter,
+                                                            bool allocate_code_blob) {
+  if (allocate_code_blob) {
+    new_adapter = AdapterBlob::create(masm->code(), 0, 0, NULL);
+  }
   return AdapterHandlerLibrary::new_entry(
     fingerprint,
     CAST_FROM_FN_PTR(address,zero_null_code_stub),
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 14356b76f4c..d486aee1d8f 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -814,19 +814,24 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
     ret = pthread_attr_setguardsize(&attr, 0);
   }
 
+  ResourceMark rm;
   pthread_t tid = 0;
+
   if (ret == 0) {
-    ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    int limit = 3;
+    do {
+      ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    } while (ret == EAGAIN && limit-- > 0);
   }
 
   if (ret == 0) {
     char buf[64];
-    log_info(os, thread)("Thread started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
-      (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+    log_info(os, thread)("Thread \"%s\" started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
+                         thread->name(), (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
   } else {
     char buf[64];
-    log_warning(os, thread)("Failed to start thread - pthread_create failed (%d=%s) for attributes: %s.",
-      ret, os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+    log_warning(os, thread)("Failed to start thread \"%s\" - pthread_create failed (%d=%s) for attributes: %s.",
+                            thread->name(), ret, os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
     // Log some OS information which might explain why creating the thread failed.
     log_info(os, thread)("Number of threads approx. running in the VM: %d", Threads::number_of_threads());
     LogStream st(Log(os, thread)::info());
@@ -2512,11 +2517,6 @@ void os::set_native_thread_name(const char *name) {
   return;
 }
 
-bool os::bind_to_processor(uint processor_id) {
-  // Not yet implemented.
-  return false;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // debug support
 
@@ -2665,9 +2665,7 @@ int os::open(const char *path, int oflag, int mode) {
 // create binary file, rewriting existing file if required
 int os::create_binary_file(const char* path, bool rewrite_existing) {
   int oflags = O_WRONLY | O_CREAT;
-  if (!rewrite_existing) {
-    oflags |= O_EXCL;
-  }
+  oflags |= rewrite_existing ? O_TRUNC : O_EXCL;
   return ::open64(path, oflags, S_IREAD | S_IWRITE);
 }
 
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 4e10c1cb908..900fedc1f78 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -631,16 +631,22 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   ThreadState state;
 
   {
+
+    ResourceMark rm;
     pthread_t tid;
-    int ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    int ret = 0;
+    int limit = 3;
+    do {
+      ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    } while (ret == EAGAIN && limit-- > 0);
 
     char buf[64];
     if (ret == 0) {
-      log_info(os, thread)("Thread started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
-        (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+      log_info(os, thread)("Thread \"%s\" started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
+                           thread->name(), (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
     } else {
-      log_warning(os, thread)("Failed to start thread - pthread_create failed (%s) for attributes: %s.",
-        os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+      log_warning(os, thread)("Failed to start thread \"%s\" - pthread_create failed (%s) for attributes: %s.",
+                              thread->name(), os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
       // Log some OS information which might explain why creating the thread failed.
       log_info(os, thread)("Number of threads approx. running in the VM: %d", Threads::number_of_threads());
       LogStream st(Log(os, thread)::info());
@@ -2165,11 +2171,6 @@ void os::set_native_thread_name(const char *name) {
 #endif
 }
 
-bool os::bind_to_processor(uint processor_id) {
-  // Not yet implemented.
-  return false;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // debug support
 
@@ -2355,9 +2356,7 @@ int os::open(const char *path, int oflag, int mode) {
 // create binary file, rewriting existing file if required
 int os::create_binary_file(const char* path, bool rewrite_existing) {
   int oflags = O_WRONLY | O_CREAT;
-  if (!rewrite_existing) {
-    oflags |= O_EXCL;
-  }
+  oflags |= rewrite_existing ? O_TRUNC : O_EXCL;
   return ::open(path, oflags, S_IREAD | S_IWRITE);
 }
 
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index a3424322be8..6f7c8ec6e19 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -412,7 +412,7 @@ void os::init_system_properties_values() {
   //        ...
   //        7: The default directories, normally /lib and /usr/lib.
 #ifndef OVERRIDE_LIBPATH
-  #if defined(AMD64) || (defined(_LP64) && defined(SPARC)) || defined(PPC64) || defined(S390)
+  #if defined(_LP64)
     #define DEFAULT_LIBPATH "/usr/lib64:/lib64:/lib:/usr/lib"
   #else
     #define DEFAULT_LIBPATH "/lib:/usr/lib"
@@ -863,16 +863,21 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   ThreadState state;
 
   {
+    ResourceMark rm;
     pthread_t tid;
-    int ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    int ret = 0;
+    int limit = 3;
+    do {
+      ret = pthread_create(&tid, &attr, (void* (*)(void*)) thread_native_entry, thread);
+    } while (ret == EAGAIN && limit-- > 0);
 
     char buf[64];
     if (ret == 0) {
-      log_info(os, thread)("Thread started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
-        (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+      log_info(os, thread)("Thread \"%s\" started (pthread id: " UINTX_FORMAT ", attributes: %s). ",
+                           thread->name(), (uintx) tid, os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
     } else {
-      log_warning(os, thread)("Failed to start thread - pthread_create failed (%s) for attributes: %s.",
-        os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
+      log_warning(os, thread)("Failed to start thread \"%s\" - pthread_create failed (%s) for attributes: %s.",
+                              thread->name(), os::errno_name(ret), os::Posix::describe_pthread_attr(buf, sizeof(buf), &attr));
       // Log some OS information which might explain why creating the thread failed.
       log_info(os, thread)("Number of threads approx. running in the VM: %d", Threads::number_of_threads());
       LogStream st(Log(os, thread)::info());
@@ -1668,6 +1673,9 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
 #ifndef EM_RISCV
   #define EM_RISCV      243               /* RISC-V */
 #endif
+#ifndef EM_LOONGARCH
+  #define EM_LOONGARCH  258               /* LoongArch */
+#endif
 
   static const arch_t arch_array[]={
     {EM_386,         EM_386,     ELFCLASS32, ELFDATA2LSB, (char*)"IA 32"},
@@ -1695,6 +1703,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
     {EM_68K,         EM_68K,     ELFCLASS32, ELFDATA2MSB, (char*)"M68k"},
     {EM_AARCH64,     EM_AARCH64, ELFCLASS64, ELFDATA2LSB, (char*)"AARCH64"},
     {EM_RISCV,       EM_RISCV,   ELFCLASS64, ELFDATA2LSB, (char*)"RISC-V"},
+    {EM_LOONGARCH,   EM_LOONGARCH, ELFCLASS64, ELFDATA2LSB, (char*)"LoongArch"},
   };
 
 #if  (defined IA32)
@@ -1731,9 +1740,11 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
   static  Elf32_Half running_arch_code=EM_SH;
 #elif  (defined RISCV)
   static  Elf32_Half running_arch_code=EM_RISCV;
+#elif  (defined LOONGARCH)
+  static  Elf32_Half running_arch_code=EM_LOONGARCH;
 #else
     #error Method os::dll_load requires that one of following is defined:\
-        AARCH64, ALPHA, ARM, AMD64, IA32, IA64, M68K, MIPS, MIPSEL, PARISC, __powerpc__, __powerpc64__, RISCV, S390, SH, __sparc
+        AARCH64, ALPHA, ARM, AMD64, IA32, IA64, LOONGARCH, M68K, MIPS, MIPSEL, PARISC, __powerpc__, __powerpc64__, RISCV, S390, SH, __sparc
 #endif
 
   // Identify compatibility class for VM's architecture and library's architecture
@@ -2137,44 +2148,51 @@ void os::Linux::print_system_memory_info(outputStream* st) {
                       "/sys/kernel/mm/transparent_hugepage/defrag", st);
 }
 
-void os::Linux::print_process_memory_info(outputStream* st) {
-
-  st->print_cr("Process Memory:");
-
-  // Print virtual and resident set size; peak values; swap; and for
-  //  rss its components if the kernel is recent enough.
-  ssize_t vmsize = -1, vmpeak = -1, vmswap = -1,
-      vmrss = -1, vmhwm = -1, rssanon = -1, rssfile = -1, rssshmem = -1;
-  const int num_values = 8;
-  int num_found = 0;
+bool os::Linux::query_process_memory_info(os::Linux::meminfo_t* info) {
   FILE* f = ::fopen("/proc/self/status", "r");
+  const int num_values = sizeof(os::Linux::meminfo_t) / sizeof(size_t);
+  int num_found = 0;
   char buf[256];
+  info->vmsize = info->vmpeak = info->vmrss = info->vmhwm = info->vmswap =
+      info->rssanon = info->rssfile = info->rssshmem = -1;
   if (f != NULL) {
     while (::fgets(buf, sizeof(buf), f) != NULL && num_found < num_values) {
-      if ( (vmsize == -1    && sscanf(buf, "VmSize: " SSIZE_FORMAT " kB", &vmsize) == 1) ||
-           (vmpeak == -1    && sscanf(buf, "VmPeak: " SSIZE_FORMAT " kB", &vmpeak) == 1) ||
-           (vmswap == -1    && sscanf(buf, "VmSwap: " SSIZE_FORMAT " kB", &vmswap) == 1) ||
-           (vmhwm == -1     && sscanf(buf, "VmHWM: " SSIZE_FORMAT " kB", &vmhwm) == 1) ||
-           (vmrss == -1     && sscanf(buf, "VmRSS: " SSIZE_FORMAT " kB", &vmrss) == 1) ||
-           (rssanon == -1   && sscanf(buf, "RssAnon: " SSIZE_FORMAT " kB", &rssanon) == 1) ||
-           (rssfile == -1   && sscanf(buf, "RssFile: " SSIZE_FORMAT " kB", &rssfile) == 1) ||
-           (rssshmem == -1  && sscanf(buf, "RssShmem: " SSIZE_FORMAT " kB", &rssshmem) == 1)
+      if ( (info->vmsize == -1    && sscanf(buf, "VmSize: " SSIZE_FORMAT " kB", &info->vmsize) == 1) ||
+           (info->vmpeak == -1    && sscanf(buf, "VmPeak: " SSIZE_FORMAT " kB", &info->vmpeak) == 1) ||
+           (info->vmswap == -1    && sscanf(buf, "VmSwap: " SSIZE_FORMAT " kB", &info->vmswap) == 1) ||
+           (info->vmhwm == -1     && sscanf(buf, "VmHWM: " SSIZE_FORMAT " kB", &info->vmhwm) == 1) ||
+           (info->vmrss == -1     && sscanf(buf, "VmRSS: " SSIZE_FORMAT " kB", &info->vmrss) == 1) ||
+           (info->rssanon == -1   && sscanf(buf, "RssAnon: " SSIZE_FORMAT " kB", &info->rssanon) == 1) || // Needs Linux 4.5
+           (info->rssfile == -1   && sscanf(buf, "RssFile: " SSIZE_FORMAT " kB", &info->rssfile) == 1) || // Needs Linux 4.5
+           (info->rssshmem == -1  && sscanf(buf, "RssShmem: " SSIZE_FORMAT " kB", &info->rssshmem) == 1)  // Needs Linux 4.5
            )
       {
         num_found ++;
       }
     }
     fclose(f);
+    return true;
+  }
+  return false;
+}
+
+void os::Linux::print_process_memory_info(outputStream* st) {
 
-    st->print_cr("Virtual Size: " SSIZE_FORMAT "K (peak: " SSIZE_FORMAT "K)", vmsize, vmpeak);
-    st->print("Resident Set Size: " SSIZE_FORMAT "K (peak: " SSIZE_FORMAT "K)", vmrss, vmhwm);
-    if (rssanon != -1) { // requires kernel >= 4.5
+  st->print_cr("Process Memory:");
+
+  // Print virtual and resident set size; peak values; swap; and for
+  //  rss its components if the kernel is recent enough.
+  meminfo_t info;
+  if (query_process_memory_info(&info)) {
+    st->print_cr("Virtual Size: " SSIZE_FORMAT "K (peak: " SSIZE_FORMAT "K)", info.vmsize, info.vmpeak);
+    st->print("Resident Set Size: " SSIZE_FORMAT "K (peak: " SSIZE_FORMAT "K)", info.vmrss, info.vmhwm);
+    if (info.rssanon != -1) { // requires kernel >= 4.5
       st->print(" (anon: " SSIZE_FORMAT "K, file: " SSIZE_FORMAT "K, shmem: " SSIZE_FORMAT "K)",
-                  rssanon, rssfile, rssshmem);
+                info.rssanon, info.rssfile, info.rssshmem);
     }
     st->cr();
-    if (vmswap != -1) { // requires kernel >= 2.6.34
-      st->print_cr("Swapped out: " SSIZE_FORMAT "K", vmswap);
+    if (info.vmswap != -1) { // requires kernel >= 2.6.34
+      st->print_cr("Swapped out: " SSIZE_FORMAT "K", info.vmswap);
     }
   } else {
     st->print_cr("Could not open /proc/self/status to get process memory related information");
@@ -2195,7 +2213,7 @@ void os::Linux::print_process_memory_info(outputStream* st) {
     struct glibc_mallinfo mi = _mallinfo();
     total_allocated = (size_t)(unsigned)mi.uordblks;
     // Since mallinfo members are int, glibc values may have wrapped. Warn about this.
-    might_have_wrapped = (vmrss * K) > UINT_MAX && (vmrss * K) > (total_allocated + UINT_MAX);
+    might_have_wrapped = (info.vmrss * K) > UINT_MAX && (info.vmrss * K) > (total_allocated + UINT_MAX);
   }
   if (_mallinfo2 != NULL || _mallinfo != NULL) {
     st->print_cr("C-Heap outstanding allocations: " SIZE_FORMAT "K%s",
@@ -4785,11 +4803,6 @@ void os::set_native_thread_name(const char *name) {
   }
 }
 
-bool os::bind_to_processor(uint processor_id) {
-  // Not yet implemented.
-  return false;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // debug support
 
@@ -4968,9 +4981,7 @@ int os::open(const char *path, int oflag, int mode) {
 // create binary file, rewriting existing file if required
 int os::create_binary_file(const char* path, bool rewrite_existing) {
   int oflags = O_WRONLY | O_CREAT;
-  if (!rewrite_existing) {
-    oflags |= O_EXCL;
-  }
+  oflags |= rewrite_existing ? O_TRUNC : O_EXCL;
   return ::open64(path, oflags, S_IREAD | S_IWRITE);
 }
 
diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp
index ada8db6977e..692dae042ab 100644
--- a/src/hotspot/os/linux/os_linux.hpp
+++ b/src/hotspot/os/linux/os_linux.hpp
@@ -174,6 +174,23 @@ class Linux {
   // Return the namespace pid if so, otherwise -1.
   static int get_namespace_pid(int vmid);
 
+  // Output structure for query_process_memory_info()
+  struct meminfo_t {
+    ssize_t vmsize;     // current virtual size
+    ssize_t vmpeak;     // peak virtual size
+    ssize_t vmrss;      // current resident set size
+    ssize_t vmhwm;      // peak resident set size
+    ssize_t vmswap;     // swapped out
+    ssize_t rssanon;    // resident set size (anonymous mappings, needs 4.5)
+    ssize_t rssfile;    // resident set size (file mappings, needs 4.5)
+    ssize_t rssshmem;   // resident set size (shared mappings, needs 4.5)
+  };
+
+  // Attempts to query memory information about the current process and return it in the output structure.
+  // May fail (returns false) or succeed (returns true) but not all output fields are available; unavailable
+  // fields will contain -1.
+  static bool query_process_memory_info(meminfo_t* info);
+
   // Stack repair handling
 
   // none present
diff --git a/src/hotspot/os/linux/trimCHeapDCmd.cpp b/src/hotspot/os/linux/trimCHeapDCmd.cpp
new file mode 100644
index 00000000000..ee93ac5e8c8
--- /dev/null
+++ b/src/hotspot/os/linux/trimCHeapDCmd.cpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "logging/log.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/ostream.hpp"
+#include "trimCHeapDCmd.hpp"
+
+#include <malloc.h>
+
+void TrimCLibcHeapDCmd::execute(DCmdSource source, TRAPS) {
+#ifdef __GLIBC__
+  stringStream ss_report(1024); // Note: before calling trim
+
+  os::Linux::meminfo_t info1;
+  os::Linux::meminfo_t info2;
+  // Query memory before...
+  bool have_info1 = os::Linux::query_process_memory_info(&info1);
+
+  _output->print_cr("Attempting trim...");
+  ::malloc_trim(0);
+  _output->print_cr("Done.");
+
+  // ...and after trim.
+  bool have_info2 = os::Linux::query_process_memory_info(&info2);
+
+  // Print report both to output stream as well to UL
+  bool wrote_something = false;
+  if (have_info1 && have_info2) {
+    if (info1.vmsize != -1 && info2.vmsize != -1) {
+      ss_report.print_cr("Virtual size before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
+                         info1.vmsize, info2.vmsize, (info2.vmsize - info1.vmsize));
+      wrote_something = true;
+    }
+    if (info1.vmrss != -1 && info2.vmrss != -1) {
+      ss_report.print_cr("RSS before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
+                         info1.vmrss, info2.vmrss, (info2.vmrss - info1.vmrss));
+      wrote_something = true;
+    }
+    if (info1.vmswap != -1 && info2.vmswap != -1) {
+      ss_report.print_cr("Swap before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
+                         info1.vmswap, info2.vmswap, (info2.vmswap - info1.vmswap));
+      wrote_something = true;
+    }
+  }
+  if (!wrote_something) {
+    ss_report.print_raw("No details available.");
+  }
+
+  _output->print_raw(ss_report.base());
+  log_info(os)("malloc_trim:\n%s", ss_report.base());
+#else
+  _output->print_cr("Not available.");
+#endif
+}
diff --git a/src/hotspot/os/linux/trimCHeapDCmd.hpp b/src/hotspot/os/linux/trimCHeapDCmd.hpp
new file mode 100644
index 00000000000..4c5b5cc2219
--- /dev/null
+++ b/src/hotspot/os/linux/trimCHeapDCmd.hpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef OS_LINUX_TRIMCHEAPDCMD_HPP
+#define OS_LINUX_TRIMCHEAPDCMD_HPP
+
+#include "services/diagnosticCommand.hpp"
+
+class outputStream;
+
+class TrimCLibcHeapDCmd : public DCmd {
+public:
+  TrimCLibcHeapDCmd(outputStream* output, bool heap) : DCmd(output, heap) {}
+  static const char* name() {
+    return "System.trim_native_heap";
+  }
+  static const char* description() {
+    return "Attempts to free up memory by trimming the C-heap.";
+  }
+  static const char* impact() {
+    return "Low";
+  }
+  static const JavaPermission permission() {
+    JavaPermission p = { "java.lang.management.ManagementPermission", "control", NULL };
+    return p;
+  }
+  virtual void execute(DCmdSource source, TRAPS);
+};
+
+#endif // OS_LINUX_TRIMCHEAPDCMD_HPP
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 6e996b11993..773c392b7e6 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -743,21 +743,27 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   // flag appears to work with _beginthredex() as well.
 
   const unsigned initflag = CREATE_SUSPENDED | STACK_SIZE_PARAM_IS_A_RESERVATION;
-  HANDLE thread_handle =
-    (HANDLE)_beginthreadex(NULL,
-                           (unsigned)stack_size,
-                           (unsigned (__stdcall *)(void*)) thread_native_entry,
-                           thread,
-                           initflag,
-                           &thread_id);
-
+  HANDLE thread_handle;
+  int limit = 3;
+  do {
+    thread_handle =
+      (HANDLE)_beginthreadex(NULL,
+                             (unsigned)stack_size,
+                             (unsigned (__stdcall *)(void*)) thread_native_entry,
+                             thread,
+                             initflag,
+                             &thread_id);
+  } while (thread_handle == NULL && errno == EAGAIN && limit-- > 0);
+
+  ResourceMark rm;
   char buf[64];
   if (thread_handle != NULL) {
-    log_info(os, thread)("Thread started (tid: %u, attributes: %s)",
-      thread_id, describe_beginthreadex_attributes(buf, sizeof(buf), stack_size, initflag));
+    log_info(os, thread)("Thread \"%s\" started (tid: %u, attributes: %s)",
+                         thread->name(), thread_id,
+                         describe_beginthreadex_attributes(buf, sizeof(buf), stack_size, initflag));
   } else {
-    log_warning(os, thread)("Failed to start thread - _beginthreadex failed (%s) for attributes: %s.",
-      os::errno_name(errno), describe_beginthreadex_attributes(buf, sizeof(buf), stack_size, initflag));
+    log_warning(os, thread)("Failed to start thread \"%s\" - _beginthreadex failed (%s) for attributes: %s.",
+                            thread->name(), os::errno_name(errno), describe_beginthreadex_attributes(buf, sizeof(buf), stack_size, initflag));
     // Log some OS information which might explain why creating the thread failed.
     log_info(os, thread)("Number of threads approx. running in the VM: %d", Threads::number_of_threads());
     LogStream st(Log(os, thread)::info());
@@ -892,7 +898,7 @@ static SetThreadDescriptionFnPtr _SetThreadDescription = NULL;
 DEBUG_ONLY(static GetThreadDescriptionFnPtr _GetThreadDescription = NULL;)
 
 // forward decl.
-errno_t convert_to_unicode(char const* char_path, LPWSTR* unicode_path);
+static errno_t convert_to_unicode(char const* char_path, LPWSTR* unicode_path);
 
 void os::set_native_thread_name(const char *name) {
 
@@ -971,11 +977,6 @@ void os::set_native_thread_name(const char *name) {
   } __except(EXCEPTION_EXECUTE_HANDLER) {}
 }
 
-bool os::bind_to_processor(uint processor_id) {
-  // Not yet implemented.
-  return false;
-}
-
 void os::win32::initialize_performance_counter() {
   LARGE_INTEGER count;
   QueryPerformanceFrequency(&count);
@@ -2288,7 +2289,7 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo,
 
   // Save pc in thread
   if (thread != nullptr && thread->is_Java_thread()) {
-    thread->as_Java_thread()->set_saved_exception_pc((address)(DWORD_PTR)exceptionInfo->ContextRecord->PC_NAME);
+    JavaThread::cast(thread)->set_saved_exception_pc((address)(DWORD_PTR)exceptionInfo->ContextRecord->PC_NAME);
   }
 
   // Set pc to handler
@@ -2582,7 +2583,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
 #endif
 
   if (t != NULL && t->is_Java_thread()) {
-    JavaThread* thread = t->as_Java_thread();
+    JavaThread* thread = JavaThread::cast(t);
     bool in_java = thread->thread_state() == _thread_in_Java;
     bool in_native = thread->thread_state() == _thread_in_native;
     bool in_vm = thread->thread_state() == _thread_in_vm;
@@ -4806,9 +4807,7 @@ bool os::dir_is_empty(const char* path) {
 // create binary file, rewriting existing file if required
 int os::create_binary_file(const char* path, bool rewrite_existing) {
   int oflags = _O_CREAT | _O_WRONLY | _O_BINARY;
-  if (!rewrite_existing) {
-    oflags |= _O_EXCL;
-  }
+  oflags |= rewrite_existing ? _O_TRUNC : _O_EXCL;
   return ::open(path, oflags, _S_IREAD | _S_IWRITE);
 }
 
diff --git a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp b/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
index e0c2961e484..fba59870d7c 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
@@ -27,6 +27,8 @@
 #ifndef OS_CPU_BSD_AARCH64_ATOMIC_BSD_AARCH64_HPP
 #define OS_CPU_BSD_AARCH64_ATOMIC_BSD_AARCH64_HPP
 
+#include "utilities/debug.hpp"
+
 // Implementation of class atomic
 // Note that memory_order_conservative requires a full barrier after atomic stores.
 // See https://patchwork.kernel.org/patch/3575821/
@@ -64,17 +66,40 @@ inline T Atomic::PlatformCmpxchg<byte_size>::operator()(T volatile* dest,
                                                         T exchange_value,
                                                         atomic_memory_order order) const {
   STATIC_ASSERT(byte_size == sizeof(T));
-  if (order == memory_order_relaxed) {
+  if (order == memory_order_conservative) {
     T value = compare_value;
+    FULL_MEM_BARRIER;
     __atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
                               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+    FULL_MEM_BARRIER;
     return value;
   } else {
+    STATIC_ASSERT (
+       // The modes that align with C++11 are intended to
+       // follow the same semantics.
+       memory_order_relaxed == __ATOMIC_RELAXED &&
+       memory_order_acquire == __ATOMIC_ACQUIRE &&
+       memory_order_release == __ATOMIC_RELEASE &&
+       memory_order_acq_rel == __ATOMIC_ACQ_REL &&
+       memory_order_seq_cst == __ATOMIC_SEQ_CST);
+
+    // Some sanity checking on the memory order. It makes no
+    // sense to have a release operation for a store that never
+    // happens.
+    int failure_memory_order;
+    switch (order) {
+    case memory_order_release:
+      failure_memory_order = memory_order_relaxed; break;
+    case memory_order_acq_rel:
+      failure_memory_order = memory_order_acquire; break;
+    default:
+      failure_memory_order = order;
+    }
+    assert(failure_memory_order <= order, "must be");
+
     T value = compare_value;
-    FULL_MEM_BARRIER;
     __atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
-                              __ATOMIC_RELAXED, __ATOMIC_RELAXED);
-    FULL_MEM_BARRIER;
+                              order, failure_memory_order);
     return value;
   }
 }
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
index f5d2c2b69c2..3007587d9c2 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
@@ -112,7 +112,55 @@ aarch64_atomic_cmpxchg_8_default_impl:
         dmb     ish
         ret
 
-        .globl aarch64_atomic_cmpxchg_1_relaxed_default_impl
+        .globl aarch64_atomic_cmpxchg_4_release_default_impl
+        .align 5
+aarch64_atomic_cmpxchg_4_release_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldxr    w3, [x0]
+        cmp     w3, w1
+        b.ne    1f
+        stlxr   w8, w2, [x0]
+        cbnz    w8, 0b
+1:      mov     w0, w3
+        ret
+
+        .globl aarch64_atomic_cmpxchg_8_release_default_impl
+        .align 5
+aarch64_atomic_cmpxchg_8_release_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldxr    x3, [x0]
+        cmp     x3, x1
+        b.ne    1f
+        stlxr   w8, x2, [x0]
+        cbnz    w8, 0b
+1:      mov     x0, x3
+        ret
+
+        .globl aarch64_atomic_cmpxchg_4_seq_cst_default_impl
+        .align 5
+aarch64_atomic_cmpxchg_4_seq_cst_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldaxr   w3, [x0]
+        cmp     w3, w1
+        b.ne    1f
+        stlxr   w8, w2, [x0]
+        cbnz    w8, 0b
+1:      mov     w0, w3
+        ret
+
+        .globl aarch64_atomic_cmpxchg_8_seq_cst_default_impl
+        .align 5
+aarch64_atomic_cmpxchg_8_seq_cst_default_impl:
+        prfm    pstl1strm, [x0]
+0:      ldaxr   x3, [x0]
+        cmp     x3, x1
+        b.ne    1f
+        stlxr   w8, x2, [x0]
+        cbnz    w8, 0b
+1:      mov     x0, x3
+        ret
+
+.globl aarch64_atomic_cmpxchg_1_relaxed_default_impl
         .align 5
 aarch64_atomic_cmpxchg_1_relaxed_default_impl:
         prfm    pstl1strm, [x0]
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
index 77e860ed5ec..316e877ec1f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
@@ -151,6 +151,11 @@ inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
   switch (order) {
   case memory_order_relaxed:
     stub = aarch64_atomic_cmpxchg_4_relaxed_impl; break;
+  case memory_order_release:
+    stub = aarch64_atomic_cmpxchg_4_release_impl; break;
+  case memory_order_acq_rel:
+  case memory_order_seq_cst:
+    stub = aarch64_atomic_cmpxchg_4_seq_cst_impl; break;
   default:
     stub = aarch64_atomic_cmpxchg_4_impl; break;
   }
@@ -169,6 +174,11 @@ inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
   switch (order) {
   case memory_order_relaxed:
     stub = aarch64_atomic_cmpxchg_8_relaxed_impl; break;
+  case memory_order_release:
+    stub = aarch64_atomic_cmpxchg_8_release_impl; break;
+  case memory_order_acq_rel:
+  case memory_order_seq_cst:
+    stub = aarch64_atomic_cmpxchg_8_seq_cst_impl; break;
   default:
     stub = aarch64_atomic_cmpxchg_8_impl; break;
   }
diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
index 6fc0c840642..7df65bed1ee 100644
--- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
+++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
@@ -468,9 +468,8 @@ void os::print_register_info(outputStream *st, const void *context) {
   st->print_cr("Register to memory mapping:");
   st->cr();
   for (int r = 0; r < ARM_REGS_IN_CONTEXT; r++) {
-    st->print_cr("  %-3s = " INTPTR_FORMAT, as_Register(r)->name(), reg_area[r]);
+    st->print("  %-3s = ", as_Register(r)->name());
     print_location(st, reg_area[r]);
-    st->cr();
   }
   st->cr();
 }
diff --git a/src/hotspot/share/adlc/arena.hpp b/src/hotspot/share/adlc/arena.hpp
index d0dac9d45ca..1fa99ed0e24 100644
--- a/src/hotspot/share/adlc/arena.hpp
+++ b/src/hotspot/share/adlc/arena.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -118,8 +118,7 @@ class Arena: public CHeapObj {
     }
   }
   // Further assume size is padded out to words
-  // Warning:  in LP64, Amalloc_4 is really Amalloc_8
-  void *Amalloc_4(size_t x) {
+  void *AmallocWords(size_t x) {
     assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
     if (_hwm + x > _max) {
       return grow(x);
diff --git a/src/hotspot/share/adlc/dict2.cpp b/src/hotspot/share/adlc/dict2.cpp
index 2dc60b250bf..4f3e33dd626 100644
--- a/src/hotspot/share/adlc/dict2.cpp
+++ b/src/hotspot/share/adlc/dict2.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -74,7 +74,7 @@ void Dict::init() {
 
   _size = 16;                   // Size is a power of 2
   _cnt = 0;                     // Dictionary is empty
-  _bin = (bucket*)_arena->Amalloc_4(sizeof(bucket) * _size);
+  _bin = (bucket*)_arena->AmallocWords(sizeof(bucket) * _size);
   memset(_bin, 0, sizeof(bucket) * _size);
 }
 
@@ -115,7 +115,7 @@ void Dict::doubhash(void) {
     if( !j ) j = 1;             // Handle zero-sized buckets
     nb->_max = j<<1;
     // Allocate worst case space for key-value pairs
-    nb->_keyvals = (const void**)_arena->Amalloc_4( sizeof(void *)*nb->_max*2 );
+    nb->_keyvals = (const void**)_arena->AmallocWords( sizeof(void *)*nb->_max*2 );
     int nbcnt = 0;
 
     for( j=0; j<b->_cnt; j++ ) {  // Rehash all keys in this bucket
@@ -138,11 +138,11 @@ void Dict::doubhash(void) {
 //------------------------------Dict-----------------------------------------
 // Deep copy a dictionary.
 Dict::Dict( const Dict &d ) : _size(d._size), _cnt(d._cnt), _hash(d._hash),_cmp(d._cmp), _arena(d._arena) {
-  _bin = (bucket*)_arena->Amalloc_4(sizeof(bucket)*_size);
+  _bin = (bucket*)_arena->AmallocWords(sizeof(bucket)*_size);
   memcpy( _bin, d._bin, sizeof(bucket)*_size );
   for( int i=0; i<_size; i++ ) {
     if( !_bin[i]._keyvals ) continue;
-    _bin[i]._keyvals=(const void**)_arena->Amalloc_4( sizeof(void *)*_bin[i]._max*2);
+    _bin[i]._keyvals=(const void**)_arena->AmallocWords( sizeof(void *)*_bin[i]._max*2);
     memcpy( _bin[i]._keyvals, d._bin[i]._keyvals,_bin[i]._cnt*2*sizeof(void*));
   }
 }
@@ -195,7 +195,7 @@ const void *Dict::Insert(const void *key, const void *val) {
   if( b->_cnt == b->_max ) {    // Must grow bucket?
     if( !b->_keyvals ) {
       b->_max = 2;              // Initial bucket size
-      b->_keyvals = (const void**)_arena->Amalloc_4( sizeof(void *)*b->_max*2 );
+      b->_keyvals = (const void**)_arena->AmallocWords( sizeof(void *)*b->_max*2 );
     } else {
       b->_keyvals = (const void**)_arena->Arealloc( b->_keyvals, sizeof(void *)*b->_max*2, sizeof(void *)*b->_max*4 );
       b->_max <<= 1;            // Double bucket
diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp
index 9f665bacab1..6f63b1b660d 100644
--- a/src/hotspot/share/adlc/main.cpp
+++ b/src/hotspot/share/adlc/main.cpp
@@ -230,7 +230,6 @@ int main(int argc, char *argv[])
   AD.addInclude(AD._CPP_file, "opto/regalloc.hpp");
   AD.addInclude(AD._CPP_file, "opto/regmask.hpp");
   AD.addInclude(AD._CPP_file, "opto/runtime.hpp");
-  AD.addInclude(AD._CPP_file, "runtime/biasedLocking.hpp");
   AD.addInclude(AD._CPP_file, "runtime/safepointMechanism.hpp");
   AD.addInclude(AD._CPP_file, "runtime/sharedRuntime.hpp");
   AD.addInclude(AD._CPP_file, "runtime/stubRoutines.hpp");
diff --git a/src/hotspot/share/adlc/output_c.cpp b/src/hotspot/share/adlc/output_c.cpp
index a0dbb7b1d28..847e6cb2222 100644
--- a/src/hotspot/share/adlc/output_c.cpp
+++ b/src/hotspot/share/adlc/output_c.cpp
@@ -1342,6 +1342,9 @@ static void generate_peepreplace( FILE *fp, FormDict &globals, PeepMatch *pmatch
     assert( false, "ShouldNotReachHere();");
   }
 
+  for (int i = 0; i <= max_position; i++) {
+    fprintf(fp, "        inst%d->set_removed();\n", i);
+  }
   // Return the new sub-tree
   fprintf(fp, "        deleted = %d;\n", max_position+1 /*zero to one based*/);
   fprintf(fp, "        return root;  // return new root;\n");
@@ -1526,7 +1529,6 @@ void ArchDesc::defineExpand(FILE *fp, InstructForm *node) {
       }
 
       if (node->is_ideal_fastlock() && new_inst->is_ideal_fastlock()) {
-        fprintf(fp, "  ((MachFastLockNode*)n%d)->_counters = _counters;\n", cnt);
         fprintf(fp, "  ((MachFastLockNode*)n%d)->_rtm_counters = _rtm_counters;\n", cnt);
         fprintf(fp, "  ((MachFastLockNode*)n%d)->_stack_rtm_counters = _stack_rtm_counters;\n", cnt);
       }
@@ -3941,7 +3943,6 @@ void ArchDesc::buildMachNode(FILE *fp_cpp, InstructForm *inst, const char *inden
     fprintf(fp_cpp, "%s node->_probs = _leaf->as_Jump()->_probs;\n", indent);
   }
   if( inst->is_ideal_fastlock() ) {
-    fprintf(fp_cpp, "%s node->_counters = _leaf->as_FastLock()->counters();\n", indent);
     fprintf(fp_cpp, "%s node->_rtm_counters = _leaf->as_FastLock()->rtm_counters();\n", indent);
     fprintf(fp_cpp, "%s node->_stack_rtm_counters = _leaf->as_FastLock()->stack_rtm_counters();\n", indent);
   }
diff --git a/src/hotspot/share/asm/assembler.hpp b/src/hotspot/share/asm/assembler.hpp
index 202cba10c5d..10fa79eb30a 100644
--- a/src/hotspot/share/asm/assembler.hpp
+++ b/src/hotspot/share/asm/assembler.hpp
@@ -91,7 +91,7 @@ class Label {
   int _patch_index;
   GrowableArray<int>* _patch_overflow;
 
-  Label(const Label&) { ShouldNotReachHere(); }
+  NONCOPYABLE(Label);
  protected:
 
   // The label will be bound to a location near its users.
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index 770439d2996..237f84dacbe 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -229,7 +229,11 @@ class CodeSection {
     set_end(curr);
   }
 
-  void emit_int32(int32_t x) { *((int32_t*) end()) = x; set_end(end() + sizeof(int32_t)); }
+  void emit_int32(int32_t x) {
+    address curr = end();
+    *((int32_t*) curr) = x;
+    set_end(curr + sizeof(int32_t));
+  }
   void emit_int32(int8_t x1, int8_t x2, int8_t x3, int8_t x4)  {
     address curr = end();
     *((int8_t*)  curr++) = x1;
diff --git a/src/hotspot/share/asm/macroAssembler_common.cpp b/src/hotspot/share/asm/macroAssembler_common.cpp
index 83326e7fb02..1fcb3109706 100644
--- a/src/hotspot/share/asm/macroAssembler_common.cpp
+++ b/src/hotspot/share/asm/macroAssembler_common.cpp
@@ -173,7 +173,12 @@ void MacroAssembler::shuffle_inline_args(bool is_packing, bool receiver_only,
       BasicType bt = sig->at(sig_index)._bt;
       if (SigEntry::skip_value_delimiters(sig, sig_index)) {
         VMReg from_reg = regs[from_index].first();
-        done &= move_helper(from_reg, regs_to[to_index].first(), bt, reg_state);
+        if (from_reg->is_valid()) {
+          done &= move_helper(from_reg, regs_to[to_index].first(), bt, reg_state);
+        } else {
+          // halves of T_LONG or T_DOUBLE
+          assert(bt == T_VOID, "unexpected basic type");
+        }
         to_index += step;
         from_index += step;
       } else if (is_packing) {
diff --git a/src/hotspot/share/asm/register.hpp b/src/hotspot/share/asm/register.hpp
index 66f58d52953..dea7e9dd3d0 100644
--- a/src/hotspot/share/asm/register.hpp
+++ b/src/hotspot/share/asm/register.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,55 +45,16 @@ class AbstractRegisterImpl {
   int value() const                              { return (int)(intx)this; }
 };
 
-
-//
-// Macros for use in defining Register instances.  We'd like to be
-// able to simply define const instances of the RegisterImpl* for each
-// of the registers needed on a system in a header file.  However many
-// compilers don't handle this very well and end up producing a
-// private definition in every file which includes the header file.
-// Along with the static constructors necessary for initialization it
-// can consume a significant amount of space in the result library.
-//
-// The following macros allow us to declare the instance in a .hpp and
-// produce an enumeration value which has the same number.  Then in a
-// .cpp the the register instance can be defined using the enumeration
-// value.  This avoids the use of static constructors and multiple
-// definitions per .cpp.  In addition #defines for the register can be
-// produced so that the constant registers can be inlined.  These
-// macros should not be used inside other macros, because you may get
-// multiple evaluations of the macros which can give bad results.
-//
-// Here are some example uses and expansions.  Note that the macro
-// invocation is terminated with a ;.
-//
-// CONSTANT_REGISTER_DECLARATION(Register, G0, 0);
-//
-// extern const Register G0 ;
-// enum { G0_RegisterEnumValue = 0 } ;
-//
-// REGISTER_DECLARATION(Register, Gmethod, G5);
-//
-// extern const Register Gmethod ;
-// enum { Gmethod_RegisterEnumValue = G5_RegisterEnumValue } ;
-//
-// REGISTER_DEFINITION(Register, G0);
-//
-// const Register G0 = ( ( Register ) G0_RegisterEnumValue ) ;
-//
-
 #define AS_REGISTER(type,name)         ((type)name##_##type##EnumValue)
 
-#define CONSTANT_REGISTER_DECLARATION(type, name, value) \
-extern const type name;                                  \
+#define CONSTANT_REGISTER_DECLARATION(type, name, value)        \
+const type name = ((type)value);                                \
 enum { name##_##type##EnumValue = (value) }
 
-#define REGISTER_DECLARATION(type, name, value) \
-extern const type name;                         \
-enum { name##_##type##EnumValue = value##_##type##EnumValue }
+#define REGISTER_DECLARATION(type, name, value)                 \
+const type name = ((type)value)
 
-#define REGISTER_DEFINITION(type, name) \
-const type name = ((type)name##_##type##EnumValue)
+#define REGISTER_DEFINITION(type, name)
 
 #include CPU_HEADER(register)
 
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp
index a0e700d741f..c877b9a08c7 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.cpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp
@@ -797,7 +797,7 @@ void Canonicalizer::do_If(If* x) {
         if (cmp->x() == cmp->y()) {
           do_If(canon);
         } else {
-          if (compilation()->profile_branches() || compilation()->count_backedges()) {
+          if (compilation()->profile_branches() || compilation()->is_profiling()) {
             // TODO: If profiling, leave floating point comparisons unoptimized.
             // We currently do not support profiling of the unordered case.
             switch(cmp->op()) {
@@ -860,172 +860,17 @@ void Canonicalizer::do_Throw          (Throw*           x) {}
 void Canonicalizer::do_Base           (Base*            x) {}
 void Canonicalizer::do_OsrEntry       (OsrEntry*        x) {}
 void Canonicalizer::do_ExceptionObject(ExceptionObject* x) {}
-
-static bool match_index_and_scale(Instruction*  instr,
-                                  Instruction** index,
-                                  int*          log2_scale) {
-  // Skip conversion ops. This works only on 32bit because of the implicit l2i that the
-  // unsafe performs.
-#ifndef _LP64
-  Convert* convert = instr->as_Convert();
-  if (convert != NULL && convert->op() == Bytecodes::_i2l) {
-    assert(convert->value()->type() == intType, "invalid input type");
-    instr = convert->value();
-  }
-#endif
-
-  ShiftOp* shift = instr->as_ShiftOp();
-  if (shift != NULL) {
-    if (shift->op() == Bytecodes::_lshl) {
-      assert(shift->x()->type() == longType, "invalid input type");
-    } else {
-#ifndef _LP64
-      if (shift->op() == Bytecodes::_ishl) {
-        assert(shift->x()->type() == intType, "invalid input type");
-      } else {
-        return false;
-      }
-#else
-      return false;
-#endif
-    }
-
-
-    // Constant shift value?
-    Constant* con = shift->y()->as_Constant();
-    if (con == NULL) return false;
-    // Well-known type and value?
-    IntConstant* val = con->type()->as_IntConstant();
-    assert(val != NULL, "Should be an int constant");
-
-    *index = shift->x();
-    int tmp_scale = val->value();
-    if (tmp_scale >= 0 && tmp_scale < 4) {
-      *log2_scale = tmp_scale;
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  ArithmeticOp* arith = instr->as_ArithmeticOp();
-  if (arith != NULL) {
-    // See if either arg is a known constant
-    Constant* con = arith->x()->as_Constant();
-    if (con != NULL) {
-      *index = arith->y();
-    } else {
-      con = arith->y()->as_Constant();
-      if (con == NULL) return false;
-      *index = arith->x();
-    }
-    long const_value;
-    // Check for integer multiply
-    if (arith->op() == Bytecodes::_lmul) {
-      assert((*index)->type() == longType, "invalid input type");
-      LongConstant* val = con->type()->as_LongConstant();
-      assert(val != NULL, "expecting a long constant");
-      const_value = val->value();
-    } else {
-#ifndef _LP64
-      if (arith->op() == Bytecodes::_imul) {
-        assert((*index)->type() == intType, "invalid input type");
-        IntConstant* val = con->type()->as_IntConstant();
-        assert(val != NULL, "expecting an int constant");
-        const_value = val->value();
-      } else {
-        return false;
-      }
-#else
-      return false;
-#endif
-    }
-    switch (const_value) {
-    case 1: *log2_scale = 0; return true;
-    case 2: *log2_scale = 1; return true;
-    case 4: *log2_scale = 2; return true;
-    case 8: *log2_scale = 3; return true;
-    default:            return false;
-    }
-  }
-
-  // Unknown instruction sequence; don't touch it
-  return false;
-}
-
-
-static bool match(UnsafeRawOp* x,
-                  Instruction** base,
-                  Instruction** index,
-                  int*          log2_scale) {
-  ArithmeticOp* root = x->base()->as_ArithmeticOp();
-  if (root == NULL) return false;
-  // Limit ourselves to addition for now
-  if (root->op() != Bytecodes::_ladd) return false;
-
-  bool match_found = false;
-  // Try to find shift or scale op
-  if (match_index_and_scale(root->y(), index, log2_scale)) {
-    *base = root->x();
-    match_found = true;
-  } else if (match_index_and_scale(root->x(), index, log2_scale)) {
-    *base = root->y();
-    match_found = true;
-  } else if (NOT_LP64(root->y()->as_Convert() != NULL) LP64_ONLY(false)) {
-    // Skipping i2l works only on 32bit because of the implicit l2i that the unsafe performs.
-    // 64bit needs a real sign-extending conversion.
-    Convert* convert = root->y()->as_Convert();
-    if (convert->op() == Bytecodes::_i2l) {
-      assert(convert->value()->type() == intType, "should be an int");
-      // pick base and index, setting scale at 1
-      *base  = root->x();
-      *index = convert->value();
-      *log2_scale = 0;
-      match_found = true;
-    }
-  }
-  // The default solution
-  if (!match_found) {
-    *base = root->x();
-    *index = root->y();
-    *log2_scale = 0;
-  }
-
-  // If the value is pinned then it will be always be computed so
-  // there's no profit to reshaping the expression.
-  return !root->is_pinned();
-}
-
-
-void Canonicalizer::do_UnsafeRawOp(UnsafeRawOp* x) {
-  Instruction* base = NULL;
-  Instruction* index = NULL;
-  int          log2_scale;
-
-  if (match(x, &base, &index, &log2_scale)) {
-    x->set_base(base);
-    x->set_index(index);
-    x->set_log2_scale(log2_scale);
-    if (PrintUnsafeOptimization) {
-      tty->print_cr("Canonicalizer: UnsafeRawOp id %d: base = id %d, index = id %d, log2_scale = %d",
-                    x->id(), x->base()->id(), x->index()->id(), x->log2_scale());
-    }
-  }
-}
-
-void Canonicalizer::do_RoundFP(RoundFP* x) {}
-void Canonicalizer::do_UnsafeGetRaw(UnsafeGetRaw* x) { if (OptimizeUnsafes) do_UnsafeRawOp(x); }
-void Canonicalizer::do_UnsafePutRaw(UnsafePutRaw* x) { if (OptimizeUnsafes) do_UnsafeRawOp(x); }
-void Canonicalizer::do_UnsafeGetObject(UnsafeGetObject* x) {}
-void Canonicalizer::do_UnsafePutObject(UnsafePutObject* x) {}
-void Canonicalizer::do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) {}
-void Canonicalizer::do_ProfileCall(ProfileCall* x) {}
+void Canonicalizer::do_RoundFP        (RoundFP*         x) {}
+void Canonicalizer::do_UnsafeGet      (UnsafeGet*       x) {}
+void Canonicalizer::do_UnsafePut      (UnsafePut*       x) {}
+void Canonicalizer::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {}
+void Canonicalizer::do_ProfileCall    (ProfileCall*     x) {}
 void Canonicalizer::do_ProfileReturnType(ProfileReturnType* x) {}
-void Canonicalizer::do_ProfileInvoke(ProfileInvoke* x) {}
-void Canonicalizer::do_ProfileACmpTypes(ProfileACmpTypes* x) {}
-void Canonicalizer::do_RuntimeCall(RuntimeCall* x) {}
+void Canonicalizer::do_ProfileInvoke    (ProfileInvoke* x) {}
+void Canonicalizer::do_ProfileACmpTypes (ProfileACmpTypes* x) {}
+void Canonicalizer::do_RuntimeCall      (RuntimeCall* x) {}
 void Canonicalizer::do_RangeCheckPredicate(RangeCheckPredicate* x) {}
 #ifdef ASSERT
-void Canonicalizer::do_Assert(Assert* x) {}
+void Canonicalizer::do_Assert         (Assert*          x) {}
 #endif
-void Canonicalizer::do_MemBar(MemBar* x) {}
+void Canonicalizer::do_MemBar         (MemBar*          x) {}
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.hpp b/src/hotspot/share/c1/c1_Canonicalizer.hpp
index 27e684f86e5..ca9b86d445c 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.hpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.hpp
@@ -46,12 +46,6 @@ class Canonicalizer: InstructionVisitor {
 #endif
   void move_const_to_right(Op2* x);
   void do_Op2(Op2* x);
-  void do_UnsafeRawOp(UnsafeRawOp* x);
-
-  void unsafe_raw_match(UnsafeRawOp* x,
-                        Instruction** base,
-                        Instruction** index,
-                        int* scale);
 
  public:
   Canonicalizer(Compilation* c, Value x, int bci) : _compilation(c), _canonical(x), _bci(bci) {
@@ -101,11 +95,9 @@ class Canonicalizer: InstructionVisitor {
   virtual void do_OsrEntry       (OsrEntry*        x);
   virtual void do_ExceptionObject(ExceptionObject* x);
   virtual void do_RoundFP        (RoundFP*         x);
-  virtual void do_UnsafeGetRaw   (UnsafeGetRaw*    x);
-  virtual void do_UnsafePutRaw   (UnsafePutRaw*    x);
-  virtual void do_UnsafeGetObject(UnsafeGetObject* x);
-  virtual void do_UnsafePutObject(UnsafePutObject* x);
-  virtual void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x);
+  virtual void do_UnsafeGet      (UnsafeGet*       x);
+  virtual void do_UnsafePut      (UnsafePut*       x);
+  virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
   virtual void do_ProfileCall    (ProfileCall*     x);
   virtual void do_ProfileReturnType (ProfileReturnType*  x);
   virtual void do_ProfileACmpTypes(ProfileACmpTypes*  x);
diff --git a/src/hotspot/share/c1/c1_CodeStubs.hpp b/src/hotspot/share/c1/c1_CodeStubs.hpp
index afa86abaf80..23a5079813b 100644
--- a/src/hotspot/share/c1/c1_CodeStubs.hpp
+++ b/src/hotspot/share/c1/c1_CodeStubs.hpp
@@ -56,8 +56,6 @@ class CodeStub: public CompilationResourceObj {
   virtual void emit_code(LIR_Assembler* e) = 0;
   virtual CodeEmitInfo* info() const             { return NULL; }
   virtual bool is_exception_throw_stub() const   { return false; }
-  virtual bool is_range_check_stub() const       { return false; }
-  virtual bool is_divbyzero_stub() const         { return false; }
   virtual bool is_simple_exception_stub() const  { return false; }
 #ifndef PRODUCT
   virtual void print_name(outputStream* out) const = 0;
@@ -67,15 +65,7 @@ class CodeStub: public CompilationResourceObj {
   Label* entry()                                 { return &_entry; }
   Label* continuation()                          { return &_continuation; }
   // for LIR
-  virtual void visit(LIR_OpVisitState* visit) {
-#ifndef PRODUCT
-    if (LIRTracePeephole && Verbose) {
-      tty->print("no visitor for ");
-      print_name(tty);
-      tty->cr();
-    }
-#endif
-  }
+  virtual void visit(LIR_OpVisitState* visit) = 0;
 };
 
 class CodeStubList: public GrowableArray<CodeStub*> {
@@ -181,7 +171,6 @@ class RangeCheckStub: public CodeStub {
   virtual void emit_code(LIR_Assembler* e);
   virtual CodeEmitInfo* info() const             { return _info; }
   virtual bool is_exception_throw_stub() const   { return true; }
-  virtual bool is_range_check_stub() const       { return true; }
   virtual void visit(LIR_OpVisitState* visitor) {
     visitor->do_slow_case(_info);
     visitor->do_input(_index);
@@ -224,7 +213,6 @@ class DivByZeroStub: public CodeStub {
   virtual void emit_code(LIR_Assembler* e);
   virtual CodeEmitInfo* info() const             { return _info; }
   virtual bool is_exception_throw_stub() const   { return true; }
-  virtual bool is_divbyzero_stub() const         { return true; }
   virtual void visit(LIR_OpVisitState* visitor) {
     visitor->do_slow_case(_info);
   }
diff --git a/src/hotspot/share/c1/c1_Compilation.hpp b/src/hotspot/share/c1/c1_Compilation.hpp
index 4c518fd48e2..3121eaa0a5d 100644
--- a/src/hotspot/share/c1/c1_Compilation.hpp
+++ b/src/hotspot/share/c1/c1_Compilation.hpp
@@ -230,8 +230,6 @@ class Compilation: public StackObj {
     return env()->comp_level() == CompLevel_full_profile ||
            env()->comp_level() == CompLevel_limited_profile;
   }
-  bool count_invocations() { return is_profiling(); }
-  bool count_backedges()   { return is_profiling(); }
 
   // Helpers for generation of profile information
   bool profile_branches() {
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index 41e5500f9d7..dc6bba6e20a 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -2074,7 +2074,7 @@ void GraphBuilder::withfield(int field_index) {
   Value val = pop(type);
   Value obj = apop();
 
-  if (!holder->is_loaded()) {
+  if (!holder->is_loaded() || !holder->is_inlinetype()) {
     apush(append_split(new Deoptimize(holder, state_before)));
     return;
   }
@@ -2083,7 +2083,6 @@ void GraphBuilder::withfield(int field_index) {
   const bool needs_patching = !field_modify->will_link(method(), Bytecodes::_withfield) ||
                               (!field_modify->is_flattened() && PatchALot);
   const int offset_modify = !needs_patching ? field_modify->offset() : -1;
-  assert(holder->is_inlinetype(), "must be an inline klass");
 
   scope()->set_wrote_final();
   scope()->set_wrote_fields();
@@ -3441,7 +3440,7 @@ BlockBegin* GraphBuilder::setup_start_block(int osr_bci, BlockBegin* std_entry,
   // In addition, with range check elimination, we may need a valid block
   // that dominates all the rest to insert range predicates.
   BlockBegin* new_header_block;
-  if (std_entry->number_of_preds() > 0 || count_invocations() || count_backedges() || RangeCheckElimination) {
+  if (std_entry->number_of_preds() > 0 || is_profiling() || RangeCheckElimination) {
     new_header_block = header_block(std_entry, BlockBegin::std_entry_flag, state);
   } else {
     new_header_block = std_entry;
@@ -3528,10 +3527,11 @@ void GraphBuilder::setup_osr_entry_block() {
       // doesn't so pretend that the interpreter passed in null.
       get = append(new Constant(objectNull));
     } else {
-      get = append(new UnsafeGetRaw(as_BasicType(local->type()), e,
-                                    append(new Constant(new IntConstant(offset))),
-                                    0,
-                                    true /*unaligned*/, true /*wide*/));
+      Value off_val = append(new Constant(new IntConstant(offset)));
+      get = append(new UnsafeGet(as_BasicType(local->type()), e,
+                                 off_val,
+                                 false/*is_volatile*/,
+                                 true/*is_raw*/));
     }
     _state->store_local(index, get);
   }
@@ -3861,60 +3861,60 @@ void GraphBuilder::build_graph_for_intrinsic(ciMethod* callee, bool ignore_retur
 
   // Some intrinsics need special IR nodes.
   switch(id) {
-  case vmIntrinsics::_getReference       : append_unsafe_get_obj(callee, T_OBJECT,  false); return;
-  case vmIntrinsics::_getBoolean         : append_unsafe_get_obj(callee, T_BOOLEAN, false); return;
-  case vmIntrinsics::_getByte            : append_unsafe_get_obj(callee, T_BYTE,    false); return;
-  case vmIntrinsics::_getShort           : append_unsafe_get_obj(callee, T_SHORT,   false); return;
-  case vmIntrinsics::_getChar            : append_unsafe_get_obj(callee, T_CHAR,    false); return;
-  case vmIntrinsics::_getInt             : append_unsafe_get_obj(callee, T_INT,     false); return;
-  case vmIntrinsics::_getLong            : append_unsafe_get_obj(callee, T_LONG,    false); return;
-  case vmIntrinsics::_getFloat           : append_unsafe_get_obj(callee, T_FLOAT,   false); return;
-  case vmIntrinsics::_getDouble          : append_unsafe_get_obj(callee, T_DOUBLE,  false); return;
-  case vmIntrinsics::_putReference       : append_unsafe_put_obj(callee, T_OBJECT,  false); return;
-  case vmIntrinsics::_putBoolean         : append_unsafe_put_obj(callee, T_BOOLEAN, false); return;
-  case vmIntrinsics::_putByte            : append_unsafe_put_obj(callee, T_BYTE,    false); return;
-  case vmIntrinsics::_putShort           : append_unsafe_put_obj(callee, T_SHORT,   false); return;
-  case vmIntrinsics::_putChar            : append_unsafe_put_obj(callee, T_CHAR,    false); return;
-  case vmIntrinsics::_putInt             : append_unsafe_put_obj(callee, T_INT,     false); return;
-  case vmIntrinsics::_putLong            : append_unsafe_put_obj(callee, T_LONG,    false); return;
-  case vmIntrinsics::_putFloat           : append_unsafe_put_obj(callee, T_FLOAT,   false); return;
-  case vmIntrinsics::_putDouble          : append_unsafe_put_obj(callee, T_DOUBLE,  false); return;
-  case vmIntrinsics::_getShortUnaligned  : append_unsafe_get_obj(callee, T_SHORT,   false); return;
-  case vmIntrinsics::_getCharUnaligned   : append_unsafe_get_obj(callee, T_CHAR,    false); return;
-  case vmIntrinsics::_getIntUnaligned    : append_unsafe_get_obj(callee, T_INT,     false); return;
-  case vmIntrinsics::_getLongUnaligned   : append_unsafe_get_obj(callee, T_LONG,    false); return;
-  case vmIntrinsics::_putShortUnaligned  : append_unsafe_put_obj(callee, T_SHORT,   false); return;
-  case vmIntrinsics::_putCharUnaligned   : append_unsafe_put_obj(callee, T_CHAR,    false); return;
-  case vmIntrinsics::_putIntUnaligned    : append_unsafe_put_obj(callee, T_INT,     false); return;
-  case vmIntrinsics::_putLongUnaligned   : append_unsafe_put_obj(callee, T_LONG,    false); return;
-  case vmIntrinsics::_getReferenceVolatile  : append_unsafe_get_obj(callee, T_OBJECT,  true); return;
-  case vmIntrinsics::_getBooleanVolatile : append_unsafe_get_obj(callee, T_BOOLEAN, true); return;
-  case vmIntrinsics::_getByteVolatile    : append_unsafe_get_obj(callee, T_BYTE,    true); return;
-  case vmIntrinsics::_getShortVolatile   : append_unsafe_get_obj(callee, T_SHORT,   true); return;
-  case vmIntrinsics::_getCharVolatile    : append_unsafe_get_obj(callee, T_CHAR,    true); return;
-  case vmIntrinsics::_getIntVolatile     : append_unsafe_get_obj(callee, T_INT,     true); return;
-  case vmIntrinsics::_getLongVolatile    : append_unsafe_get_obj(callee, T_LONG,    true); return;
-  case vmIntrinsics::_getFloatVolatile   : append_unsafe_get_obj(callee, T_FLOAT,   true); return;
-  case vmIntrinsics::_getDoubleVolatile  : append_unsafe_get_obj(callee, T_DOUBLE,  true); return;
-  case vmIntrinsics::_putReferenceVolatile : append_unsafe_put_obj(callee, T_OBJECT,  true); return;
-  case vmIntrinsics::_putBooleanVolatile : append_unsafe_put_obj(callee, T_BOOLEAN, true); return;
-  case vmIntrinsics::_putByteVolatile    : append_unsafe_put_obj(callee, T_BYTE,    true); return;
-  case vmIntrinsics::_putShortVolatile   : append_unsafe_put_obj(callee, T_SHORT,   true); return;
-  case vmIntrinsics::_putCharVolatile    : append_unsafe_put_obj(callee, T_CHAR,    true); return;
-  case vmIntrinsics::_putIntVolatile     : append_unsafe_put_obj(callee, T_INT,     true); return;
-  case vmIntrinsics::_putLongVolatile    : append_unsafe_put_obj(callee, T_LONG,    true); return;
-  case vmIntrinsics::_putFloatVolatile   : append_unsafe_put_obj(callee, T_FLOAT,   true); return;
-  case vmIntrinsics::_putDoubleVolatile  : append_unsafe_put_obj(callee, T_DOUBLE,  true); return;
+  case vmIntrinsics::_getReference           : append_unsafe_get(callee, T_OBJECT,  false); return;
+  case vmIntrinsics::_getBoolean             : append_unsafe_get(callee, T_BOOLEAN, false); return;
+  case vmIntrinsics::_getByte                : append_unsafe_get(callee, T_BYTE,    false); return;
+  case vmIntrinsics::_getShort               : append_unsafe_get(callee, T_SHORT,   false); return;
+  case vmIntrinsics::_getChar                : append_unsafe_get(callee, T_CHAR,    false); return;
+  case vmIntrinsics::_getInt                 : append_unsafe_get(callee, T_INT,     false); return;
+  case vmIntrinsics::_getLong                : append_unsafe_get(callee, T_LONG,    false); return;
+  case vmIntrinsics::_getFloat               : append_unsafe_get(callee, T_FLOAT,   false); return;
+  case vmIntrinsics::_getDouble              : append_unsafe_get(callee, T_DOUBLE,  false); return;
+  case vmIntrinsics::_putReference           : append_unsafe_put(callee, T_OBJECT,  false); return;
+  case vmIntrinsics::_putBoolean             : append_unsafe_put(callee, T_BOOLEAN, false); return;
+  case vmIntrinsics::_putByte                : append_unsafe_put(callee, T_BYTE,    false); return;
+  case vmIntrinsics::_putShort               : append_unsafe_put(callee, T_SHORT,   false); return;
+  case vmIntrinsics::_putChar                : append_unsafe_put(callee, T_CHAR,    false); return;
+  case vmIntrinsics::_putInt                 : append_unsafe_put(callee, T_INT,     false); return;
+  case vmIntrinsics::_putLong                : append_unsafe_put(callee, T_LONG,    false); return;
+  case vmIntrinsics::_putFloat               : append_unsafe_put(callee, T_FLOAT,   false); return;
+  case vmIntrinsics::_putDouble              : append_unsafe_put(callee, T_DOUBLE,  false); return;
+  case vmIntrinsics::_getShortUnaligned      : append_unsafe_get(callee, T_SHORT,   false); return;
+  case vmIntrinsics::_getCharUnaligned       : append_unsafe_get(callee, T_CHAR,    false); return;
+  case vmIntrinsics::_getIntUnaligned        : append_unsafe_get(callee, T_INT,     false); return;
+  case vmIntrinsics::_getLongUnaligned       : append_unsafe_get(callee, T_LONG,    false); return;
+  case vmIntrinsics::_putShortUnaligned      : append_unsafe_put(callee, T_SHORT,   false); return;
+  case vmIntrinsics::_putCharUnaligned       : append_unsafe_put(callee, T_CHAR,    false); return;
+  case vmIntrinsics::_putIntUnaligned        : append_unsafe_put(callee, T_INT,     false); return;
+  case vmIntrinsics::_putLongUnaligned       : append_unsafe_put(callee, T_LONG,    false); return;
+  case vmIntrinsics::_getReferenceVolatile   : append_unsafe_get(callee, T_OBJECT,  true); return;
+  case vmIntrinsics::_getBooleanVolatile     : append_unsafe_get(callee, T_BOOLEAN, true); return;
+  case vmIntrinsics::_getByteVolatile        : append_unsafe_get(callee, T_BYTE,    true); return;
+  case vmIntrinsics::_getShortVolatile       : append_unsafe_get(callee, T_SHORT,   true); return;
+  case vmIntrinsics::_getCharVolatile        : append_unsafe_get(callee, T_CHAR,    true); return;
+  case vmIntrinsics::_getIntVolatile         : append_unsafe_get(callee, T_INT,     true); return;
+  case vmIntrinsics::_getLongVolatile        : append_unsafe_get(callee, T_LONG,    true); return;
+  case vmIntrinsics::_getFloatVolatile       : append_unsafe_get(callee, T_FLOAT,   true); return;
+  case vmIntrinsics::_getDoubleVolatile      : append_unsafe_get(callee, T_DOUBLE,  true); return;
+  case vmIntrinsics::_putReferenceVolatile   : append_unsafe_put(callee, T_OBJECT,  true); return;
+  case vmIntrinsics::_putBooleanVolatile     : append_unsafe_put(callee, T_BOOLEAN, true); return;
+  case vmIntrinsics::_putByteVolatile        : append_unsafe_put(callee, T_BYTE,    true); return;
+  case vmIntrinsics::_putShortVolatile       : append_unsafe_put(callee, T_SHORT,   true); return;
+  case vmIntrinsics::_putCharVolatile        : append_unsafe_put(callee, T_CHAR,    true); return;
+  case vmIntrinsics::_putIntVolatile         : append_unsafe_put(callee, T_INT,     true); return;
+  case vmIntrinsics::_putLongVolatile        : append_unsafe_put(callee, T_LONG,    true); return;
+  case vmIntrinsics::_putFloatVolatile       : append_unsafe_put(callee, T_FLOAT,   true); return;
+  case vmIntrinsics::_putDoubleVolatile      : append_unsafe_put(callee, T_DOUBLE,  true); return;
   case vmIntrinsics::_compareAndSetLong:
   case vmIntrinsics::_compareAndSetInt:
   case vmIntrinsics::_compareAndSetReference : append_unsafe_CAS(callee); return;
   case vmIntrinsics::_getAndAddInt:
-  case vmIntrinsics::_getAndAddLong      : append_unsafe_get_and_set_obj(callee, true); return;
-  case vmIntrinsics::_getAndSetInt       :
-  case vmIntrinsics::_getAndSetLong      :
-  case vmIntrinsics::_getAndSetReference : append_unsafe_get_and_set_obj(callee, false); return;
-  case vmIntrinsics::_getCharStringU     : append_char_access(callee, false); return;
-  case vmIntrinsics::_putCharStringU     : append_char_access(callee, true); return;
+  case vmIntrinsics::_getAndAddLong          : append_unsafe_get_and_set(callee, true); return;
+  case vmIntrinsics::_getAndSetInt           :
+  case vmIntrinsics::_getAndSetLong          :
+  case vmIntrinsics::_getAndSetReference     : append_unsafe_get_and_set(callee, false); return;
+  case vmIntrinsics::_getCharStringU         : append_char_access(callee, false); return;
+  case vmIntrinsics::_putCharStringU         : append_char_access(callee, true); return;
   default:
     break;
   }
@@ -4592,20 +4592,20 @@ void GraphBuilder::pop_scope_for_jsr() {
   _scope_data = scope_data()->parent();
 }
 
-void GraphBuilder::append_unsafe_get_obj(ciMethod* callee, BasicType t, bool is_volatile) {
+void GraphBuilder::append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile) {
   Values* args = state()->pop_arguments(callee->arg_size());
   null_check(args->at(0));
   Instruction* offset = args->at(2);
 #ifndef _LP64
   offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT)));
 #endif
-  Instruction* op = append(new UnsafeGetObject(t, args->at(1), offset, is_volatile));
+  Instruction* op = append(new UnsafeGet(t, args->at(1), offset, is_volatile));
   push(op->type(), op);
   compilation()->set_has_unsafe_access(true);
 }
 
 
-void GraphBuilder::append_unsafe_put_obj(ciMethod* callee, BasicType t, bool is_volatile) {
+void GraphBuilder::append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile) {
   Values* args = state()->pop_arguments(callee->arg_size());
   null_check(args->at(0));
   Instruction* offset = args->at(2);
@@ -4617,29 +4617,11 @@ void GraphBuilder::append_unsafe_put_obj(ciMethod* callee, BasicType t, bool is_
     Value mask = append(new Constant(new IntConstant(1)));
     val = append(new LogicOp(Bytecodes::_iand, val, mask));
   }
-  Instruction* op = append(new UnsafePutObject(t, args->at(1), offset, val, is_volatile));
+  Instruction* op = append(new UnsafePut(t, args->at(1), offset, val, is_volatile));
   compilation()->set_has_unsafe_access(true);
   kill_all();
 }
 
-
-void GraphBuilder::append_unsafe_get_raw(ciMethod* callee, BasicType t) {
-  Values* args = state()->pop_arguments(callee->arg_size());
-  null_check(args->at(0));
-  Instruction* op = append(new UnsafeGetRaw(t, args->at(1), false));
-  push(op->type(), op);
-  compilation()->set_has_unsafe_access(true);
-}
-
-
-void GraphBuilder::append_unsafe_put_raw(ciMethod* callee, BasicType t) {
-  Values* args = state()->pop_arguments(callee->arg_size());
-  null_check(args->at(0));
-  Instruction* op = append(new UnsafePutRaw(t, args->at(1), args->at(2)));
-  compilation()->set_has_unsafe_access(true);
-}
-
-
 void GraphBuilder::append_unsafe_CAS(ciMethod* callee) {
   ValueStack* state_before = copy_state_for_exception();
   ValueType* result_type = as_ValueType(callee->return_type());
@@ -4727,7 +4709,7 @@ void GraphBuilder::print_inlining(ciMethod* callee, const char* msg, bool succes
   }
 }
 
-void GraphBuilder::append_unsafe_get_and_set_obj(ciMethod* callee, bool is_add) {
+void GraphBuilder::append_unsafe_get_and_set(ciMethod* callee, bool is_add) {
   Values* args = state()->pop_arguments(callee->arg_size());
   BasicType t = callee->return_type()->basic_type();
   null_check(args->at(0));
@@ -4735,7 +4717,7 @@ void GraphBuilder::append_unsafe_get_and_set_obj(ciMethod* callee, bool is_add)
 #ifndef _LP64
   offset = append(new Convert(Bytecodes::_l2i, offset, as_ValueType(T_INT)));
 #endif
-  Instruction* op = append(new UnsafeGetAndSetObject(t, args->at(1), offset, args->at(3), is_add));
+  Instruction* op = append(new UnsafeGetAndSet(t, args->at(1), offset, args->at(3), is_add));
   compilation()->set_has_unsafe_access(true);
   kill_all();
   push(op->type(), op);
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.hpp b/src/hotspot/share/c1/c1_GraphBuilder.hpp
index 636e0524efa..2eaf7772f21 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.hpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.hpp
@@ -416,12 +416,10 @@ class GraphBuilder {
   void pop_scope();
   void pop_scope_for_jsr();
 
-  void append_unsafe_get_obj(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_put_obj(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_get_raw(ciMethod* callee, BasicType t);
-  void append_unsafe_put_raw(ciMethod* callee, BasicType t);
+  void append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile);
+  void append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile);
   void append_unsafe_CAS(ciMethod* callee);
-  void append_unsafe_get_and_set_obj(ciMethod* callee, bool is_add);
+  void append_unsafe_get_and_set(ciMethod* callee, bool is_add);
   void append_char_access(ciMethod* callee, bool is_store);
 
   void print_inlining(ciMethod* callee, const char* msg, bool success = true);
@@ -432,8 +430,6 @@ class GraphBuilder {
 
   // Shortcuts to profiling control.
   bool is_profiling()          { return _compilation->is_profiling();          }
-  bool count_invocations()     { return _compilation->count_invocations();     }
-  bool count_backedges()       { return _compilation->count_backedges();       }
   bool profile_branches()      { return _compilation->profile_branches();      }
   bool profile_calls()         { return _compilation->profile_calls();         }
   bool profile_inlined_calls() { return _compilation->profile_inlined_calls(); }
diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp
index 20c68fad4f6..b7e4ad1e5a9 100644
--- a/src/hotspot/share/c1/c1_Instruction.hpp
+++ b/src/hotspot/share/c1/c1_Instruction.hpp
@@ -97,13 +97,9 @@ class       Throw;
 class       Base;
 class   RoundFP;
 class   UnsafeOp;
-class     UnsafeRawOp;
-class       UnsafeGetRaw;
-class       UnsafePutRaw;
-class     UnsafeObjectOp;
-class       UnsafeGetObject;
-class       UnsafePutObject;
-class         UnsafeGetAndSetObject;
+class     UnsafeGet;
+class     UnsafePut;
+class     UnsafeGetAndSet;
 class   ProfileCall;
 class   ProfileReturnType;
 class   ProfileACmpTypes;
@@ -200,11 +196,9 @@ class InstructionVisitor: public StackObj {
   virtual void do_OsrEntry       (OsrEntry*        x) = 0;
   virtual void do_ExceptionObject(ExceptionObject* x) = 0;
   virtual void do_RoundFP        (RoundFP*         x) = 0;
-  virtual void do_UnsafeGetRaw   (UnsafeGetRaw*    x) = 0;
-  virtual void do_UnsafePutRaw   (UnsafePutRaw*    x) = 0;
-  virtual void do_UnsafeGetObject(UnsafeGetObject* x) = 0;
-  virtual void do_UnsafePutObject(UnsafePutObject* x) = 0;
-  virtual void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) = 0;
+  virtual void do_UnsafeGet      (UnsafeGet*       x) = 0;
+  virtual void do_UnsafePut      (UnsafePut*       x) = 0;
+  virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x) = 0;
   virtual void do_ProfileCall    (ProfileCall*     x) = 0;
   virtual void do_ProfileReturnType (ProfileReturnType*  x) = 0;
   virtual void do_ProfileACmpTypes(ProfileACmpTypes*  x) = 0;
@@ -2337,13 +2331,16 @@ LEAF(RoundFP, Instruction)
 
 BASE(UnsafeOp, Instruction)
  private:
-  BasicType _basic_type;    // ValueType can not express byte-sized integers
+  Value _object;                                 // Object to be fetched from or mutated
+  Value _offset;                                 // Offset within object
+  bool  _is_volatile;                            // true if volatile - dl/JSR166
+  BasicType _basic_type;                         // ValueType can not express byte-sized integers
 
  protected:
   // creation
-  UnsafeOp(BasicType basic_type, bool is_put)
-  : Instruction(is_put ? voidType : as_ValueType(basic_type))
-  , _basic_type(basic_type)
+  UnsafeOp(BasicType basic_type, Value object, Value offset, bool is_put, bool is_volatile)
+    : Instruction(is_put ? voidType : as_ValueType(basic_type)),
+    _object(object), _offset(offset), _is_volatile(is_volatile), _basic_type(basic_type)
   {
     //Note:  Unsafe ops are not not guaranteed to throw NPE.
     // Convservatively, Unsafe operations must be pinned though we could be
@@ -2354,148 +2351,42 @@ BASE(UnsafeOp, Instruction)
  public:
   // accessors
   BasicType basic_type()                         { return _basic_type; }
+  Value object()                                 { return _object; }
+  Value offset()                                 { return _offset; }
+  bool  is_volatile()                            { return _is_volatile; }
 
   // generic
-  virtual void input_values_do(ValueVisitor* f)   { }
-};
-
-
-BASE(UnsafeRawOp, UnsafeOp)
- private:
-  Value _base;                                   // Base address (a Java long)
-  Value _index;                                  // Index if computed by optimizer; initialized to NULL
-  int   _log2_scale;                             // Scale factor: 0, 1, 2, or 3.
-                                                 // Indicates log2 of number of bytes (1, 2, 4, or 8)
-                                                 // to scale index by.
-
- protected:
-  UnsafeRawOp(BasicType basic_type, Value addr, bool is_put)
-  : UnsafeOp(basic_type, is_put)
-  , _base(addr)
-  , _index(NULL)
-  , _log2_scale(0)
-  {
-    // Can not use ASSERT_VALUES because index may be NULL
-    assert(addr != NULL && addr->type()->is_long(), "just checking");
-  }
-
-  UnsafeRawOp(BasicType basic_type, Value base, Value index, int log2_scale, bool is_put)
-  : UnsafeOp(basic_type, is_put)
-  , _base(base)
-  , _index(index)
-  , _log2_scale(log2_scale)
-  {
-  }
-
- public:
-  // accessors
-  Value base()                                   { return _base; }
-  Value index()                                  { return _index; }
-  bool  has_index()                              { return (_index != NULL); }
-  int   log2_scale()                             { return _log2_scale; }
-
-  // setters
-  void set_base (Value base)                     { _base  = base; }
-  void set_index(Value index)                    { _index = index; }
-  void set_log2_scale(int log2_scale)            { _log2_scale = log2_scale; }
-
-  // generic
-  virtual void input_values_do(ValueVisitor* f)   { UnsafeOp::input_values_do(f);
-                                                   f->visit(&_base);
-                                                   if (has_index()) f->visit(&_index); }
+  virtual void input_values_do(ValueVisitor* f)   { f->visit(&_object);
+                                                    f->visit(&_offset); }
 };
 
-
-LEAF(UnsafeGetRaw, UnsafeRawOp)
+LEAF(UnsafeGet, UnsafeOp)
  private:
- bool _may_be_unaligned, _is_wide;  // For OSREntry
-
+  bool _is_raw;
  public:
- UnsafeGetRaw(BasicType basic_type, Value addr, bool may_be_unaligned, bool is_wide = false)
-  : UnsafeRawOp(basic_type, addr, false) {
-    _may_be_unaligned = may_be_unaligned;
-    _is_wide = is_wide;
-  }
-
- UnsafeGetRaw(BasicType basic_type, Value base, Value index, int log2_scale, bool may_be_unaligned, bool is_wide = false)
-  : UnsafeRawOp(basic_type, base, index, log2_scale, false) {
-    _may_be_unaligned = may_be_unaligned;
-    _is_wide = is_wide;
-  }
-
-  bool may_be_unaligned()                         { return _may_be_unaligned; }
-  bool is_wide()                                  { return _is_wide; }
-};
-
-
-LEAF(UnsafePutRaw, UnsafeRawOp)
- private:
-  Value _value;                                  // Value to be stored
-
- public:
-  UnsafePutRaw(BasicType basic_type, Value addr, Value value)
-  : UnsafeRawOp(basic_type, addr, true)
-  , _value(value)
+  UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile)
+  : UnsafeOp(basic_type, object, offset, false, is_volatile)
   {
-    assert(value != NULL, "just checking");
     ASSERT_VALUES
+    _is_raw = false;
   }
-
-  UnsafePutRaw(BasicType basic_type, Value base, Value index, int log2_scale, Value value)
-  : UnsafeRawOp(basic_type, base, index, log2_scale, true)
-  , _value(value)
+  UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile, bool is_raw)
+  : UnsafeOp(basic_type, object, offset, false, is_volatile), _is_raw(is_raw)
   {
-    assert(value != NULL, "just checking");
     ASSERT_VALUES
   }
 
   // accessors
-  Value value()                                  { return _value; }
-
-  // generic
-  virtual void input_values_do(ValueVisitor* f)   { UnsafeRawOp::input_values_do(f);
-                                                   f->visit(&_value); }
+  bool is_raw()                             { return _is_raw; }
 };
 
 
-BASE(UnsafeObjectOp, UnsafeOp)
- private:
-  Value _object;                                 // Object to be fetched from or mutated
-  Value _offset;                                 // Offset within object
-  bool  _is_volatile;                            // true if volatile - dl/JSR166
- public:
-  UnsafeObjectOp(BasicType basic_type, Value object, Value offset, bool is_put, bool is_volatile)
-    : UnsafeOp(basic_type, is_put), _object(object), _offset(offset), _is_volatile(is_volatile)
-  {
-  }
-
-  // accessors
-  Value object()                                 { return _object; }
-  Value offset()                                 { return _offset; }
-  bool  is_volatile()                            { return _is_volatile; }
-  // generic
-  virtual void input_values_do(ValueVisitor* f)   { UnsafeOp::input_values_do(f);
-                                                   f->visit(&_object);
-                                                   f->visit(&_offset); }
-};
-
-
-LEAF(UnsafeGetObject, UnsafeObjectOp)
- public:
-  UnsafeGetObject(BasicType basic_type, Value object, Value offset, bool is_volatile)
-  : UnsafeObjectOp(basic_type, object, offset, false, is_volatile)
-  {
-    ASSERT_VALUES
-  }
-};
-
-
-LEAF(UnsafePutObject, UnsafeObjectOp)
+LEAF(UnsafePut, UnsafeOp)
  private:
   Value _value;                                  // Value to be stored
  public:
-  UnsafePutObject(BasicType basic_type, Value object, Value offset, Value value, bool is_volatile)
-  : UnsafeObjectOp(basic_type, object, offset, true, is_volatile)
+  UnsafePut(BasicType basic_type, Value object, Value offset, Value value, bool is_volatile)
+  : UnsafeOp(basic_type, object, offset, true, is_volatile)
     , _value(value)
   {
     ASSERT_VALUES
@@ -2505,17 +2396,17 @@ LEAF(UnsafePutObject, UnsafeObjectOp)
   Value value()                                  { return _value; }
 
   // generic
-  virtual void input_values_do(ValueVisitor* f)   { UnsafeObjectOp::input_values_do(f);
+  virtual void input_values_do(ValueVisitor* f)   { UnsafeOp::input_values_do(f);
                                                    f->visit(&_value); }
 };
 
-LEAF(UnsafeGetAndSetObject, UnsafeObjectOp)
+LEAF(UnsafeGetAndSet, UnsafeOp)
  private:
   Value _value;                                  // Value to be stored
   bool  _is_add;
  public:
-  UnsafeGetAndSetObject(BasicType basic_type, Value object, Value offset, Value value, bool is_add)
-  : UnsafeObjectOp(basic_type, object, offset, false, false)
+  UnsafeGetAndSet(BasicType basic_type, Value object, Value offset, Value value, bool is_add)
+  : UnsafeOp(basic_type, object, offset, false, false)
     , _value(value)
     , _is_add(is_add)
   {
@@ -2527,7 +2418,7 @@ LEAF(UnsafeGetAndSetObject, UnsafeObjectOp)
   Value value()                                  { return _value; }
 
   // generic
-  virtual void input_values_do(ValueVisitor* f)   { UnsafeObjectOp::input_values_do(f);
+  virtual void input_values_do(ValueVisitor* f)   { UnsafeOp::input_values_do(f);
                                                    f->visit(&_value); }
 };
 
diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.cpp b/src/hotspot/share/c1/c1_InstructionPrinter.cpp
index ef61fb3f358..6071b03759d 100644
--- a/src/hotspot/share/c1/c1_InstructionPrinter.cpp
+++ b/src/hotspot/share/c1/c1_InstructionPrinter.cpp
@@ -264,22 +264,6 @@ void InstructionPrinter::print_inline_level(BlockBegin* block) {
 
 void InstructionPrinter::print_unsafe_op(UnsafeOp* op, const char* name) {
   output()->print("%s", name);
-  output()->print(".(");
-}
-
-void InstructionPrinter::print_unsafe_raw_op(UnsafeRawOp* op, const char* name) {
-  print_unsafe_op(op, name);
-  output()->print("base ");
-  print_value(op->base());
-  if (op->has_index()) {
-    output()->print(", index "); print_value(op->index());
-    output()->print(", log2_scale %d", op->log2_scale());
-  }
-}
-
-
-void InstructionPrinter::print_unsafe_object_op(UnsafeObjectOp* op, const char* name) {
-  print_unsafe_op(op, name);
   print_value(op->object());
   output()->print(", ");
   print_value(op->offset());
@@ -824,36 +808,20 @@ void InstructionPrinter::do_RoundFP(RoundFP* x) {
   print_value(x->input());
 }
 
-
-void InstructionPrinter::do_UnsafeGetRaw(UnsafeGetRaw* x) {
-  print_unsafe_raw_op(x, "UnsafeGetRaw");
-  output()->put(')');
-}
-
-
-void InstructionPrinter::do_UnsafePutRaw(UnsafePutRaw* x) {
-  print_unsafe_raw_op(x, "UnsafePutRaw");
-  output()->print(", value ");
-  print_value(x->value());
+void InstructionPrinter::do_UnsafeGet(UnsafeGet* x) {
+  print_unsafe_op(x, x->is_raw() ? "UnsafeGet (raw)" : "UnsafeGet");
   output()->put(')');
 }
 
-
-void InstructionPrinter::do_UnsafeGetObject(UnsafeGetObject* x) {
-  print_unsafe_object_op(x, "UnsafeGetObject");
-  output()->put(')');
-}
-
-
-void InstructionPrinter::do_UnsafePutObject(UnsafePutObject* x) {
-  print_unsafe_object_op(x, "UnsafePutObject");
+void InstructionPrinter::do_UnsafePut(UnsafePut* x) {
+  print_unsafe_op(x, "UnsafePut");
   output()->print(", value ");
   print_value(x->value());
   output()->put(')');
 }
 
-void InstructionPrinter::do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) {
-  print_unsafe_object_op(x, x->is_add()?"UnsafeGetAndSetObject (add)":"UnsafeGetAndSetObject");
+void InstructionPrinter::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {
+  print_unsafe_op(x, x->is_add()?"UnsafeGetAndSet (add)":"UnsafeGetAndSet");
   output()->print(", value ");
   print_value(x->value());
   output()->put(')');
diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.hpp b/src/hotspot/share/c1/c1_InstructionPrinter.hpp
index 4300a89aa64..631a090cb9e 100644
--- a/src/hotspot/share/c1/c1_InstructionPrinter.hpp
+++ b/src/hotspot/share/c1/c1_InstructionPrinter.hpp
@@ -74,8 +74,6 @@ class InstructionPrinter: public InstructionVisitor {
   void print_stack(ValueStack* stack);
   void print_inline_level(BlockBegin* block);
   void print_unsafe_op(UnsafeOp* op, const char* name);
-  void print_unsafe_raw_op(UnsafeRawOp* op, const char* name);
-  void print_unsafe_object_op(UnsafeObjectOp* op, const char* name);
   void print_phi(int i, Value v, BlockBegin* b);
   void print_alias(Value v);
 
@@ -125,11 +123,9 @@ class InstructionPrinter: public InstructionVisitor {
   virtual void do_OsrEntry       (OsrEntry*        x);
   virtual void do_ExceptionObject(ExceptionObject* x);
   virtual void do_RoundFP        (RoundFP*         x);
-  virtual void do_UnsafeGetRaw   (UnsafeGetRaw*    x);
-  virtual void do_UnsafePutRaw   (UnsafePutRaw*    x);
-  virtual void do_UnsafeGetObject(UnsafeGetObject* x);
-  virtual void do_UnsafePutObject(UnsafePutObject* x);
-  virtual void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x);
+  virtual void do_UnsafeGet      (UnsafeGet*       x);
+  virtual void do_UnsafePut      (UnsafePut*       x);
+  virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
   virtual void do_ProfileCall    (ProfileCall*     x);
   virtual void do_ProfileReturnType (ProfileReturnType*  x);
   virtual void do_ProfileACmpTypes(ProfileACmpTypes*  x);
diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp
index bbb348f55c7..e16a17fc7df 100644
--- a/src/hotspot/share/c1/c1_LIR.cpp
+++ b/src/hotspot/share/c1/c1_LIR.cpp
@@ -439,7 +439,6 @@ void LIR_OpVisitState::visit(LIR_Op* op) {
   switch (op->code()) {
 
 // LIR_Op0
-    case lir_backwardbranch_target:    // result and info always invalid
     case lir_fpop_raw:                 // result and info always invalid
     case lir_breakpoint:               // result and info always invalid
     case lir_membar:                   // result and info always invalid
@@ -1811,7 +1810,6 @@ const char * LIR_Op::name() const {
      case lir_label:                 s = "label";         break;
      case lir_nop:                   s = "nop";           break;
      case lir_on_spin_wait:          s = "on_spin_wait";  break;
-     case lir_backwardbranch_target: s = "backbranch";    break;
      case lir_std_entry:             s = "std_entry";     break;
      case lir_osr_entry:             s = "osr_entry";     break;
      case lir_fpop_raw:              s = "fpop_raw";      break;
@@ -1964,8 +1962,6 @@ const char * LIR_Op1::name() const {
     switch (move_kind()) {
     case lir_move_normal:
       return "move";
-    case lir_move_unaligned:
-      return "unaligned move";
     case lir_move_volatile:
       return "volatile_move";
     case lir_move_wide:
diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp
index 27afd516581..0c60ffe1804 100644
--- a/src/hotspot/share/c1/c1_LIR.hpp
+++ b/src/hotspot/share/c1/c1_LIR.hpp
@@ -896,7 +896,6 @@ enum LIR_Code {
   , begin_op0
       , lir_label
       , lir_nop
-      , lir_backwardbranch_target
       , lir_std_entry
       , lir_osr_entry
       , lir_fpop_raw
@@ -1038,7 +1037,6 @@ enum LIR_PatchCode {
 enum LIR_MoveKind {
   lir_move_normal,
   lir_move_volatile,
-  lir_move_unaligned,
   lir_move_wide,
   lir_move_max_flag
 };
@@ -2214,9 +2212,6 @@ class LIR_List: public CompilationResourceObj {
   // result is a stack location for old backend and vreg for UseLinearScan
   // stack_loc_temp is an illegal register for old backend
   void roundfp(LIR_Opr reg, LIR_Opr stack_loc_temp, LIR_Opr result) { append(new LIR_OpRoundFP(reg, stack_loc_temp, result)); }
-  void unaligned_move(LIR_Address* src, LIR_Opr dst) { append(new LIR_Op1(lir_move, LIR_OprFact::address(src), dst, dst->type(), lir_patch_none, NULL, lir_move_unaligned)); }
-  void unaligned_move(LIR_Opr src, LIR_Address* dst) { append(new LIR_Op1(lir_move, src, LIR_OprFact::address(dst), src->type(), lir_patch_none, NULL, lir_move_unaligned)); }
-  void unaligned_move(LIR_Opr src, LIR_Opr dst) { append(new LIR_Op1(lir_move, src, dst, dst->type(), lir_patch_none, NULL, lir_move_unaligned)); }
   void move(LIR_Opr src, LIR_Opr dst, CodeEmitInfo* info = NULL) { append(new LIR_Op1(lir_move, src, dst, dst->type(), lir_patch_none, info)); }
   void move(LIR_Address* src, LIR_Opr dst, CodeEmitInfo* info = NULL) { append(new LIR_Op1(lir_move, LIR_OprFact::address(src), dst, src->type(), lir_patch_none, info)); }
   void move(LIR_Opr src, LIR_Address* dst, CodeEmitInfo* info = NULL) { append(new LIR_Op1(lir_move, src, LIR_OprFact::address(dst), dst->type(), lir_patch_none, info)); }
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp
index 2090785a5f4..659b0cf7a45 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.cpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp
@@ -518,7 +518,6 @@ void LIR_Assembler::emit_op1(LIR_Op1* op) {
       } else {
         move_op(op->in_opr(), op->result_opr(), op->type(),
                 op->patch_code(), op->info(), op->pop_fpu_stack(),
-                op->move_kind() == lir_move_unaligned,
                 op->move_kind() == lir_move_wide);
       }
       break;
@@ -908,7 +907,7 @@ void LIR_Assembler::roundfp_op(LIR_Opr src, LIR_Opr tmp, LIR_Opr dest, bool pop_
 }
 
 
-void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool unaligned, bool wide) {
+void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
   if (src->is_register()) {
     if (dest->is_register()) {
       assert(patch_code == lir_patch_none && info == NULL, "no patching and info allowed here");
@@ -917,7 +916,7 @@ void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
       assert(patch_code == lir_patch_none && info == NULL, "no patching and info allowed here");
       reg2stack(src, dest, type, pop_fpu_stack);
     } else if (dest->is_address()) {
-      reg2mem(src, dest, type, patch_code, info, pop_fpu_stack, wide, unaligned);
+      reg2mem(src, dest, type, patch_code, info, pop_fpu_stack, wide);
     } else {
       ShouldNotReachHere();
     }
@@ -946,8 +945,7 @@ void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
     }
 
   } else if (src->is_address()) {
-    mem2reg(src, dest, type, patch_code, info, wide, unaligned);
-
+    mem2reg(src, dest, type, patch_code, info, wide);
   } else {
     ShouldNotReachHere();
   }
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp
index 6297a6c290a..db3fbc61e7f 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.hpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp
@@ -175,12 +175,12 @@ class LIR_Assembler: public CompilationResourceObj {
   void reg2reg    (LIR_Opr src, LIR_Opr dest);
   void reg2mem    (LIR_Opr src, LIR_Opr dest, BasicType type,
                    LIR_PatchCode patch_code, CodeEmitInfo* info,
-                   bool pop_fpu_stack, bool wide, bool unaligned);
+                   bool pop_fpu_stack, bool wide);
   void stack2reg  (LIR_Opr src, LIR_Opr dest, BasicType type);
   void stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type);
   void mem2reg    (LIR_Opr src, LIR_Opr dest, BasicType type,
                    LIR_PatchCode patch_code,
-                   CodeEmitInfo* info, bool wide, bool unaligned);
+                   CodeEmitInfo* info, bool wide);
 
   void shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp);
   void shift_op(LIR_Code code, LIR_Opr left, jint  count, LIR_Opr dest);
@@ -227,7 +227,7 @@ class LIR_Assembler: public CompilationResourceObj {
 
   void roundfp_op(LIR_Opr src, LIR_Opr tmp, LIR_Opr dest, bool pop_fpu_stack);
   void move_op(LIR_Opr src, LIR_Opr result, BasicType type,
-               LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool unaligned, bool wide);
+               LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide);
   void volatile_move_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info);
   void comp_mem_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info);  // info set for null exceptions
   void comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr result, LIR_Op2* op);
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp
index 120de6ac7ac..294761bc815 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.cpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp
@@ -2475,189 +2475,8 @@ void LIRGenerator::do_RoundFP(RoundFP* x) {
   }
 }
 
-// Here UnsafeGetRaw may have x->base() and x->index() be int or long
-// on both 64 and 32 bits. Expecting x->base() to be always long on 64bit.
-void LIRGenerator::do_UnsafeGetRaw(UnsafeGetRaw* x) {
-  LIRItem base(x->base(), this);
-  LIRItem idx(this);
 
-  base.load_item();
-  if (x->has_index()) {
-    idx.set_instruction(x->index());
-    idx.load_nonconstant();
-  }
-
-  LIR_Opr reg = rlock_result(x, x->basic_type());
-
-  int   log2_scale = 0;
-  if (x->has_index()) {
-    log2_scale = x->log2_scale();
-  }
-
-  assert(!x->has_index() || idx.value() == x->index(), "should match");
-
-  LIR_Opr base_op = base.result();
-  LIR_Opr index_op = idx.result();
-#ifndef _LP64
-  if (base_op->type() == T_LONG) {
-    base_op = new_register(T_INT);
-    __ convert(Bytecodes::_l2i, base.result(), base_op);
-  }
-  if (x->has_index()) {
-    if (index_op->type() == T_LONG) {
-      LIR_Opr long_index_op = index_op;
-      if (index_op->is_constant()) {
-        long_index_op = new_register(T_LONG);
-        __ move(index_op, long_index_op);
-      }
-      index_op = new_register(T_INT);
-      __ convert(Bytecodes::_l2i, long_index_op, index_op);
-    } else {
-      assert(x->index()->type()->tag() == intTag, "must be");
-    }
-  }
-  // At this point base and index should be all ints.
-  assert(base_op->type() == T_INT && !base_op->is_constant(), "base should be an non-constant int");
-  assert(!x->has_index() || index_op->type() == T_INT, "index should be an int");
-#else
-  if (x->has_index()) {
-    if (index_op->type() == T_INT) {
-      if (!index_op->is_constant()) {
-        index_op = new_register(T_LONG);
-        __ convert(Bytecodes::_i2l, idx.result(), index_op);
-      }
-    } else {
-      assert(index_op->type() == T_LONG, "must be");
-      if (index_op->is_constant()) {
-        index_op = new_register(T_LONG);
-        __ move(idx.result(), index_op);
-      }
-    }
-  }
-  // At this point base is a long non-constant
-  // Index is a long register or a int constant.
-  // We allow the constant to stay an int because that would allow us a more compact encoding by
-  // embedding an immediate offset in the address expression. If we have a long constant, we have to
-  // move it into a register first.
-  assert(base_op->type() == T_LONG && !base_op->is_constant(), "base must be a long non-constant");
-  assert(!x->has_index() || (index_op->type() == T_INT && index_op->is_constant()) ||
-                            (index_op->type() == T_LONG && !index_op->is_constant()), "unexpected index type");
-#endif
-
-  BasicType dst_type = x->basic_type();
-
-  LIR_Address* addr;
-  if (index_op->is_constant()) {
-    assert(log2_scale == 0, "must not have a scale");
-    assert(index_op->type() == T_INT, "only int constants supported");
-    addr = new LIR_Address(base_op, index_op->as_jint(), dst_type);
-  } else {
-#ifdef X86
-    addr = new LIR_Address(base_op, index_op, LIR_Address::Scale(log2_scale), 0, dst_type);
-#elif defined(GENERATE_ADDRESS_IS_PREFERRED)
-    addr = generate_address(base_op, index_op, log2_scale, 0, dst_type);
-#else
-    if (index_op->is_illegal() || log2_scale == 0) {
-      addr = new LIR_Address(base_op, index_op, dst_type);
-    } else {
-      LIR_Opr tmp = new_pointer_register();
-      __ shift_left(index_op, log2_scale, tmp);
-      addr = new LIR_Address(base_op, tmp, dst_type);
-    }
-#endif
-  }
-
-  if (x->may_be_unaligned() && (dst_type == T_LONG || dst_type == T_DOUBLE)) {
-    __ unaligned_move(addr, reg);
-  } else {
-    if (dst_type == T_OBJECT && x->is_wide()) {
-      __ move_wide(addr, reg);
-    } else {
-      __ move(addr, reg);
-    }
-  }
-}
-
-
-void LIRGenerator::do_UnsafePutRaw(UnsafePutRaw* x) {
-  int  log2_scale = 0;
-  BasicType type = x->basic_type();
-
-  if (x->has_index()) {
-    log2_scale = x->log2_scale();
-  }
-
-  LIRItem base(x->base(), this);
-  LIRItem value(x->value(), this);
-  LIRItem idx(this);
-
-  base.load_item();
-  if (x->has_index()) {
-    idx.set_instruction(x->index());
-    idx.load_item();
-  }
-
-  if (type == T_BYTE || type == T_BOOLEAN) {
-    value.load_byte_item();
-  } else {
-    value.load_item();
-  }
-
-  set_no_result(x);
-
-  LIR_Opr base_op = base.result();
-  LIR_Opr index_op = idx.result();
-
-#ifdef GENERATE_ADDRESS_IS_PREFERRED
-  LIR_Address* addr = generate_address(base_op, index_op, log2_scale, 0, x->basic_type());
-#else
-#ifndef _LP64
-  if (base_op->type() == T_LONG) {
-    base_op = new_register(T_INT);
-    __ convert(Bytecodes::_l2i, base.result(), base_op);
-  }
-  if (x->has_index()) {
-    if (index_op->type() == T_LONG) {
-      index_op = new_register(T_INT);
-      __ convert(Bytecodes::_l2i, idx.result(), index_op);
-    }
-  }
-  // At this point base and index should be all ints and not constants
-  assert(base_op->type() == T_INT && !base_op->is_constant(), "base should be an non-constant int");
-  assert(!x->has_index() || (index_op->type() == T_INT && !index_op->is_constant()), "index should be an non-constant int");
-#else
-  if (x->has_index()) {
-    if (index_op->type() == T_INT) {
-      index_op = new_register(T_LONG);
-      __ convert(Bytecodes::_i2l, idx.result(), index_op);
-    }
-  }
-  // At this point base and index are long and non-constant
-  assert(base_op->type() == T_LONG && !base_op->is_constant(), "base must be a non-constant long");
-  assert(!x->has_index() || (index_op->type() == T_LONG && !index_op->is_constant()), "index must be a non-constant long");
-#endif
-
-  if (log2_scale != 0) {
-    // temporary fix (platform dependent code without shift on Intel would be better)
-    // TODO: ARM also allows embedded shift in the address
-    LIR_Opr tmp = new_pointer_register();
-    if (TwoOperandLIRForm) {
-      __ move(index_op, tmp);
-      index_op = tmp;
-    }
-    __ shift_left(index_op, log2_scale, tmp);
-    if (!TwoOperandLIRForm) {
-      index_op = tmp;
-    }
-  }
-
-  LIR_Address* addr = new LIR_Address(base_op, index_op, x->basic_type());
-#endif // !GENERATE_ADDRESS_IS_PREFERRED
-  __ move(value.result(), addr);
-}
-
-
-void LIRGenerator::do_UnsafeGetObject(UnsafeGetObject* x) {
+void LIRGenerator::do_UnsafeGet(UnsafeGet* x) {
   BasicType type = x->basic_type();
   LIRItem src(x->object(), this);
   LIRItem off(x->offset(), this);
@@ -2678,12 +2497,28 @@ void LIRGenerator::do_UnsafeGetObject(UnsafeGetObject* x) {
   }
 
   LIR_Opr result = rlock_result(x, type);
-  access_load_at(decorators, type,
-                 src, off.result(), result);
+  if (!x->is_raw()) {
+    access_load_at(decorators, type, src, off.result(), result);
+  } else {
+    // Currently it is only used in GraphBuilder::setup_osr_entry_block.
+    // It reads the value from [src + offset] directly.
+#ifdef _LP64
+    LIR_Opr offset = new_register(T_LONG);
+    __ convert(Bytecodes::_i2l, off.result(), offset);
+#else
+    LIR_Opr offset = off.result();
+#endif
+    LIR_Address* addr = new LIR_Address(src.result(), offset, type);
+    if (type == T_LONG || type == T_DOUBLE) {
+      __ move(addr, result);
+    } else {
+      access_load(IN_NATIVE, type, LIR_OprFact::address(addr), result);
+    }
+  }
 }
 
 
-void LIRGenerator::do_UnsafePutObject(UnsafePutObject* x) {
+void LIRGenerator::do_UnsafePut(UnsafePut* x) {
   BasicType type = x->basic_type();
   LIRItem src(x->object(), this);
   LIRItem off(x->offset(), this);
@@ -2709,7 +2544,7 @@ void LIRGenerator::do_UnsafePutObject(UnsafePutObject* x) {
   access_store_at(decorators, type, src, off.result(), data.result());
 }
 
-void LIRGenerator::do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) {
+void LIRGenerator::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {
   BasicType type = x->basic_type();
   LIRItem src(x->object(), this);
   LIRItem off(x->offset(), this);
@@ -3291,11 +3126,7 @@ void LIRGenerator::invoke_load_one_argument(LIRItem* param, LIR_Opr loc) {
     if (addr->type() == T_OBJECT) {
       __ move_wide(param->result(), addr);
     } else {
-      if (addr->type() == T_LONG || addr->type() == T_DOUBLE) {
-        __ unaligned_move(param->result(), addr);
-      } else {
-        __ move(param->result(), addr);
-      }
+      __ move(param->result(), addr);
     }
   }
 }
@@ -3919,7 +3750,7 @@ void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) {
 }
 
 void LIRGenerator::increment_backedge_counter_conditionally(LIR_Condition cond, LIR_Opr left, LIR_Opr right, CodeEmitInfo* info, int left_bci, int right_bci, int bci) {
-  if (compilation()->count_backedges()) {
+  if (compilation()->is_profiling()) {
 #if defined(X86) && !defined(_LP64)
     // BEWARE! On 32-bit x86 cmp clobbers its left argument so we need a temp copy.
     LIR_Opr left_copy = new_register(left->type());
@@ -4212,11 +4043,7 @@ LIR_Opr LIRGenerator::call_runtime(BasicTypeArray* signature, LIR_OprList* args,
 //             __ move(arg, tmp);
 //             arg = tmp;
 //           }
-      if (addr->type() == T_LONG || addr->type() == T_DOUBLE) {
-        __ unaligned_move(arg, addr);
-      } else {
-        __ move(arg, addr);
-      }
+      __ move(arg, addr);
     }
   }
 
@@ -4254,11 +4081,7 @@ LIR_Opr LIRGenerator::call_runtime(BasicTypeArray* signature, LIRItemList* args,
     } else {
       LIR_Address* addr = loc->as_address_ptr();
       arg->load_for_store(addr->type());
-      if (addr->type() == T_LONG || addr->type() == T_DOUBLE) {
-        __ unaligned_move(arg->result(), addr);
-      } else {
-        __ move(arg->result(), addr);
-      }
+      __ move(arg->result(), addr);
     }
   }
 
@@ -4307,10 +4130,3 @@ LIR_Opr LIRGenerator::mask_boolean(LIR_Opr array, LIR_Opr value, CodeEmitInfo*&
   value = value_fixed;
   return value;
 }
-
-LIR_Opr LIRGenerator::maybe_mask_boolean(StoreIndexed* x, LIR_Opr array, LIR_Opr value, CodeEmitInfo*& null_check_info) {
-  if (x->check_boolean()) {
-    value = mask_boolean(array, value, null_check_info);
-  }
-  return value;
-}
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp
index 6c29f9c8196..4cf5caf840a 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.hpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp
@@ -414,18 +414,18 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
                                     int bci, bool backedge, bool notify);
   void increment_event_counter(CodeEmitInfo* info, LIR_Opr step, int bci, bool backedge);
   void increment_invocation_counter(CodeEmitInfo *info) {
-    if (compilation()->count_invocations()) {
+    if (compilation()->is_profiling()) {
       increment_event_counter(info, LIR_OprFact::intConst(InvocationCounter::count_increment), InvocationEntryBci, false);
     }
   }
   void increment_backedge_counter(CodeEmitInfo* info, int bci) {
-    if (compilation()->count_backedges()) {
+    if (compilation()->is_profiling()) {
       increment_event_counter(info, LIR_OprFact::intConst(InvocationCounter::count_increment), bci, true);
     }
   }
   void increment_backedge_counter_conditionally(LIR_Condition cond, LIR_Opr left, LIR_Opr right, CodeEmitInfo* info, int left_bci, int right_bci, int bci);
   void increment_backedge_counter(CodeEmitInfo* info, LIR_Opr step, int bci) {
-    if (compilation()->count_backedges()) {
+    if (compilation()->is_profiling()) {
       increment_event_counter(info, step, bci, true);
     }
   }
@@ -445,9 +445,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
   void do_root (Instruction* instr);
   void walk    (Instruction* instr);
 
-  void bind_block_entry(BlockBegin* block);
-  void start_block(BlockBegin* block);
-
   LIR_Opr new_register(BasicType type);
   LIR_Opr new_register(Value value)              { return new_register(as_BasicType(value->type())); }
   LIR_Opr new_register(ValueType* type)          { return new_register(as_BasicType(type)); }
@@ -486,7 +483,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
   void do_SwitchRanges(SwitchRangeArray* x, LIR_Opr value, BlockBegin* default_sux);
 
 #ifdef JFR_HAVE_INTRINSICS
-  void do_ClassIDIntrinsic(Intrinsic* x);
   void do_getEventWriter(Intrinsic* x);
 #endif
 
@@ -504,7 +500,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
   void profile_element_type(Value element, ciMethodData* md, ciArrayLoadStoreData* load_store);
   bool profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag);
   LIR_Opr mask_boolean(LIR_Opr array, LIR_Opr value, CodeEmitInfo*& null_check_info);
-  LIR_Opr maybe_mask_boolean(StoreIndexed* x, LIR_Opr array, LIR_Opr value, CodeEmitInfo*& null_check_info);
 
  public:
   Compilation*  compilation() const              { return _compilation; }
@@ -606,11 +601,9 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
   virtual void do_OsrEntry       (OsrEntry*        x);
   virtual void do_ExceptionObject(ExceptionObject* x);
   virtual void do_RoundFP        (RoundFP*         x);
-  virtual void do_UnsafeGetRaw   (UnsafeGetRaw*    x);
-  virtual void do_UnsafePutRaw   (UnsafePutRaw*    x);
-  virtual void do_UnsafeGetObject(UnsafeGetObject* x);
-  virtual void do_UnsafePutObject(UnsafePutObject* x);
-  virtual void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x);
+  virtual void do_UnsafeGet      (UnsafeGet*       x);
+  virtual void do_UnsafePut      (UnsafePut*       x);
+  virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
   virtual void do_ProfileCall    (ProfileCall*     x);
   virtual void do_ProfileReturnType (ProfileReturnType* x);
   virtual void do_ProfileInvoke  (ProfileInvoke*   x);
diff --git a/src/hotspot/share/c1/c1_Optimizer.cpp b/src/hotspot/share/c1/c1_Optimizer.cpp
index 3f7eb48e22a..ab6a9259d27 100644
--- a/src/hotspot/share/c1/c1_Optimizer.cpp
+++ b/src/hotspot/share/c1/c1_Optimizer.cpp
@@ -26,7 +26,7 @@
 #include "c1/c1_Canonicalizer.hpp"
 #include "c1/c1_Optimizer.hpp"
 #include "c1/c1_ValueMap.hpp"
-#include "c1/c1_ValueSet.inline.hpp"
+#include "c1/c1_ValueSet.hpp"
 #include "c1/c1_ValueStack.hpp"
 #include "memory/resourceArea.hpp"
 #include "utilities/bitMap.inline.hpp"
@@ -534,11 +534,9 @@ class NullCheckVisitor: public InstructionVisitor {
   void do_OsrEntry       (OsrEntry*        x);
   void do_ExceptionObject(ExceptionObject* x);
   void do_RoundFP        (RoundFP*         x);
-  void do_UnsafeGetRaw   (UnsafeGetRaw*    x);
-  void do_UnsafePutRaw   (UnsafePutRaw*    x);
-  void do_UnsafeGetObject(UnsafeGetObject* x);
-  void do_UnsafePutObject(UnsafePutObject* x);
-  void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x);
+  void do_UnsafeGet      (UnsafeGet*       x);
+  void do_UnsafePut      (UnsafePut*       x);
+  void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
   void do_ProfileCall    (ProfileCall*     x);
   void do_ProfileReturnType (ProfileReturnType*  x);
   void do_ProfileACmpTypes(ProfileACmpTypes*  x);
@@ -724,11 +722,9 @@ void NullCheckVisitor::do_Base           (Base*            x) {}
 void NullCheckVisitor::do_OsrEntry       (OsrEntry*        x) {}
 void NullCheckVisitor::do_ExceptionObject(ExceptionObject* x) { nce()->handle_ExceptionObject(x); }
 void NullCheckVisitor::do_RoundFP        (RoundFP*         x) {}
-void NullCheckVisitor::do_UnsafeGetRaw   (UnsafeGetRaw*    x) {}
-void NullCheckVisitor::do_UnsafePutRaw   (UnsafePutRaw*    x) {}
-void NullCheckVisitor::do_UnsafeGetObject(UnsafeGetObject* x) {}
-void NullCheckVisitor::do_UnsafePutObject(UnsafePutObject* x) {}
-void NullCheckVisitor::do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) {}
+void NullCheckVisitor::do_UnsafeGet      (UnsafeGet*       x) {}
+void NullCheckVisitor::do_UnsafePut      (UnsafePut*       x) {}
+void NullCheckVisitor::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {}
 void NullCheckVisitor::do_ProfileCall    (ProfileCall*     x) { nce()->clear_last_explicit_null_check();
                                                                 nce()->handle_ProfileCall(x); }
 void NullCheckVisitor::do_ProfileReturnType (ProfileReturnType* x) { nce()->handle_ProfileReturnType(x); }
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
index 198b5afeb7b..fc5df4a4201 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
@@ -1001,7 +1001,7 @@ void RangeCheckEliminator::calc_bounds(BlockBegin *block, BlockBegin *loop_heade
           } else {
             // Has no upper bound
             Instruction *instr = ai->length();
-            if (instr != NULL) instr = ai->array();
+            if (instr == NULL) instr = ai->array();
             update_bound(pushed, ai->index(), Instruction::lss, instr, 0);
           }
         }
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
index 96b3458ff67..2dbbc3a486f 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
@@ -130,8 +130,6 @@ class RangeCheckEliminator {
     void do_MonitorEnter   (MonitorEnter*    x) { /* nothing to do */ };
     void do_MonitorExit    (MonitorExit*     x) { /* nothing to do */ };
     void do_Invoke         (Invoke*          x) { /* nothing to do */ };
-    void do_UnsafePutRaw   (UnsafePutRaw*    x) { /* nothing to do */ };
-    void do_UnsafePutObject(UnsafePutObject* x) { /* nothing to do */ };
     void do_Intrinsic      (Intrinsic*       x) { /* nothing to do */ };
     void do_Local          (Local*           x) { /* nothing to do */ };
     void do_LoadField      (LoadField*       x) { /* nothing to do */ };
@@ -162,9 +160,9 @@ class RangeCheckEliminator {
     void do_OsrEntry       (OsrEntry*        x) { /* nothing to do */ };
     void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ };
     void do_RoundFP        (RoundFP*         x) { /* nothing to do */ };
-    void do_UnsafeGetRaw   (UnsafeGetRaw*    x) { /* nothing to do */ };
-    void do_UnsafeGetObject(UnsafeGetObject* x) { /* nothing to do */ };
-    void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) { /* nothing to do */ };
+    void do_UnsafePut      (UnsafePut*       x) { /* nothing to do */ };
+    void do_UnsafeGet      (UnsafeGet*       x) { /* nothing to do */ };
+    void do_UnsafeGetAndSet(UnsafeGetAndSet* x) { /* nothing to do */ };
     void do_ProfileCall    (ProfileCall*     x) { /* nothing to do */ };
     void do_ProfileReturnType (ProfileReturnType*  x) { /* nothing to do */ };
     void do_ProfileACmpTypes(ProfileACmpTypes*  x) { /* nothing to do */ };
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 71efbc7d755..f484ac75b1f 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -61,7 +61,6 @@
 #include "oops/oop.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "runtime/atomic.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/handles.inline.hpp"
@@ -356,6 +355,11 @@ const char* Runtime1::name_for_address(address entry) {
 }
 
 static void allocate_instance(JavaThread* current, Klass* klass, TRAPS) {
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    Runtime1::_new_instance_slowcase_cnt++;
+  }
+#endif
   assert(klass->is_klass(), "not a class");
   Handle holder(current, klass->klass_holder()); // keep the klass alive
   InstanceKlass* h = InstanceKlass::cast(klass);
@@ -368,13 +372,11 @@ static void allocate_instance(JavaThread* current, Klass* klass, TRAPS) {
 JRT_END
 
 JRT_ENTRY(void, Runtime1::new_instance(JavaThread* current, Klass* klass))
-  NOT_PRODUCT(_new_instance_slowcase_cnt++;)
   allocate_instance(current, klass, CHECK);
 JRT_END
 
 // Same as new_instance but throws error for inline klasses
 JRT_ENTRY(void, Runtime1::new_instance_no_inline(JavaThread* current, Klass* klass))
-  NOT_PRODUCT(_new_instance_slowcase_cnt++;)
   if (klass->is_inline_klass()) {
     SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_InstantiationError());
   } else {
@@ -383,7 +385,11 @@ JRT_ENTRY(void, Runtime1::new_instance_no_inline(JavaThread* current, Klass* kla
 JRT_END
 
 JRT_ENTRY(void, Runtime1::new_type_array(JavaThread* current, Klass* klass, jint length))
-  NOT_PRODUCT(_new_type_array_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_type_array_slowcase_cnt++;
+  }
+#endif
   // Note: no handle for klass needed since they are not used
   //       anymore after new_typeArray() and no GC can happen before.
   //       (This may have to change if this code changes!)
@@ -401,8 +407,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::new_object_array(JavaThread* current, Klass* array_klass, jint length))
-  NOT_PRODUCT(_new_object_array_slowcase_cnt++;)
-
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_object_array_slowcase_cnt++;
+  }
+#endif
   // Note: no handle for klass needed since they are not used
   //       anymore after new_objArray() and no GC can happen before.
   //       (This may have to change if this code changes!)
@@ -442,8 +451,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::new_multi_array(JavaThread* current, Klass* klass, int rank, jint* dims))
-  NOT_PRODUCT(_new_multi_array_slowcase_cnt++;)
-
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_multi_array_slowcase_cnt++;
+  }
+#endif
   assert(klass->is_klass(), "not a class");
   assert(rank >= 1, "rank must be nonzero");
   Handle holder(current, klass->klass_holder()); // keep the klass alive
@@ -776,7 +788,11 @@ address Runtime1::exception_handler_for_pc(JavaThread* current) {
 
 
 JRT_ENTRY(void, Runtime1::throw_range_check_exception(JavaThread* current, int index, arrayOopDesc* a))
-  NOT_PRODUCT(_throw_range_check_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_range_check_exception_count++;
+  }
+#endif
   const int len = 35;
   assert(len < strlen("Index %d out of bounds for length %d"), "Must allocate more space for message.");
   char message[2 * jintAsStringSize + len];
@@ -786,7 +802,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_index_exception(JavaThread* current, int index))
-  NOT_PRODUCT(_throw_index_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_index_exception_count++;
+  }
+#endif
   char message[16];
   sprintf(message, "%d", index);
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IndexOutOfBoundsException(), message);
@@ -794,19 +814,31 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_div0_exception(JavaThread* current))
-  NOT_PRODUCT(_throw_div0_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_div0_exception_count++;
+  }
+#endif
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ArithmeticException(), "/ by zero");
 JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_null_pointer_exception(JavaThread* current))
-  NOT_PRODUCT(_throw_null_pointer_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_null_pointer_exception_count++;
+  }
+#endif
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException());
 JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_class_cast_exception(JavaThread* current, oopDesc* object))
-  NOT_PRODUCT(_throw_class_cast_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_class_cast_exception_count++;
+  }
+#endif
   ResourceMark rm(current);
   char* message = SharedRuntime::generate_class_cast_message(current, object->klass());
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ClassCastException(), message);
@@ -814,7 +846,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_incompatible_class_change_error(JavaThread* current))
-  NOT_PRODUCT(_throw_incompatible_class_change_error_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_incompatible_class_change_error_count++;
+  }
+#endif
   ResourceMark rm(current);
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IncompatibleClassChangeError());
 JRT_END
@@ -827,7 +863,11 @@ JRT_ENTRY(void, Runtime1::throw_illegal_monitor_state_exception(JavaThread* curr
 JRT_END
 
 JRT_BLOCK_ENTRY(void, Runtime1::monitorenter(JavaThread* current, oopDesc* obj, BasicObjectLock* lock))
-  NOT_PRODUCT(_monitorenter_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _monitorenter_slowcase_cnt++;
+  }
+#endif
   if (!UseFastLocking) {
     lock->set_obj(obj);
   }
@@ -837,7 +877,11 @@ JRT_END
 
 
 JRT_LEAF(void, Runtime1::monitorexit(JavaThread* current, BasicObjectLock* lock))
-  NOT_PRODUCT(_monitorexit_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _monitorexit_slowcase_cnt++;
+  }
+#endif
   assert(current->last_Java_sp(), "last_Java_sp must be set");
   oop obj = lock->obj();
   assert(oopDesc::is_oop(obj), "must be NULL or an object");
@@ -989,7 +1033,11 @@ static Klass* resolve_field_return_klass(const methodHandle& caller, int bci, TR
 // patch only naturally aligned words, as single, full-word writes.
 
 JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, Runtime1::StubID stub_id ))
-  NOT_PRODUCT(_patch_code_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _patch_code_slowcase_cnt++;
+  }
+#endif
 
   ResourceMark rm(current);
   RegisterMap reg_map(current, false);
@@ -1394,7 +1442,11 @@ JRT_END
 #else // DEOPTIMIZE_WHEN_PATCHING
 
 void Runtime1::patch_code(JavaThread* current, Runtime1::StubID stub_id) {
-  NOT_PRODUCT(_patch_code_slowcase_cnt++);
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _patch_code_slowcase_cnt++;
+  }
+#endif
 
   // Enable WXWrite: the function is called by c1 stub as a runtime function
   // (see another implementation above).
diff --git a/src/hotspot/share/c1/c1_ValueMap.cpp b/src/hotspot/share/c1/c1_ValueMap.cpp
index cb57971bdc5..cace2234373 100644
--- a/src/hotspot/share/c1/c1_ValueMap.cpp
+++ b/src/hotspot/share/c1/c1_ValueMap.cpp
@@ -26,7 +26,7 @@
 #include "c1/c1_Canonicalizer.hpp"
 #include "c1/c1_IR.hpp"
 #include "c1/c1_ValueMap.hpp"
-#include "c1/c1_ValueSet.inline.hpp"
+#include "c1/c1_ValueSet.hpp"
 #include "c1/c1_ValueStack.hpp"
 
 #ifndef PRODUCT
diff --git a/src/hotspot/share/c1/c1_ValueMap.hpp b/src/hotspot/share/c1/c1_ValueMap.hpp
index 1fb1d696427..6a4343896e8 100644
--- a/src/hotspot/share/c1/c1_ValueMap.hpp
+++ b/src/hotspot/share/c1/c1_ValueMap.hpp
@@ -157,11 +157,9 @@ class ValueNumberingVisitor: public InstructionVisitor {
   void do_MonitorEnter   (MonitorEnter*    x) { kill_memory(); }
   void do_MonitorExit    (MonitorExit*     x) { kill_memory(); }
   void do_Invoke         (Invoke*          x) { kill_memory(); }
-  void do_UnsafePutRaw   (UnsafePutRaw*    x) { kill_memory(); }
-  void do_UnsafePutObject(UnsafePutObject* x) { kill_memory(); }
-  void do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) { kill_memory(); }
-  void do_UnsafeGetRaw   (UnsafeGetRaw*    x) { /* nothing to do */ }
-  void do_UnsafeGetObject(UnsafeGetObject* x) {
+  void do_UnsafePut      (UnsafePut*       x) { kill_memory(); }
+  void do_UnsafeGetAndSet(UnsafeGetAndSet* x) { kill_memory(); }
+  void do_UnsafeGet      (UnsafeGet*       x) {
     if (x->is_volatile()) { // the JMM requires this
       kill_memory();
     }
diff --git a/src/hotspot/share/c1/c1_ValueSet.hpp b/src/hotspot/share/c1/c1_ValueSet.hpp
index 71b67f24017..afd8d081dc5 100644
--- a/src/hotspot/share/c1/c1_ValueSet.hpp
+++ b/src/hotspot/share/c1/c1_ValueSet.hpp
@@ -28,6 +28,7 @@
 #include "c1/c1_Instruction.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/bitMap.hpp"
+#include "utilities/bitMap.inline.hpp"
 
 // A ValueSet is a simple abstraction on top of a BitMap representing
 // a set of Instructions. Currently it assumes that the number of
@@ -39,17 +40,21 @@ class ValueSet: public CompilationResourceObj {
   ResourceBitMap _map;
 
  public:
-  ValueSet();
+  ValueSet() : _map(Instruction::number_of_instructions()) {}
 
-  ValueSet* copy();
-  bool contains(Value x);
-  void put     (Value x);
-  void remove  (Value x);
-  bool set_intersect(ValueSet* other);
-  void set_union(ValueSet* other);
-  void clear   ();
-  void set_from(ValueSet* other);
-  bool equals  (ValueSet* other);
+  ValueSet* copy() {
+    ValueSet* res = new ValueSet();
+    res->_map.set_from(_map);
+    return res;
+  }
+  bool contains(Value x)              { return _map.at(x->id()); }
+  void put(Value x)                   { _map.set_bit(x->id()); }
+  void remove(Value x)                { _map.clear_bit(x->id()); }
+  bool set_intersect(ValueSet* other) { return _map.set_intersection_with_result(other->_map); }
+  void set_union(ValueSet* other)     { _map.set_union(other->_map); }
+  void clear()                        { _map.clear(); }
+  void set_from(ValueSet* other)      { _map.set_from(other->_map); }
+  bool equals(ValueSet* other)        { return _map.is_same(other->_map); }
 };
 
 #endif // SHARE_C1_C1_VALUESET_HPP
diff --git a/src/hotspot/share/c1/c1_ValueSet.inline.hpp b/src/hotspot/share/c1/c1_ValueSet.inline.hpp
deleted file mode 100644
index 4daed86b561..00000000000
--- a/src/hotspot/share/c1/c1_ValueSet.inline.hpp
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_C1_C1_VALUESET_INLINE_HPP
-#define SHARE_C1_C1_VALUESET_INLINE_HPP
-
-#include "c1/c1_ValueSet.hpp"
-
-#include "c1/c1_Instruction.hpp"
-#include "utilities/bitMap.inline.hpp"
-
-inline ValueSet::ValueSet() : _map(Instruction::number_of_instructions()) {
-}
-
-inline ValueSet* ValueSet::copy() {
-  ValueSet* res = new ValueSet();
-  res->_map.set_from(_map);
-  return res;
-}
-
-inline bool ValueSet::contains(Value x) {
-  return _map.at(x->id());
-}
-
-inline void ValueSet::put(Value x) {
-  _map.set_bit(x->id());
-}
-
-inline void ValueSet::remove(Value x) {
-  _map.clear_bit(x->id());
-}
-
-inline bool ValueSet::set_intersect(ValueSet* other) {
-  return _map.set_intersection_with_result(other->_map);
-}
-
-inline void ValueSet::set_union(ValueSet* other) {
-  _map.set_union(other->_map);
-}
-
-inline void ValueSet::clear() {
-  _map.clear();
-}
-
-inline void ValueSet::set_from(ValueSet* other) {
-  _map.set_from(other->_map);
-}
-
-inline bool ValueSet::equals(ValueSet* other) {
-  return _map.is_same(other->_map);
-}
-
-#endif // SHARE_C1_C1_VALUESET_INLINE_HPP
diff --git a/src/hotspot/share/c1/c1_ValueType.cpp b/src/hotspot/share/c1/c1_ValueType.cpp
index 94a18c5e3a7..a59f27371b9 100644
--- a/src/hotspot/share/c1/c1_ValueType.cpp
+++ b/src/hotspot/share/c1/c1_ValueType.cpp
@@ -80,12 +80,6 @@ ValueType* ValueType::meet(ValueType* y) const {
 }
 
 
-ValueType* ValueType::join(ValueType* y) const {
-  Unimplemented();
-  return NULL;
-}
-
-
 ciType* ObjectConstant::exact_type() const {
   ciObject* c = constant_value();
   return (c != NULL && !c->is_null_object()) ? c->klass() : NULL;
diff --git a/src/hotspot/share/c1/c1_ValueType.hpp b/src/hotspot/share/c1/c1_ValueType.hpp
index 05dfd483bc7..9eb4d997430 100644
--- a/src/hotspot/share/c1/c1_ValueType.hpp
+++ b/src/hotspot/share/c1/c1_ValueType.hpp
@@ -34,7 +34,6 @@ class ValueType;
 class   VoidType;
 class   IntType;
 class     IntConstant;
-class     IntInterval;
 class   LongType;
 class     LongConstant;
 class   FloatType;
@@ -53,8 +52,6 @@ class     ClassType;
 class       ClassConstant;
 class     MethodType;
 class       MethodConstant;
-class     MethodDataType;
-class       MethodDataConstant;
 class   AddressType;
 class     AddressConstant;
 class   IllegalType;
@@ -132,7 +129,6 @@ class ValueType: public CompilationResourceObj {
   bool is_instance()                             { return as_InstanceType() != NULL; }
   bool is_class()                                { return as_ClassType()    != NULL; }
   bool is_method()                               { return as_MethodType()   != NULL; }
-  bool is_method_data()                          { return as_MethodDataType() != NULL; }
   bool is_address()                              { return as_AddressType()  != NULL; }
   bool is_illegal()                              { return tag() == illegalTag; }
 
@@ -155,10 +151,8 @@ class ValueType: public CompilationResourceObj {
   virtual ClassType*        as_ClassType()       { return NULL; }
   virtual MetadataType*     as_MetadataType()    { return NULL; }
   virtual MethodType*       as_MethodType()      { return NULL; }
-  virtual MethodDataType*   as_MethodDataType()  { return NULL; }
   virtual AddressType*      as_AddressType()     { return NULL; }
   virtual IllegalType*      as_IllegalType()     { return NULL; }
-
   virtual IntConstant*      as_IntConstant()     { return NULL; }
   virtual LongConstant*     as_LongConstant()    { return NULL; }
   virtual FloatConstant*    as_FloatConstant()   { return NULL; }
@@ -167,14 +161,12 @@ class ValueType: public CompilationResourceObj {
   virtual InstanceConstant* as_InstanceConstant(){ return NULL; }
   virtual ClassConstant*    as_ClassConstant()   { return NULL; }
   virtual MethodConstant*   as_MethodConstant()  { return NULL; }
-  virtual MethodDataConstant* as_MethodDataConstant() { return NULL; }
   virtual ArrayConstant*    as_ArrayConstant()   { return NULL; }
   virtual StableArrayConstant* as_StableArrayConstant()   { return NULL; }
   virtual AddressConstant*  as_AddressConstant() { return NULL; }
 
   // type operations
   ValueType* meet(ValueType* y) const;
-  ValueType* join(ValueType* y) const;
 
   // debugging
   void print(outputStream* s = tty)              { s->print("%s", name()); }
@@ -215,25 +207,6 @@ class IntConstant: public IntType {
 };
 
 
-class IntInterval: public IntType {
- private:
-  jint _beg;
-  jint _end;
-
- public:
-  IntInterval(jint beg, jint end) {
-    assert(beg <= end, "illegal interval");
-    _beg = beg;
-    _end = end;
-  }
-
-  jint beg() const                               { return _beg; }
-  jint end() const                               { return _end; }
-
-  virtual bool is_interval() const               { return true; }
-};
-
-
 class LongType: public ValueType {
  public:
   LongType(): ValueType(longTag, 2) {}
@@ -451,28 +424,6 @@ class MethodConstant: public MethodType {
 };
 
 
-class MethodDataType: public MetadataType {
- public:
-  virtual MethodDataType* as_MethodDataType()           { return this; }
-};
-
-
-class MethodDataConstant: public MethodDataType {
- private:
-  ciMethodData* _value;
-
- public:
-  MethodDataConstant(ciMethodData* value)               { _value = value; }
-
-  ciMethodData* value() const                           { return _value; }
-
-  virtual bool is_constant() const                      { return true; }
-
-  virtual MethodDataConstant* as_MethodDataConstant()   { return this; }
-  virtual ciMetadata* constant_value() const            { return _value; }
-};
-
-
 class AddressType: public ValueType {
  public:
   AddressType(): ValueType(addressTag, 1) {}
diff --git a/src/hotspot/share/c1/c1_globals.hpp b/src/hotspot/share/c1/c1_globals.hpp
index 14cd142a27d..1d370657479 100644
--- a/src/hotspot/share/c1/c1_globals.hpp
+++ b/src/hotspot/share/c1/c1_globals.hpp
@@ -197,9 +197,6 @@
   develop(bool, CommentedAssembly, trueInDebug,                             \
           "Show extra info in PrintNMethods output")                        \
                                                                             \
-  develop(bool, LIRTracePeephole, false,                                    \
-          "Trace peephole optimizer")                                       \
-                                                                            \
   develop(bool, LIRTraceExecution, false,                                   \
           "add LIR code which logs the execution of blocks")                \
                                                                             \
@@ -294,12 +291,6 @@
   develop(bool, TraceFPURegisterUsage, false,                               \
           "Trace usage of FPU registers at start of blocks (intel only)")   \
                                                                             \
-  develop(bool, OptimizeUnsafes, true,                                      \
-          "Optimize raw unsafe ops")                                        \
-                                                                            \
-  develop(bool, PrintUnsafeOptimization, false,                             \
-          "Print optimization of raw unsafe ops")                           \
-                                                                            \
   develop(intx, InstructionCountCutoff, 37000,                              \
           "If GraphBuilder adds this many instructions, bails out")         \
           range(0, max_jint)                                                \
diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp
index 49d558c1f9a..f857d8f61ea 100644
--- a/src/hotspot/share/cds/archiveBuilder.cpp
+++ b/src/hotspot/share/cds/archiveBuilder.cpp
@@ -160,7 +160,7 @@ ArchiveBuilder::ArchiveBuilder() :
   _ro_region("ro", MAX_SHARED_DELTA),
   _rw_src_objs(),
   _ro_src_objs(),
-  _src_obj_table(INITIAL_TABLE_SIZE),
+  _src_obj_table(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE),
   _num_instance_klasses(0),
   _num_obj_array_klasses(0),
   _num_type_array_klasses(0),
@@ -190,6 +190,9 @@ ArchiveBuilder::~ArchiveBuilder() {
   delete _klasses;
   delete _symbols;
   delete _special_refs;
+  if (_shared_rs.is_reserved()) {
+    _shared_rs.release();
+  }
 }
 
 bool ArchiveBuilder::is_dumping_full_module_graph() {
@@ -228,7 +231,7 @@ bool ArchiveBuilder::gather_klass_and_symbol(MetaspaceClosure::Ref* ref, bool re
         _num_type_array_klasses ++;
       }
     }
-    // See RunTimeSharedClassInfo::get_for()
+    // See RunTimeClassInfo::get_for()
     _estimated_metaspaceobj_bytes += align_up(BytesPerWord, SharedSpaceObjectAlignment);
   } else if (ref->msotype() == MetaspaceObj::SymbolType) {
     // Make sure the symbol won't be GC'ed while we are dumping the archive.
@@ -319,7 +322,7 @@ void ArchiveBuilder::sort_klasses() {
 }
 
 size_t ArchiveBuilder::estimate_archive_size() {
-  // size of the symbol table and two dictionaries, plus the RunTimeSharedClassInfo's
+  // size of the symbol table and two dictionaries, plus the RunTimeClassInfo's
   size_t symbol_table_est = SymbolTable::estimate_size_for_archive();
   size_t dictionary_est = SystemDictionaryShared::estimate_size_for_archive();
   _estimated_hashtable_bytes = symbol_table_est + dictionary_est;
@@ -462,9 +465,9 @@ bool ArchiveBuilder::gather_one_source_obj(MetaspaceClosure::Ref* enclosing_ref,
   FollowMode follow_mode = get_follow_mode(ref);
   SourceObjInfo src_info(ref, read_only, follow_mode);
   bool created;
-  SourceObjInfo* p = _src_obj_table.add_if_absent(src_obj, src_info, &created);
+  SourceObjInfo* p = _src_obj_table.put_if_absent(src_obj, src_info, &created);
   if (created) {
-    if (_src_obj_table.maybe_grow(MAX_TABLE_SIZE)) {
+    if (_src_obj_table.maybe_grow()) {
       log_info(cds, hashtables)("Expanded _src_obj_table table to %d", _src_obj_table.table_size());
     }
   }
@@ -627,8 +630,8 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
   oldtop = dump_region->top();
   if (ref->msotype() == MetaspaceObj::ClassType) {
     // Save a pointer immediate in front of an InstanceKlass, so
-    // we can do a quick lookup from InstanceKlass* -> RunTimeSharedClassInfo*
-    // without building another hashtable. See RunTimeSharedClassInfo::get_for()
+    // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
+    // without building another hashtable. See RunTimeClassInfo::get_for()
     // in systemDictionaryShared.cpp.
     Klass* klass = (Klass*)src;
     if (klass->is_instance_klass()) {
@@ -654,7 +657,7 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
 }
 
 address ArchiveBuilder::get_dumped_addr(address src_obj) const {
-  SourceObjInfo* p = _src_obj_table.lookup(src_obj);
+  SourceObjInfo* p = _src_obj_table.get(src_obj);
   assert(p != NULL, "must be");
 
   return p->dumped_addr();
@@ -1088,16 +1091,16 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo,
                                               bitmap_size_in_bytes);
 
   if (closed_heap_regions != NULL) {
-    _total_closed_heap_region_size = mapinfo->write_archive_heap_regions(
+    _total_closed_heap_region_size = mapinfo->write_heap_regions(
                                         closed_heap_regions,
                                         closed_heap_oopmaps,
-                                        MetaspaceShared::first_closed_archive_heap_region,
-                                        MetaspaceShared::max_closed_archive_heap_region);
-    _total_open_heap_region_size = mapinfo->write_archive_heap_regions(
+                                        MetaspaceShared::first_closed_heap_region,
+                                        MetaspaceShared::max_closed_heap_region);
+    _total_open_heap_region_size = mapinfo->write_heap_regions(
                                         open_heap_regions,
                                         open_heap_oopmaps,
-                                        MetaspaceShared::first_open_archive_heap_region,
-                                        MetaspaceShared::max_open_archive_heap_region);
+                                        MetaspaceShared::first_open_heap_region,
+                                        MetaspaceShared::max_open_heap_region);
   }
 
   print_region_stats(mapinfo, closed_heap_regions, open_heap_regions);
@@ -1163,12 +1166,12 @@ void ArchiveBuilder::print_bitmap_region_stats(size_t size, size_t total_size) {
                  size, size/double(total_size)*100.0, size);
 }
 
-void ArchiveBuilder::print_heap_region_stats(GrowableArray<MemRegion> *heap_mem,
+void ArchiveBuilder::print_heap_region_stats(GrowableArray<MemRegion>* regions,
                                              const char *name, size_t total_size) {
-  int arr_len = heap_mem == NULL ? 0 : heap_mem->length();
+  int arr_len = regions == NULL ? 0 : regions->length();
   for (int i = 0; i < arr_len; i++) {
-      char* start = (char*)heap_mem->at(i).start();
-      size_t size = heap_mem->at(i).byte_size();
+      char* start = (char*)regions->at(i).start();
+      size_t size = regions->at(i).byte_size();
       char* top = start + size;
       log_debug(cds)("%s%d space: " SIZE_FORMAT_W(9) " [ %4.1f%% of total] out of " SIZE_FORMAT_W(9) " bytes [100.0%% used] at " INTPTR_FORMAT,
                      name, i, size, size/double(total_size)*100.0, size, p2i(start));
diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp
index 6c69c61c6ed..58067f15dfc 100644
--- a/src/hotspot/share/cds/archiveBuilder.hpp
+++ b/src/hotspot/share/cds/archiveBuilder.hpp
@@ -33,7 +33,7 @@
 #include "runtime/os.hpp"
 #include "utilities/bitMap.hpp"
 #include "utilities/growableArray.hpp"
-#include "utilities/hashtable.hpp"
+#include "utilities/resizeableResourceHash.hpp"
 #include "utilities/resourceHash.hpp"
 
 struct ArchiveHeapOopmapInfo;
@@ -184,8 +184,8 @@ class ArchiveBuilder : public StackObj {
 
   class SrcObjTableCleaner {
   public:
-    bool do_entry(address key, const SourceObjInfo* value) {
-      delete value->ref();
+    bool do_entry(address key, const SourceObjInfo& value) {
+      delete value.ref();
       return true;
     }
   };
@@ -204,7 +204,7 @@ class ArchiveBuilder : public StackObj {
 
   SourceObjList _rw_src_objs;                 // objs to put in rw region
   SourceObjList _ro_src_objs;                 // objs to put in ro region
-  KVHashtable<address, SourceObjInfo, mtClassShared> _src_obj_table;
+  ResizeableResourceHashtable<address, SourceObjInfo, ResourceObj::C_HEAP, mtClassShared> _src_obj_table;
   GrowableArray<Klass*>* _klasses;
   GrowableArray<Symbol*>* _symbols;
   GrowableArray<SpecialRefInfo>* _special_refs;
@@ -221,7 +221,7 @@ class ArchiveBuilder : public StackObj {
                           GrowableArray<MemRegion>* closed_heap_regions,
                           GrowableArray<MemRegion>* open_heap_regions);
   void print_bitmap_region_stats(size_t size, size_t total_size);
-  void print_heap_region_stats(GrowableArray<MemRegion> *heap_mem,
+  void print_heap_region_stats(GrowableArray<MemRegion>* regions,
                                const char *name, size_t total_size);
 
   // For global access.
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index bd279d35777..beabd780c04 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -39,6 +39,7 @@
 #include "oops/compressedOops.inline.hpp"
 #include "runtime/arguments.hpp"
 #include "utilities/bitMap.inline.hpp"
+#include "utilities/formatBuffer.hpp"
 
 CHeapBitMap* ArchivePtrMarker::_ptrmap = NULL;
 VirtualSpace* ArchivePtrMarker::_vs;
@@ -307,12 +308,12 @@ void ReadClosure::do_tag(int tag) {
 
 void ReadClosure::do_oop(oop *p) {
   narrowOop o = CompressedOops::narrow_oop_cast(nextPtr());
-  if (CompressedOops::is_null(o) || !HeapShared::open_archive_heap_region_mapped()) {
+  if (CompressedOops::is_null(o) || !HeapShared::open_regions_mapped()) {
     *p = NULL;
   } else {
     assert(HeapShared::is_heap_object_archiving_allowed(),
            "Archived heap object is not allowed");
-    assert(HeapShared::open_archive_heap_region_mapped(),
+    assert(HeapShared::open_regions_mapped(),
            "Open archive heap region is not mapped");
     *p = HeapShared::decode_from_archive(o);
   }
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index cdb3d99ab53..588ad1b6da9 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -71,6 +71,11 @@ class ArchivePtrMarker : AllStatic {
   static CHeapBitMap* ptrmap() {
     return _ptrmap;
   }
+
+  static void reset_map_and_vs() {
+    _ptrmap = nullptr;
+    _vs = nullptr;
+  }
 };
 
 // SharedDataRelocator is used to shift pointers in the CDS archive.
diff --git a/src/hotspot/share/cds/cdsProtectionDomain.cpp b/src/hotspot/share/cds/cdsProtectionDomain.cpp
new file mode 100644
index 00000000000..e5b06d0c2bb
--- /dev/null
+++ b/src/hotspot/share/cds/cdsProtectionDomain.cpp
@@ -0,0 +1,338 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "cds/cdsProtectionDomain.hpp"
+#include "classfile/classLoader.hpp"
+#include "classfile/classLoaderExt.hpp"
+#include "classfile/javaClasses.hpp"
+#include "classfile/moduleEntry.hpp"
+#include "classfile/symbolTable.hpp"
+#include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
+#include "classfile/vmSymbols.hpp"
+#include "memory/oopFactory.hpp"
+#include "memory/resourceArea.hpp"
+#include "memory/universe.hpp"
+#include "oops/instanceKlass.hpp"
+#include "oops/symbol.hpp"
+#include "runtime/javaCalls.hpp"
+
+OopHandle CDSProtectionDomain::_shared_protection_domains;
+OopHandle CDSProtectionDomain::_shared_jar_urls;
+OopHandle CDSProtectionDomain::_shared_jar_manifests;
+
+// Initializes the java.lang.Package and java.security.ProtectionDomain objects associated with
+// the given InstanceKlass.
+// Returns the ProtectionDomain for the InstanceKlass.
+Handle CDSProtectionDomain::init_security_info(Handle class_loader, InstanceKlass* ik, PackageEntry* pkg_entry, TRAPS) {
+  Handle pd;
+
+  if (ik != NULL) {
+    int index = ik->shared_classpath_index();
+    assert(index >= 0, "Sanity");
+    SharedClassPathEntry* ent = FileMapInfo::shared_path(index);
+    Symbol* class_name = ik->name();
+
+    if (ent->is_modules_image()) {
+      // For shared app/platform classes originated from the run-time image:
+      //   The ProtectionDomains are cached in the corresponding ModuleEntries
+      //   for fast access by the VM.
+      // all packages from module image are already created during VM bootstrap in
+      // Modules::define_module().
+      assert(pkg_entry != NULL, "archived class in module image cannot be from unnamed package");
+      ModuleEntry* mod_entry = pkg_entry->module();
+      pd = get_shared_protection_domain(class_loader, mod_entry, CHECK_(pd));
+    } else {
+      // For shared app/platform classes originated from JAR files on the class path:
+      //   Each of the 3 SystemDictionaryShared::_shared_xxx arrays has the same length
+      //   as the shared classpath table in the shared archive (see
+      //   FileMap::_shared_path_table in filemap.hpp for details).
+      //
+      //   If a shared InstanceKlass k is loaded from the class path, let
+      //
+      //     index = k->shared_classpath_index():
+      //
+      //   FileMap::_shared_path_table[index] identifies the JAR file that contains k.
+      //
+      //   k's protection domain is:
+      //
+      //     ProtectionDomain pd = _shared_protection_domains[index];
+      //
+      //   and k's Package is initialized using
+      //
+      //     manifest = _shared_jar_manifests[index];
+      //     url = _shared_jar_urls[index];
+      //     define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
+      //
+      //   Note that if an element of these 3 _shared_xxx arrays is NULL, it will be initialized by
+      //   the corresponding SystemDictionaryShared::get_shared_xxx() function.
+      Handle manifest = get_shared_jar_manifest(index, CHECK_(pd));
+      Handle url = get_shared_jar_url(index, CHECK_(pd));
+      int index_offset = index - ClassLoaderExt::app_class_paths_start_index();
+      if (index_offset < PackageEntry::max_index_for_defined_in_class_path()) {
+        if (pkg_entry == NULL || !pkg_entry->is_defined_by_cds_in_class_path(index_offset)) {
+          // define_shared_package only needs to be called once for each package in a jar specified
+          // in the shared class path.
+          define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
+          if (pkg_entry != NULL) {
+            pkg_entry->set_defined_by_cds_in_class_path(index_offset);
+          }
+        }
+      } else {
+        define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
+      }
+      pd = get_shared_protection_domain(class_loader, index, url, CHECK_(pd));
+    }
+  }
+  return pd;
+}
+
+Handle CDSProtectionDomain::get_package_name(Symbol* class_name, TRAPS) {
+  ResourceMark rm(THREAD);
+  Handle pkgname_string;
+  TempNewSymbol pkg = ClassLoader::package_from_class_name(class_name);
+  if (pkg != NULL) { // Package prefix found
+    const char* pkgname = pkg->as_klass_external_name();
+    pkgname_string = java_lang_String::create_from_str(pkgname,
+                                                       CHECK_(pkgname_string));
+  }
+  return pkgname_string;
+}
+
+PackageEntry* CDSProtectionDomain::get_package_entry_from_class(InstanceKlass* ik, Handle class_loader) {
+  PackageEntry* pkg_entry = ik->package();
+  if (MetaspaceShared::use_full_module_graph() && ik->is_shared() && pkg_entry != NULL) {
+    assert(MetaspaceShared::is_in_shared_metaspace(pkg_entry), "must be");
+    assert(!ik->is_shared_unregistered_class(), "unexpected archived package entry for an unregistered class");
+    assert(ik->module()->is_named(), "unexpected archived package entry for a class in an unnamed module");
+    return pkg_entry;
+  }
+  TempNewSymbol pkg_name = ClassLoader::package_from_class_name(ik->name());
+  if (pkg_name != NULL) {
+    pkg_entry = SystemDictionaryShared::class_loader_data(class_loader)->packages()->lookup_only(pkg_name);
+  } else {
+    pkg_entry = NULL;
+  }
+  return pkg_entry;
+}
+
+// Define Package for shared app classes from JAR file and also checks for
+// package sealing (all done in Java code)
+// See http://docs.oracle.com/javase/tutorial/deployment/jar/sealman.html
+void CDSProtectionDomain::define_shared_package(Symbol*  class_name,
+                                                   Handle class_loader,
+                                                   Handle manifest,
+                                                   Handle url,
+                                                   TRAPS) {
+  assert(SystemDictionary::is_system_class_loader(class_loader()), "unexpected class loader");
+  // get_package_name() returns a NULL handle if the class is in unnamed package
+  Handle pkgname_string = get_package_name(class_name, CHECK);
+  if (pkgname_string.not_null()) {
+    Klass* app_classLoader_klass = vmClasses::jdk_internal_loader_ClassLoaders_AppClassLoader_klass();
+    JavaValue result(T_OBJECT);
+    JavaCallArguments args(3);
+    args.set_receiver(class_loader);
+    args.push_oop(pkgname_string);
+    args.push_oop(manifest);
+    args.push_oop(url);
+    JavaCalls::call_virtual(&result, app_classLoader_klass,
+                            vmSymbols::defineOrCheckPackage_name(),
+                            vmSymbols::defineOrCheckPackage_signature(),
+                            &args,
+                            CHECK);
+  }
+}
+
+Handle CDSProtectionDomain::create_jar_manifest(const char* manifest_chars, size_t size, TRAPS) {
+  typeArrayOop buf = oopFactory::new_byteArray((int)size, CHECK_NH);
+  typeArrayHandle bufhandle(THREAD, buf);
+  ArrayAccess<>::arraycopy_from_native(reinterpret_cast<const jbyte*>(manifest_chars),
+                                         buf, typeArrayOopDesc::element_offset<jbyte>(0), size);
+  Handle bais = JavaCalls::construct_new_instance(vmClasses::ByteArrayInputStream_klass(),
+                      vmSymbols::byte_array_void_signature(),
+                      bufhandle, CHECK_NH);
+  // manifest = new Manifest(ByteArrayInputStream)
+  Handle manifest = JavaCalls::construct_new_instance(vmClasses::Jar_Manifest_klass(),
+                      vmSymbols::input_stream_void_signature(),
+                      bais, CHECK_NH);
+  return manifest;
+}
+
+Handle CDSProtectionDomain::get_shared_jar_manifest(int shared_path_index, TRAPS) {
+  Handle manifest;
+  if (shared_jar_manifest(shared_path_index) == NULL) {
+    SharedClassPathEntry* ent = FileMapInfo::shared_path(shared_path_index);
+    size_t size = (size_t)ent->manifest_size();
+    if (size == 0) {
+      return Handle();
+    }
+
+    // ByteArrayInputStream bais = new ByteArrayInputStream(buf);
+    const char* src = ent->manifest();
+    assert(src != NULL, "No Manifest data");
+    manifest = create_jar_manifest(src, size, CHECK_NH);
+    atomic_set_shared_jar_manifest(shared_path_index, manifest());
+  }
+  manifest = Handle(THREAD, shared_jar_manifest(shared_path_index));
+  assert(manifest.not_null(), "sanity");
+  return manifest;
+}
+
+Handle CDSProtectionDomain::get_shared_jar_url(int shared_path_index, TRAPS) {
+  Handle url_h;
+  if (shared_jar_url(shared_path_index) == NULL) {
+    JavaValue result(T_OBJECT);
+    const char* path = FileMapInfo::shared_path_name(shared_path_index);
+    Handle path_string = java_lang_String::create_from_str(path, CHECK_(url_h));
+    Klass* classLoaders_klass =
+        vmClasses::jdk_internal_loader_ClassLoaders_klass();
+    JavaCalls::call_static(&result, classLoaders_klass,
+                           vmSymbols::toFileURL_name(),
+                           vmSymbols::toFileURL_signature(),
+                           path_string, CHECK_(url_h));
+
+    atomic_set_shared_jar_url(shared_path_index, result.get_oop());
+  }
+
+  url_h = Handle(THREAD, shared_jar_url(shared_path_index));
+  assert(url_h.not_null(), "sanity");
+  return url_h;
+}
+
+// Get the ProtectionDomain associated with the CodeSource from the classloader.
+Handle CDSProtectionDomain::get_protection_domain_from_classloader(Handle class_loader,
+                                                                      Handle url, TRAPS) {
+  // CodeSource cs = new CodeSource(url, null);
+  Handle cs = JavaCalls::construct_new_instance(vmClasses::CodeSource_klass(),
+                  vmSymbols::url_code_signer_array_void_signature(),
+                  url, Handle(), CHECK_NH);
+
+  // protection_domain = SecureClassLoader.getProtectionDomain(cs);
+  Klass* secureClassLoader_klass = vmClasses::SecureClassLoader_klass();
+  JavaValue obj_result(T_OBJECT);
+  JavaCalls::call_virtual(&obj_result, class_loader, secureClassLoader_klass,
+                          vmSymbols::getProtectionDomain_name(),
+                          vmSymbols::getProtectionDomain_signature(),
+                          cs, CHECK_NH);
+  return Handle(THREAD, obj_result.get_oop());
+}
+
+// Returns the ProtectionDomain associated with the JAR file identified by the url.
+Handle CDSProtectionDomain::get_shared_protection_domain(Handle class_loader,
+                                                            int shared_path_index,
+                                                            Handle url,
+                                                            TRAPS) {
+  Handle protection_domain;
+  if (shared_protection_domain(shared_path_index) == NULL) {
+    Handle pd = get_protection_domain_from_classloader(class_loader, url, THREAD);
+    atomic_set_shared_protection_domain(shared_path_index, pd());
+  }
+
+  // Acquire from the cache because if another thread beats the current one to
+  // set the shared protection_domain and the atomic_set fails, the current thread
+  // needs to get the updated protection_domain from the cache.
+  protection_domain = Handle(THREAD, shared_protection_domain(shared_path_index));
+  assert(protection_domain.not_null(), "sanity");
+  return protection_domain;
+}
+
+// Returns the ProtectionDomain associated with the moduleEntry.
+Handle CDSProtectionDomain::get_shared_protection_domain(Handle class_loader,
+                                                         ModuleEntry* mod, TRAPS) {
+  ClassLoaderData *loader_data = mod->loader_data();
+  if (mod->shared_protection_domain() == NULL) {
+    Symbol* location = mod->location();
+    if (location != NULL) {
+      Handle location_string = java_lang_String::create_from_symbol(
+                                     location, CHECK_NH);
+      Handle url;
+      JavaValue result(T_OBJECT);
+      if (location->starts_with("jrt:/")) {
+        url = JavaCalls::construct_new_instance(vmClasses::URL_klass(),
+                                                vmSymbols::string_void_signature(),
+                                                location_string, CHECK_NH);
+      } else {
+        Klass* classLoaders_klass =
+          vmClasses::jdk_internal_loader_ClassLoaders_klass();
+        JavaCalls::call_static(&result, classLoaders_klass, vmSymbols::toFileURL_name(),
+                               vmSymbols::toFileURL_signature(),
+                               location_string, CHECK_NH);
+        url = Handle(THREAD, result.get_oop());
+      }
+
+      Handle pd = get_protection_domain_from_classloader(class_loader, url,
+                                                         CHECK_NH);
+      mod->set_shared_protection_domain(loader_data, pd);
+    }
+  }
+
+  Handle protection_domain(THREAD, mod->shared_protection_domain());
+  assert(protection_domain.not_null(), "sanity");
+  return protection_domain;
+}
+
+void CDSProtectionDomain::atomic_set_array_index(OopHandle array, int index, oop o) {
+  // Benign race condition:  array.obj_at(index) may already be filled in.
+  // The important thing here is that all threads pick up the same result.
+  // It doesn't matter which racing thread wins, as long as only one
+  // result is used by all threads, and all future queries.
+  ((objArrayOop)array.resolve())->atomic_compare_exchange_oop(index, o, NULL);
+}
+
+oop CDSProtectionDomain::shared_protection_domain(int index) {
+  return ((objArrayOop)_shared_protection_domains.resolve())->obj_at(index);
+}
+
+void CDSProtectionDomain::allocate_shared_protection_domain_array(int size, TRAPS) {
+  if (_shared_protection_domains.resolve() == NULL) {
+    oop spd = oopFactory::new_objArray(
+        vmClasses::ProtectionDomain_klass(), size, CHECK);
+    _shared_protection_domains = OopHandle(Universe::vm_global(), spd);
+  }
+}
+
+oop CDSProtectionDomain::shared_jar_url(int index) {
+  return ((objArrayOop)_shared_jar_urls.resolve())->obj_at(index);
+}
+
+void CDSProtectionDomain::allocate_shared_jar_url_array(int size, TRAPS) {
+  if (_shared_jar_urls.resolve() == NULL) {
+    oop sju = oopFactory::new_objArray(
+        vmClasses::URL_klass(), size, CHECK);
+    _shared_jar_urls = OopHandle(Universe::vm_global(), sju);
+  }
+}
+
+oop CDSProtectionDomain::shared_jar_manifest(int index) {
+  return ((objArrayOop)_shared_jar_manifests.resolve())->obj_at(index);
+}
+
+void CDSProtectionDomain::allocate_shared_jar_manifest_array(int size, TRAPS) {
+  if (_shared_jar_manifests.resolve() == NULL) {
+    oop sjm = oopFactory::new_objArray(
+        vmClasses::Jar_Manifest_klass(), size, CHECK);
+    _shared_jar_manifests = OopHandle(Universe::vm_global(), sjm);
+  }
+}
diff --git a/src/hotspot/share/cds/cdsProtectionDomain.hpp b/src/hotspot/share/cds/cdsProtectionDomain.hpp
new file mode 100644
index 00000000000..1e048a96285
--- /dev/null
+++ b/src/hotspot/share/cds/cdsProtectionDomain.hpp
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARED_CDS_CDSPROTECTIONDOMAIN_HPP
+#define SHARED_CDS_CDSPROTECTIONDOMAIN_HPP
+#include "oops/oopHandle.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/thread.hpp"
+#include "classfile/moduleEntry.hpp"
+
+class InstanceKlass;
+class Symbol;
+class PackageEntry;
+class ModuleEntry;
+
+// CDS security
+class CDSProtectionDomain : AllStatic {
+  // See init_security_info for more info.
+  static OopHandle _shared_protection_domains;
+  static OopHandle _shared_jar_urls;
+  static OopHandle _shared_jar_manifests;
+
+public:
+  // Package handling:
+  //
+  // 1. For named modules in the runtime image
+  //    BOOT classes: Reuses the existing JVM_GetSystemPackage(s) interfaces
+  //                  to get packages in named modules for shared classes.
+  //                  Package for non-shared classes in named module is also
+  //                  handled using JVM_GetSystemPackage(s).
+  //
+  //    APP  classes: VM calls ClassLoaders.AppClassLoader::definePackage(String, Module)
+  //                  to define package for shared app classes from named
+  //                  modules.
+  //
+  //    PLATFORM  classes: VM calls ClassLoaders.PlatformClassLoader::definePackage(String, Module)
+  //                  to define package for shared platform classes from named
+  //                  modules.
+  //
+  // 2. For unnamed modules
+  //    BOOT classes: Reuses the existing JVM_GetSystemPackage(s) interfaces to
+  //                  get packages for shared boot classes in unnamed modules.
+  //
+  //    APP  classes: VM calls ClassLoaders.AppClassLoader::defineOrCheckPackage()
+  //                  with with the manifest and url from archived data.
+  //
+  //    PLATFORM  classes: No package is defined.
+  //
+  // The following two define_shared_package() functions are used to define
+  // package for shared APP and PLATFORM classes.
+  static Handle        get_package_name(Symbol*  class_name, TRAPS);
+  static PackageEntry* get_package_entry_from_class(InstanceKlass* ik, Handle class_loader);
+  static void define_shared_package(Symbol*  class_name,
+                                    Handle class_loader,
+                                    Handle manifest,
+                                    Handle url,
+                                    TRAPS);
+  static Handle create_jar_manifest(const char* man, size_t size, TRAPS);
+  static Handle get_shared_jar_manifest(int shared_path_index, TRAPS);
+  static Handle get_shared_jar_url(int shared_path_index, TRAPS);
+  static Handle get_protection_domain_from_classloader(Handle class_loader,
+                                                       Handle url, TRAPS);
+  static Handle get_shared_protection_domain(Handle class_loader,
+                                             int shared_path_index,
+                                             Handle url,
+                                             TRAPS);
+  static Handle get_shared_protection_domain(Handle class_loader,
+                                             ModuleEntry* mod, TRAPS);
+  static void atomic_set_array_index(OopHandle array, int index, oop o);
+  static oop shared_protection_domain(int index);
+  static void allocate_shared_protection_domain_array(int size, TRAPS);
+  static oop shared_jar_url(int index);
+  static void allocate_shared_jar_url_array(int size, TRAPS);
+  static oop shared_jar_manifest(int index);
+  static void allocate_shared_jar_manifest_array(int size, TRAPS);
+  static Handle init_security_info(Handle class_loader, InstanceKlass* ik, PackageEntry* pkg_entry, TRAPS);
+
+  static void allocate_shared_data_arrays(int size, TRAPS) {
+    allocate_shared_protection_domain_array(size, CHECK);
+    allocate_shared_jar_url_array(size, CHECK);
+    allocate_shared_jar_manifest_array(size, CHECK);
+  }
+  static void atomic_set_shared_protection_domain(int index, oop pd) {
+    atomic_set_array_index(_shared_protection_domains, index, pd);
+  }
+  static void atomic_set_shared_jar_url(int index, oop url) {
+    atomic_set_array_index(_shared_jar_urls, index, url);
+  }
+  static void atomic_set_shared_jar_manifest(int index, oop man) {
+    atomic_set_array_index(_shared_jar_manifests, index, man);
+  }
+};
+
+#endif // SHARED_CDS_CDSPROTECTIONDOMAIN_HPP
diff --git a/src/hotspot/share/cds/classListParser.cpp b/src/hotspot/share/cds/classListParser.cpp
index 12b04ea577b..225e454e279 100644
--- a/src/hotspot/share/cds/classListParser.cpp
+++ b/src/hotspot/share/cds/classListParser.cpp
@@ -54,7 +54,7 @@
 volatile Thread* ClassListParser::_parsing_thread = NULL;
 ClassListParser* ClassListParser::_instance = NULL;
 
-ClassListParser::ClassListParser(const char* file) : _id2klass_table(INITIAL_TABLE_SIZE) {
+ClassListParser::ClassListParser(const char* file) : _id2klass_table(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE) {
   _classlist_file = file;
   _file = NULL;
   // Use os::open() because neither fopen() nor os::fopen()
@@ -510,7 +510,7 @@ InstanceKlass* ClassListParser::load_class_from_source(Symbol* class_name, TRAPS
             specified_num_interfaces, expected_num_interfaces);
     }
 
-    bool added = SystemDictionaryShared::add_unregistered_class(THREAD, k);
+    bool added = SystemDictionaryShared::add_unregistered_class_for_static_archive(THREAD, k);
     if (!added) {
       // We allow only a single unregistered class for each unique name.
       error("Duplicated class %s", _class_name);
@@ -556,7 +556,7 @@ void ClassListParser::populate_cds_indy_info(const constantPoolHandle &pool, int
   }
 }
 
-bool ClassListParser::is_matching_cp_entry(constantPoolHandle &pool, int cp_index, TRAPS) {
+bool ClassListParser::is_matching_cp_entry(const constantPoolHandle &pool, int cp_index, TRAPS) {
   ResourceMark rm(THREAD);
   CDSIndyInfo cii;
   populate_cds_indy_info(pool, cp_index, &cii, CHECK_0);
@@ -692,11 +692,14 @@ Klass* ClassListParser::load_current_class(Symbol* class_name_symbol, TRAPS) {
     InstanceKlass* ik = InstanceKlass::cast(klass);
     int id = this->id();
     SystemDictionaryShared::update_shared_entry(ik, id);
-    InstanceKlass** old_ptr = table()->lookup(id);
-    if (old_ptr != NULL) {
+    bool created;
+    id2klass_table()->put_if_absent(id, ik, &created);
+    if (!created) {
       error("Duplicated ID %d for class %s", id, _class_name);
     }
-    table()->add(id, ik);
+    if (id2klass_table()->maybe_grow()) {
+      log_info(cds, hashtables)("Expanded id2klass_table() to %d", id2klass_table()->table_size());
+    }
   }
 
   return klass;
@@ -707,7 +710,7 @@ bool ClassListParser::is_loading_from_source() {
 }
 
 InstanceKlass* ClassListParser::lookup_class_by_id(int id) {
-  InstanceKlass** klass_ptr = table()->lookup(id);
+  InstanceKlass** klass_ptr = id2klass_table()->get(id);
   if (klass_ptr == NULL) {
     error("Class ID %d has not been defined", id);
   }
diff --git a/src/hotspot/share/cds/classListParser.hpp b/src/hotspot/share/cds/classListParser.hpp
index 9929cdca609..684605d9688 100644
--- a/src/hotspot/share/cds/classListParser.hpp
+++ b/src/hotspot/share/cds/classListParser.hpp
@@ -28,11 +28,12 @@
 #include "utilities/exceptions.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/growableArray.hpp"
-#include "utilities/hashtable.inline.hpp"
+#include "utilities/resizeableResourceHash.hpp"
 
 #define LAMBDA_PROXY_TAG "@lambda-proxy"
 #define LAMBDA_FORM_TAG  "@lambda-form-invoker"
 
+class constantPoolHandle;
 class Thread;
 
 class CDSIndyInfo {
@@ -66,7 +67,9 @@ class CDSIndyInfo {
 };
 
 class ClassListParser : public StackObj {
-  typedef KVHashtable<int, InstanceKlass*, mtInternal> ID2KlassTable;
+  // Must be C_HEAP allocated -- we don't want nested resource allocations.
+  typedef ResizeableResourceHashtable<int, InstanceKlass*,
+                                      ResourceObj::C_HEAP, mtClassShared> ID2KlassTable;
 
   enum {
     _unspecified      = -999,
@@ -80,7 +83,9 @@ class ClassListParser : public StackObj {
     _line_buf_size        = _max_allowed_line_len + _line_buf_extra
   };
 
-  static const int INITIAL_TABLE_SIZE = 1987;
+  // Use a small initial size in debug build to test resizing logic
+  static const int INITIAL_TABLE_SIZE = DEBUG_ONLY(17) NOT_DEBUG(1987);
+  static const int MAX_TABLE_SIZE = 61333;
   static volatile Thread* _parsing_thread; // the thread that created _instance
   static ClassListParser* _instance; // the singleton.
   const char* _classlist_file;
@@ -106,13 +111,13 @@ class ClassListParser : public StackObj {
   bool parse_int_option(const char* option_name, int* value);
   bool parse_uint_option(const char* option_name, int* value);
   InstanceKlass* load_class_from_source(Symbol* class_name, TRAPS);
-  ID2KlassTable* table() {
+  ID2KlassTable* id2klass_table() {
     return &_id2klass_table;
   }
   InstanceKlass* lookup_class_by_id(int id);
   void print_specified_interfaces();
   void print_actual_interfaces(InstanceKlass *ik);
-  bool is_matching_cp_entry(constantPoolHandle &pool, int cp_index, TRAPS);
+  bool is_matching_cp_entry(const constantPoolHandle &pool, int cp_index, TRAPS);
 
   void resolve_indy(JavaThread* current, Symbol* class_name_symbol);
   void resolve_indy_impl(Symbol* class_name_symbol, TRAPS);
@@ -161,7 +166,7 @@ class ClassListParser : public StackObj {
     return _super;
   }
   void check_already_loaded(const char* which, int id) {
-    if (_id2klass_table.lookup(id) == NULL) {
+    if (!id2klass_table()->contains(id)) {
       error("%s id %d is not yet loaded", which, id);
     }
   }
diff --git a/src/hotspot/share/cds/dumpTimeClassInfo.cpp b/src/hotspot/share/cds/dumpTimeClassInfo.cpp
new file mode 100644
index 00000000000..5a9028d3225
--- /dev/null
+++ b/src/hotspot/share/cds/dumpTimeClassInfo.cpp
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "cds/archiveBuilder.hpp"
+#include "cds/dumpTimeClassInfo.hpp"
+#include "classfile/classLoader.hpp"
+#include "classfile/classLoaderData.inline.hpp"
+#include "classfile/systemDictionaryShared.hpp"
+#include "memory/resourceArea.hpp"
+
+DumpTimeClassInfo DumpTimeClassInfo::clone() {
+  DumpTimeClassInfo clone;
+  clone._klass = _klass;
+  clone._nest_host = _nest_host;
+  clone._failed_verification = _failed_verification;
+  clone._is_archived_lambda_proxy = _is_archived_lambda_proxy;
+  clone._has_checked_exclusion = _has_checked_exclusion;
+  clone._id = _id;
+  clone._clsfile_size = _clsfile_size;
+  clone._clsfile_crc32 = _clsfile_crc32;
+  clone._excluded = _excluded;
+  clone._is_early_klass = _is_early_klass;
+  clone._verifier_constraints = NULL;
+  clone._verifier_constraint_flags = NULL;
+  clone._loader_constraints = NULL;
+  int clone_num_verifier_constraints = num_verifier_constraints();
+  if (clone_num_verifier_constraints > 0) {
+    clone._verifier_constraints = new (ResourceObj::C_HEAP, mtClass) GrowableArray<DTVerifierConstraint>(clone_num_verifier_constraints, mtClass);
+    clone._verifier_constraint_flags = new (ResourceObj::C_HEAP, mtClass) GrowableArray<char>(clone_num_verifier_constraints, mtClass);
+    for (int i = 0; i < clone_num_verifier_constraints; i++) {
+      clone._verifier_constraints->append(_verifier_constraints->at(i));
+      clone._verifier_constraint_flags->append(_verifier_constraint_flags->at(i));
+    }
+  }
+  int clone_num_loader_constraints = num_loader_constraints();
+  if (clone_num_loader_constraints > 0) {
+    clone._loader_constraints = new (ResourceObj::C_HEAP, mtClass) GrowableArray<DTLoaderConstraint>(clone_num_loader_constraints, mtClass);
+    for (int i = 0; i < clone_num_loader_constraints; i++) {
+      clone._loader_constraints->append(_loader_constraints->at(i));
+    }
+  }
+  return clone;
+}
+
+void DumpTimeClassInfo::add_verification_constraint(InstanceKlass* k, Symbol* name,
+         Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object) {
+  if (_verifier_constraints == NULL) {
+    _verifier_constraints = new (ResourceObj::C_HEAP, mtClass) GrowableArray<DTVerifierConstraint>(4, mtClass);
+  }
+  if (_verifier_constraint_flags == NULL) {
+    _verifier_constraint_flags = new (ResourceObj::C_HEAP, mtClass) GrowableArray<char>(4, mtClass);
+  }
+  GrowableArray<DTVerifierConstraint>* vc_array = _verifier_constraints;
+  for (int i = 0; i < vc_array->length(); i++) {
+    DTVerifierConstraint* p = vc_array->adr_at(i);
+    if (name == p->_name && from_name == p->_from_name) {
+      return;
+    }
+  }
+  DTVerifierConstraint cons(name, from_name);
+  vc_array->append(cons);
+
+  GrowableArray<char>* vcflags_array = _verifier_constraint_flags;
+  char c = 0;
+  c |= from_field_is_protected ? SystemDictionaryShared::FROM_FIELD_IS_PROTECTED : 0;
+  c |= from_is_array           ? SystemDictionaryShared::FROM_IS_ARRAY           : 0;
+  c |= from_is_object          ? SystemDictionaryShared::FROM_IS_OBJECT          : 0;
+  vcflags_array->append(c);
+
+  if (log_is_enabled(Trace, cds, verification)) {
+    ResourceMark rm;
+    log_trace(cds, verification)("add_verification_constraint: %s: %s must be subclass of %s [0x%x] array len %d flags len %d",
+                                 k->external_name(), from_name->as_klass_external_name(),
+                                 name->as_klass_external_name(), c, vc_array->length(), vcflags_array->length());
+  }
+}
+
+static char get_loader_type_by(oop  loader) {
+  assert(SystemDictionary::is_builtin_class_loader(loader), "Must be built-in loader");
+  if (SystemDictionary::is_boot_class_loader(loader)) {
+    return (char)ClassLoader::BOOT_LOADER;
+  } else if (SystemDictionary::is_platform_class_loader(loader)) {
+    return (char)ClassLoader::PLATFORM_LOADER;
+  } else {
+    assert(SystemDictionary::is_system_class_loader(loader), "Class loader mismatch");
+    return (char)ClassLoader::APP_LOADER;
+  }
+}
+
+void DumpTimeClassInfo::record_linking_constraint(Symbol* name, Handle loader1, Handle loader2) {
+  assert(loader1 != loader2, "sanity");
+  LogTarget(Info, class, loader, constraints) log;
+  if (_loader_constraints == NULL) {
+    _loader_constraints = new (ResourceObj::C_HEAP, mtClass) GrowableArray<DTLoaderConstraint>(4, mtClass);
+  }
+  char lt1 = get_loader_type_by(loader1());
+  char lt2 = get_loader_type_by(loader2());
+  DTLoaderConstraint lc(name, lt1, lt2);
+  for (int i = 0; i < _loader_constraints->length(); i++) {
+    DTLoaderConstraint dt = _loader_constraints->at(i);
+    if (lc.equals(dt)) {
+      if (log.is_enabled()) {
+        ResourceMark rm;
+        // Use loader[0]/loader[1] to be consistent with the logs in loaderConstraints.cpp
+        log.print("[CDS record loader constraint for class: %s constraint_name: %s loader[0]: %s loader[1]: %s already added]",
+                  _klass->external_name(), name->as_C_string(),
+                  ClassLoaderData::class_loader_data(loader1())->loader_name_and_id(),
+                  ClassLoaderData::class_loader_data(loader2())->loader_name_and_id());
+      }
+      return;
+    }
+  }
+  _loader_constraints->append(lc);
+  if (log.is_enabled()) {
+    ResourceMark rm;
+    // Use loader[0]/loader[1] to be consistent with the logs in loaderConstraints.cpp
+    log.print("[CDS record loader constraint for class: %s constraint_name: %s loader[0]: %s loader[1]: %s total %d]",
+              _klass->external_name(), name->as_C_string(),
+              ClassLoaderData::class_loader_data(loader1())->loader_name_and_id(),
+              ClassLoaderData::class_loader_data(loader2())->loader_name_and_id(),
+              _loader_constraints->length());
+  }
+}
+
+bool DumpTimeClassInfo::is_builtin() {
+  return SystemDictionaryShared::is_builtin(_klass);
+}
+
+DumpTimeClassInfo* DumpTimeSharedClassTable::find_or_allocate_info_for(InstanceKlass* k, bool dump_in_progress) {
+  bool created = false;
+  DumpTimeClassInfo* p;
+  if (!dump_in_progress) {
+    p = put_if_absent(k, &created);
+  } else {
+    p = get(k);
+  }
+  if (created) {
+    assert(!SystemDictionaryShared::no_class_loading_should_happen(),
+           "no new classes can be loaded while dumping archive");
+    p->_klass = k;
+  } else {
+    if (!dump_in_progress) {
+      assert(p->_klass == k, "Sanity");
+    }
+  }
+  return p;
+}
+
+class CountClassByCategory : StackObj {
+  DumpTimeSharedClassTable* _table;
+public:
+  CountClassByCategory(DumpTimeSharedClassTable* table) : _table(table) {}
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
+    if (!info.is_excluded()) {
+      if (info.is_builtin()) {
+        _table->inc_builtin_count();
+      } else {
+        _table->inc_unregistered_count();
+      }
+    }
+    return true; // keep on iterating
+  }
+};
+
+void DumpTimeSharedClassTable::update_counts() {
+  _builtin_count = 0;
+  _unregistered_count = 0;
+  CountClassByCategory counter(this);
+  iterate(&counter);
+}
diff --git a/src/hotspot/share/cds/dumpTimeClassInfo.hpp b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
new file mode 100644
index 00000000000..7b47ab91c01
--- /dev/null
+++ b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
@@ -0,0 +1,199 @@
+
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARED_CDS_DUMPTIMESHAREDCLASSINFO_HPP
+#define SHARED_CDS_DUMPTIMESHAREDCLASSINFO_HPP
+#include "cds/archiveBuilder.hpp"
+#include "cds/archiveUtils.hpp"
+#include "cds/metaspaceShared.hpp"
+#include "classfile/compactHashtable.hpp"
+#include "memory/metaspaceClosure.hpp"
+#include "oops/instanceKlass.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "utilities/growableArray.hpp"
+
+class Method;
+class Symbol;
+
+class DumpTimeClassInfo: public CHeapObj<mtClass> {
+  bool                         _excluded;
+  bool                         _is_early_klass;
+  bool                         _has_checked_exclusion;
+public:
+  struct DTLoaderConstraint {
+    Symbol* _name;
+    char _loader_type1;
+    char _loader_type2;
+    DTLoaderConstraint(Symbol* name, char l1, char l2) : _name(name), _loader_type1(l1), _loader_type2(l2) {
+      _name->increment_refcount();
+    }
+    DTLoaderConstraint() : _name(NULL), _loader_type1('0'), _loader_type2('0') {}
+    bool equals(const DTLoaderConstraint& t) {
+      return t._name == _name &&
+             ((t._loader_type1 == _loader_type1 && t._loader_type2 == _loader_type2) ||
+              (t._loader_type2 == _loader_type1 && t._loader_type1 == _loader_type2));
+    }
+  };
+
+  struct DTVerifierConstraint {
+    Symbol* _name;
+    Symbol* _from_name;
+    DTVerifierConstraint() : _name(NULL), _from_name(NULL) {}
+    DTVerifierConstraint(Symbol* n, Symbol* fn) : _name(n), _from_name(fn) {
+      _name->increment_refcount();
+      _from_name->increment_refcount();
+    }
+  };
+
+  InstanceKlass*               _klass;
+  InstanceKlass*               _nest_host;
+  bool                         _failed_verification;
+  bool                         _is_archived_lambda_proxy;
+  int                          _id;
+  int                          _clsfile_size;
+  int                          _clsfile_crc32;
+  GrowableArray<DTVerifierConstraint>* _verifier_constraints;
+  GrowableArray<char>*                 _verifier_constraint_flags;
+  GrowableArray<DTLoaderConstraint>*   _loader_constraints;
+
+  DumpTimeClassInfo() {
+    _klass = NULL;
+    _nest_host = NULL;
+    _failed_verification = false;
+    _is_archived_lambda_proxy = false;
+    _has_checked_exclusion = false;
+    _id = -1;
+    _clsfile_size = -1;
+    _clsfile_crc32 = -1;
+    _excluded = false;
+    _is_early_klass = JvmtiExport::is_early_phase();
+    _verifier_constraints = NULL;
+    _verifier_constraint_flags = NULL;
+    _loader_constraints = NULL;
+  }
+
+  void add_verification_constraint(InstanceKlass* k, Symbol* name,
+         Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object);
+  void record_linking_constraint(Symbol* name, Handle loader1, Handle loader2);
+
+  bool is_builtin();
+
+  int num_verifier_constraints() {
+    if (_verifier_constraint_flags != NULL) {
+      return _verifier_constraint_flags->length();
+    } else {
+      return 0;
+    }
+  }
+
+  int num_loader_constraints() {
+    if (_loader_constraints != NULL) {
+      return _loader_constraints->length();
+    } else {
+      return 0;
+    }
+  }
+
+  void metaspace_pointers_do(MetaspaceClosure* it) {
+    it->push(&_klass);
+    it->push(&_nest_host);
+    if (_verifier_constraints != NULL) {
+      for (int i = 0; i < _verifier_constraints->length(); i++) {
+        DTVerifierConstraint* cons = _verifier_constraints->adr_at(i);
+        it->push(&cons->_name);
+        it->push(&cons->_from_name);
+      }
+    }
+    if (_loader_constraints != NULL) {
+      for (int i = 0; i < _loader_constraints->length(); i++) {
+        DTLoaderConstraint* lc = _loader_constraints->adr_at(i);
+        it->push(&lc->_name);
+      }
+    }
+  }
+
+  bool is_excluded() {
+    // _klass may become NULL due to DynamicArchiveBuilder::set_to_null
+    return _excluded || _failed_verification || _klass == NULL;
+  }
+
+  // Was this class loaded while JvmtiExport::is_early_phase()==true
+  bool is_early_klass() {
+    return _is_early_klass;
+  }
+
+  // simple accessors
+  void set_excluded()                               { _excluded = true; }
+  bool has_checked_exclusion() const                { return _has_checked_exclusion; }
+  void set_has_checked_exclusion()                  { _has_checked_exclusion = true; }
+  bool failed_verification() const                  { return _failed_verification; }
+  void set_failed_verification()                    { _failed_verification = true; }
+  InstanceKlass* nest_host() const                  { return _nest_host; }
+  void set_nest_host(InstanceKlass* nest_host)      { _nest_host = nest_host; }
+  DumpTimeClassInfo clone();
+};
+
+
+inline unsigned DumpTimeSharedClassTable_hash(InstanceKlass* const& k) {
+  if (DumpSharedSpaces) {
+    // Deterministic archive contents
+    uintx delta = k->name() - MetaspaceShared::symbol_rs_base();
+    return primitive_hash<uintx>(delta);
+  } else {
+    // Deterministic archive is not possible because classes can be loaded
+    // in multiple threads.
+    return primitive_hash<InstanceKlass*>(k);
+  }
+}
+
+class DumpTimeSharedClassTable: public ResourceHashtable<
+  InstanceKlass*,
+  DumpTimeClassInfo,
+  &DumpTimeSharedClassTable_hash,
+  primitive_equals<InstanceKlass*>,
+  15889, // prime number
+  ResourceObj::C_HEAP>
+{
+  int _builtin_count;
+  int _unregistered_count;
+public:
+  DumpTimeSharedClassTable() {
+    _builtin_count = 0;
+    _unregistered_count = 0;
+  }
+  DumpTimeClassInfo* find_or_allocate_info_for(InstanceKlass* k, bool dump_in_progress);
+  void inc_builtin_count()      { _builtin_count++; }
+  void inc_unregistered_count() { _unregistered_count++; }
+  void update_counts();
+  int count_of(bool is_builtin) const {
+    if (is_builtin) {
+      return _builtin_count;
+    } else {
+      return _unregistered_count;
+    }
+  }
+};
+
+#endif // SHARED_CDS_DUMPTIMESHAREDCLASSINFO_HPP
diff --git a/src/hotspot/share/cds/dynamicArchive.cpp b/src/hotspot/share/cds/dynamicArchive.cpp
index 065597c652d..d868eb535b8 100644
--- a/src/hotspot/share/cds/dynamicArchive.cpp
+++ b/src/hotspot/share/cds/dynamicArchive.cpp
@@ -83,6 +83,7 @@ class DynamicArchiveBuilder : public ArchiveBuilder {
 
   void init_header();
   void release_header();
+  void post_dump();
   void sort_methods();
   void sort_methods(InstanceKlass* ik) const;
   void remark_pointers_for_instance_klass(InstanceKlass* k, bool should_mark) const;
@@ -107,9 +108,14 @@ class DynamicArchiveBuilder : public ArchiveBuilder {
 
     verify_universe("Before CDS dynamic dump");
     DEBUG_ONLY(SystemDictionaryShared::NoClassLoadingMark nclm);
-    SystemDictionaryShared::check_excluded_classes();
 
+    // Block concurrent class unloading from changing the _dumptime_table
     MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+    SystemDictionaryShared::check_excluded_classes();
+
+    // save dumptime tables
+    SystemDictionaryShared::clone_dumptime_tables();
+
     init_header();
     gather_source_objs();
     reserve_buffer();
@@ -156,6 +162,11 @@ class DynamicArchiveBuilder : public ArchiveBuilder {
     write_archive(serialized_data);
     release_header();
 
+    post_dump();
+
+    // Restore dumptime tables
+    SystemDictionaryShared::restore_dumptime_tables();
+
     assert(_num_dump_regions_used == _total_dump_regions, "must be");
     verify_universe("After CDS dynamic dump");
   }
@@ -192,6 +203,10 @@ void DynamicArchiveBuilder::release_header() {
   _header = NULL;
 }
 
+void DynamicArchiveBuilder::post_dump() {
+  ArchivePtrMarker::reset_map_and_vs();
+}
+
 void DynamicArchiveBuilder::sort_methods() {
   InstanceKlass::disable_method_binary_search();
   for (int i = 0; i < klasses()->length(); i++) {
@@ -255,8 +270,14 @@ void DynamicArchiveBuilder::sort_methods(InstanceKlass* ik) const {
   if (ik->default_methods() != NULL) {
     Method::sort_methods(ik->default_methods(), /*set_idnums=*/false, dynamic_dump_method_comparator);
   }
-  ik->vtable().initialize_vtable();
-  ik->itable().initialize_itable();
+  if (ik->is_linked()) {
+    // If the class has already been linked, we must relayout the i/v tables, whose order depends
+    // on the method sorting order.
+    // If the class is unlinked, we cannot layout the i/v tables yet. This is OK, as the
+    // i/v tables will be initialized at runtime after bytecode verification.
+    ik->vtable().initialize_vtable();
+    ik->itable().initialize_itable();
+  }
 
   // Set all the pointer marking bits after sorting.
   remark_pointers_for_instance_klass(ik, true);
@@ -317,7 +338,7 @@ class VM_PopulateDynamicDumpSharedSpace: public VM_GC_Sync_Operation {
   VMOp_Type type() const { return VMOp_PopulateDumpSharedSpace; }
   void doit() {
     ResourceMark rm;
-    if (SystemDictionaryShared::empty_dumptime_table()) {
+    if (SystemDictionaryShared::is_dumptime_table_empty()) {
       log_warning(cds, dynamic)("There is no class to be included in the dynamic archive.");
       return;
     }
@@ -331,12 +352,12 @@ class VM_PopulateDynamicDumpSharedSpace: public VM_GC_Sync_Operation {
   }
 };
 
-void DynamicArchive::prepare_for_dynamic_dumping_at_exit() {
+void DynamicArchive::prepare_for_dynamic_dumping() {
   EXCEPTION_MARK;
   ResourceMark rm(THREAD);
-  MetaspaceShared::link_and_cleanup_shared_classes(THREAD);
+  MetaspaceShared::link_shared_classes(THREAD);
   if (HAS_PENDING_EXCEPTION) {
-    log_error(cds)("ArchiveClassesAtExit has failed");
+    log_error(cds)("Dynamic dump has failed");
     log_error(cds)("%s: %s", PENDING_EXCEPTION->klass()->external_name(),
                    java_lang_String::as_utf8_string(java_lang_Throwable::message(PENDING_EXCEPTION)));
     // We cannot continue to dump the archive anymore.
@@ -345,42 +366,37 @@ void DynamicArchive::prepare_for_dynamic_dumping_at_exit() {
   }
 }
 
-bool DynamicArchive::_has_been_dumped_once = false;
-
 void DynamicArchive::dump(const char* archive_name, TRAPS) {
   assert(UseSharedSpaces && RecordDynamicDumpInfo, "already checked in arguments.cpp?");
   assert(ArchiveClassesAtExit == nullptr, "already checked in arguments.cpp?");
-  // During dynamic archive dumping, some of the data structures are overwritten so
-  // we cannot dump the dynamic archive again. TODO: this should be fixed.
-  if (has_been_dumped_once()) {
-    THROW_MSG(vmSymbols::java_lang_RuntimeException(),
-        "Dynamic dump has been done, and should only be done once");
-  } else {
-    // prevent multiple dumps.
-    set_has_been_dumped_once();
-    ArchiveClassesAtExit = archive_name;
-    if (Arguments::init_shared_archive_paths()) {
-      dump();
-    } else {
-      ArchiveClassesAtExit = nullptr;
-      THROW_MSG(vmSymbols::java_lang_RuntimeException(),
-              "Could not setup SharedDynamicArchivePath");
+  ArchiveClassesAtExit = archive_name;
+  if (Arguments::init_shared_archive_paths()) {
+    prepare_for_dynamic_dumping();
+    if (DynamicDumpSharedSpaces) {
+      dump(CHECK);
     }
-    // prevent do dynamic dump at exit.
+  } else {
     ArchiveClassesAtExit = nullptr;
-    if (!Arguments::init_shared_archive_paths()) {
-      THROW_MSG(vmSymbols::java_lang_RuntimeException(),
-             "Could not restore SharedDynamicArchivePath");
-    }
+    THROW_MSG(vmSymbols::java_lang_RuntimeException(),
+              "Could not setup SharedDynamicArchivePath");
+  }
+  // prevent do dynamic dump at exit.
+  ArchiveClassesAtExit = nullptr;
+  if (!Arguments::init_shared_archive_paths()) {
+    THROW_MSG(vmSymbols::java_lang_RuntimeException(),
+              "Could not restore SharedDynamicArchivePath");
   }
 }
 
-void DynamicArchive::dump() {
+void DynamicArchive::dump(TRAPS) {
   if (Arguments::GetSharedDynamicArchivePath() == NULL) {
     log_warning(cds, dynamic)("SharedDynamicArchivePath is not specified");
     return;
   }
 
+  // copy shared path table to saved.
+  FileMapInfo::clone_shared_path_table(CHECK);
+
   VM_PopulateDynamicDumpSharedSpace op;
   VMThread::execute(&op);
 }
diff --git a/src/hotspot/share/cds/dynamicArchive.hpp b/src/hotspot/share/cds/dynamicArchive.hpp
index 86d9be01f69..192801b4b60 100644
--- a/src/hotspot/share/cds/dynamicArchive.hpp
+++ b/src/hotspot/share/cds/dynamicArchive.hpp
@@ -58,13 +58,10 @@ class DynamicArchiveHeader : public FileMapHeader {
 };
 
 class DynamicArchive : AllStatic {
-  static bool _has_been_dumped_once;
 public:
-  static void prepare_for_dynamic_dumping_at_exit();
+  static void prepare_for_dynamic_dumping();
   static void dump(const char* archive_name, TRAPS);
-  static void dump();
-  static bool has_been_dumped_once() { return _has_been_dumped_once; }
-  static void set_has_been_dumped_once() { _has_been_dumped_once = true; }
+  static void dump(TRAPS);
   static bool is_mapped() { return FileMapInfo::dynamic_info() != NULL; }
   static bool validate(FileMapInfo* dynamic_info);
 };
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index ccc88ce08e2..bfbf07fcc02 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -478,6 +478,24 @@ void FileMapInfo::copy_shared_path_table(ClassLoaderData* loader_data, TRAPS) {
   for (int i = 0; i < _shared_path_table.size(); i++) {
     _saved_shared_path_table.path_at(i)->copy_from(shared_path(i), loader_data, CHECK);
   }
+  _saved_shared_path_table_array = array;
+}
+
+void FileMapInfo::clone_shared_path_table(TRAPS) {
+  Arguments::assert_is_dumping_archive();
+
+  ClassLoaderData* loader_data = ClassLoaderData::the_null_class_loader_data();
+  ClassPathEntry* jrt = ClassLoader::get_jrt_entry();
+
+  assert(jrt != NULL,
+         "No modular java runtime image present when allocating the CDS classpath entry table");
+
+  if (_saved_shared_path_table_array != NULL) {
+    MetadataFactory::free_array<u8>(loader_data, _saved_shared_path_table_array);
+    _saved_shared_path_table_array = NULL;
+  }
+
+  copy_shared_path_table(loader_data, CHECK);
 }
 
 void FileMapInfo::allocate_shared_path_table(TRAPS) {
@@ -503,8 +521,7 @@ void FileMapInfo::allocate_shared_path_table(TRAPS) {
   }
 
   assert(i == _shared_path_table.size(), "number of shared path entry mismatch");
-
-  copy_shared_path_table(loader_data, CHECK);
+  clone_shared_path_table(CHECK);
 }
 
 int FileMapInfo::add_shared_classpaths(int i, const char* which, ClassPathEntry *cpe, TRAPS) {
@@ -1389,13 +1406,13 @@ char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap,
 
 // Write out the given archive heap memory regions.  GC code combines multiple
 // consecutive archive GC regions into one MemRegion whenever possible and
-// produces the 'heap_mem' array.
+// produces the 'regions' array.
 //
 // If the archive heap memory size is smaller than a single dump time GC region
 // size, there is only one MemRegion in the array.
 //
 // If the archive heap memory size is bigger than one dump time GC region size,
-// the 'heap_mem' array may contain more than one consolidated MemRegions. When
+// the 'regions' array may contain more than one consolidated MemRegions. When
 // the first/bottom archive GC region is a partial GC region (with the empty
 // portion at the higher address within the region), one MemRegion is used for
 // the bottom partial archive GC region. The rest of the consecutive archive
@@ -1418,13 +1435,13 @@ char* FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap,
 //            ^^^
 //             |
 //             +-- gap
-size_t FileMapInfo::write_archive_heap_regions(GrowableArray<MemRegion> *heap_mem,
-                                               GrowableArray<ArchiveHeapOopmapInfo> *oopmaps,
-                                               int first_region_id, int max_num_regions) {
+size_t FileMapInfo::write_heap_regions(GrowableArray<MemRegion>* regions,
+                                       GrowableArray<ArchiveHeapOopmapInfo>* oopmaps,
+                                       int first_region_id, int max_num_regions) {
   assert(max_num_regions <= 2, "Only support maximum 2 memory regions");
 
-  int arr_len = heap_mem == NULL ? 0 : heap_mem->length();
-  if(arr_len > max_num_regions) {
+  int arr_len = regions == NULL ? 0 : regions->length();
+  if (arr_len > max_num_regions) {
     fail_stop("Unable to write archive heap memory regions: "
               "number of memory regions exceeds maximum due to fragmentation. "
               "Please increase java heap size "
@@ -1437,8 +1454,8 @@ size_t FileMapInfo::write_archive_heap_regions(GrowableArray<MemRegion> *heap_me
     char* start = NULL;
     size_t size = 0;
     if (i < arr_len) {
-      start = (char*)heap_mem->at(i).start();
-      size = heap_mem->at(i).byte_size();
+      start = (char*)regions->at(i).start();
+      size = regions->at(i).byte_size();
       total_size += size;
     }
 
@@ -1749,14 +1766,14 @@ address FileMapInfo::decode_start_address(FileMapRegion* spc, bool with_current_
   }
 }
 
-static MemRegion *closed_archive_heap_ranges = NULL;
-static MemRegion *open_archive_heap_ranges = NULL;
-static int num_closed_archive_heap_ranges = 0;
-static int num_open_archive_heap_ranges = 0;
+static MemRegion *closed_heap_regions = NULL;
+static MemRegion *open_heap_regions = NULL;
+static int num_closed_heap_regions = 0;
+static int num_open_heap_regions = 0;
 
 #if INCLUDE_CDS_JAVA_HEAP
 bool FileMapInfo::has_heap_regions() {
-  return (space_at(MetaspaceShared::first_closed_archive_heap_region)->used() > 0);
+  return (space_at(MetaspaceShared::first_closed_heap_region)->used() > 0);
 }
 
 // Returns the address range of the archived heap regions computed using the
@@ -1767,7 +1784,7 @@ MemRegion FileMapInfo::get_heap_regions_range_with_current_oop_encoding_mode() {
   address start = (address) max_uintx;
   address end   = NULL;
 
-  for (int i = MetaspaceShared::first_closed_archive_heap_region;
+  for (int i = MetaspaceShared::first_closed_heap_region;
            i <= MetaspaceShared::last_valid_region;
            i++) {
     FileMapRegion* si = space_at(i);
@@ -1882,7 +1899,7 @@ void FileMapInfo::map_heap_regions_impl() {
   log_info(cds)("CDS heap data relocation delta = " INTX_FORMAT " bytes", delta);
   HeapShared::init_narrow_oop_decoding(narrow_oop_base() + delta, narrow_oop_shift());
 
-  FileMapRegion* si = space_at(MetaspaceShared::first_closed_archive_heap_region);
+  FileMapRegion* si = space_at(MetaspaceShared::first_closed_heap_region);
   address relocated_closed_heap_region_bottom = start_address_as_decoded_from_archive(si);
   if (!is_aligned(relocated_closed_heap_region_bottom, HeapRegion::GrainBytes)) {
     // Align the bottom of the closed archive heap regions at G1 region boundary.
@@ -1901,20 +1918,19 @@ void FileMapInfo::map_heap_regions_impl() {
   assert(is_aligned(relocated_closed_heap_region_bottom, HeapRegion::GrainBytes),
          "must be");
 
-  // Map the closed_archive_heap regions, GC does not write into the regions.
-  if (map_heap_data(&closed_archive_heap_ranges,
-                    MetaspaceShared::first_closed_archive_heap_region,
-                    MetaspaceShared::max_closed_archive_heap_region,
-                    &num_closed_archive_heap_ranges)) {
-    HeapShared::set_closed_archive_heap_region_mapped();
-
-    // Now, map open_archive heap regions, GC can write into the regions.
-    if (map_heap_data(&open_archive_heap_ranges,
-                      MetaspaceShared::first_open_archive_heap_region,
-                      MetaspaceShared::max_open_archive_heap_region,
-                      &num_open_archive_heap_ranges,
-                      true /* open */)) {
-      HeapShared::set_open_archive_heap_region_mapped();
+  // Map the closed heap regions: GC does not write into these regions.
+  if (map_heap_regions(MetaspaceShared::first_closed_heap_region,
+                       MetaspaceShared::max_closed_heap_region,
+                       /*is_open_archive=*/ false,
+                       &closed_heap_regions, &num_closed_heap_regions)) {
+    HeapShared::set_closed_regions_mapped();
+
+    // Now, map the open heap regions: GC can write into these regions.
+    if (map_heap_regions(MetaspaceShared::first_open_heap_region,
+                         MetaspaceShared::max_open_heap_region,
+                         /*is_open_archive=*/ true,
+                         &open_heap_regions, &num_open_heap_regions)) {
+      HeapShared::set_open_regions_mapped();
       HeapShared::set_roots(header()->heap_obj_roots());
     }
   }
@@ -1925,19 +1941,19 @@ void FileMapInfo::map_heap_regions() {
     map_heap_regions_impl();
   }
 
-  if (!HeapShared::closed_archive_heap_region_mapped()) {
-    assert(closed_archive_heap_ranges == NULL &&
-           num_closed_archive_heap_ranges == 0, "sanity");
+  if (!HeapShared::closed_regions_mapped()) {
+    assert(closed_heap_regions == NULL &&
+           num_closed_heap_regions == 0, "sanity");
   }
 
-  if (!HeapShared::open_archive_heap_region_mapped()) {
-    assert(open_archive_heap_ranges == NULL && num_open_archive_heap_ranges == 0, "sanity");
+  if (!HeapShared::open_regions_mapped()) {
+    assert(open_heap_regions == NULL && num_open_heap_regions == 0, "sanity");
     MetaspaceShared::disable_full_module_graph();
   }
 }
 
-bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
-                                int max, int* num, bool is_open_archive) {
+bool FileMapInfo::map_heap_regions(int first, int max,  bool is_open_archive,
+                                   MemRegion** regions_ret, int* num_regions_ret) {
   MemRegion* regions = MemRegion::create_array(max, mtInternal);
 
   struct Cleanup {
@@ -1949,7 +1965,7 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
   } cleanup(regions, max);
 
   FileMapRegion* si;
-  int region_num = 0;
+  int num_regions = 0;
 
   for (int i = first;
            i < first + max; i++) {
@@ -1957,26 +1973,26 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
     size_t size = si->used();
     if (size > 0) {
       HeapWord* start = (HeapWord*)start_address_as_decoded_from_archive(si);
-      regions[region_num] = MemRegion(start, size / HeapWordSize);
-      region_num ++;
+      regions[num_regions] = MemRegion(start, size / HeapWordSize);
+      num_regions ++;
       log_info(cds)("Trying to map heap data: region[%d] at " INTPTR_FORMAT ", size = " SIZE_FORMAT_W(8) " bytes",
                     i, p2i(start), size);
     }
   }
 
-  if (region_num == 0) {
+  if (num_regions == 0) {
     return false; // no archived java heap data
   }
 
-  // Check that ranges are within the java heap
-  if (!G1CollectedHeap::heap()->check_archive_addresses(regions, region_num)) {
+  // Check that regions are within the java heap
+  if (!G1CollectedHeap::heap()->check_archive_addresses(regions, num_regions)) {
     log_info(cds)("UseSharedSpaces: Unable to allocate region, range is not within java heap.");
     return false;
   }
 
   // allocate from java heap
   if (!G1CollectedHeap::heap()->alloc_archive_regions(
-             regions, region_num, is_open_archive)) {
+             regions, num_regions, is_open_archive)) {
     log_info(cds)("UseSharedSpaces: Unable to allocate region, java heap range is already in use.");
     return false;
   }
@@ -1984,7 +2000,7 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
   // Map the archived heap data. No need to call MemTracker::record_virtual_memory_type()
   // for mapped regions as they are part of the reserved java heap, which is
   // already recorded.
-  for (int i = 0; i < region_num; i++) {
+  for (int i = 0; i < num_regions; i++) {
     si = space_at(first + i);
     char* addr = (char*)regions[i].start();
     char* base = os::map_memory(_fd, _full_path, si->file_offset(),
@@ -1992,7 +2008,7 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
                                 si->allow_exec());
     if (base == NULL || base != addr) {
       // dealloc the regions from java heap
-      dealloc_archive_heap_regions(regions, region_num);
+      dealloc_heap_regions(regions, num_regions);
       log_info(cds)("UseSharedSpaces: Unable to map at required address in java heap. "
                     INTPTR_FORMAT ", size = " SIZE_FORMAT " bytes",
                     p2i(addr), regions[i].byte_size());
@@ -2001,7 +2017,7 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
 
     if (VerifySharedSpaces && !region_crc_check(addr, regions[i].byte_size(), si->crc())) {
       // dealloc the regions from java heap
-      dealloc_archive_heap_regions(regions, region_num);
+      dealloc_heap_regions(regions, num_regions);
       log_info(cds)("UseSharedSpaces: mapped heap regions are corrupt");
       return false;
     }
@@ -2009,36 +2025,36 @@ bool FileMapInfo::map_heap_data(MemRegion **heap_mem, int first,
 
   cleanup._aborted = false;
   // the shared heap data is mapped successfully
-  *heap_mem = regions;
-  *num = region_num;
+  *regions_ret = regions;
+  *num_regions_ret = num_regions;
   return true;
 }
 
-void FileMapInfo::patch_archived_heap_embedded_pointers() {
+void FileMapInfo::patch_heap_embedded_pointers() {
   if (!_heap_pointers_need_patching) {
     return;
   }
 
   log_info(cds)("patching heap embedded pointers");
-  patch_archived_heap_embedded_pointers(closed_archive_heap_ranges,
-                                        num_closed_archive_heap_ranges,
-                                        MetaspaceShared::first_closed_archive_heap_region);
+  patch_heap_embedded_pointers(closed_heap_regions,
+                               num_closed_heap_regions,
+                               MetaspaceShared::first_closed_heap_region);
 
-  patch_archived_heap_embedded_pointers(open_archive_heap_ranges,
-                                        num_open_archive_heap_ranges,
-                                        MetaspaceShared::first_open_archive_heap_region);
+  patch_heap_embedded_pointers(open_heap_regions,
+                               num_open_heap_regions,
+                               MetaspaceShared::first_open_heap_region);
 }
 
-void FileMapInfo::patch_archived_heap_embedded_pointers(MemRegion* ranges, int num_ranges,
-                                                        int first_region_idx) {
+void FileMapInfo::patch_heap_embedded_pointers(MemRegion* regions, int num_regions,
+                                               int first_region_idx) {
   char* bitmap_base = map_bitmap_region();
   if (bitmap_base == NULL) {
     return;
   }
-  for (int i=0; i<num_ranges; i++) {
+  for (int i=0; i<num_regions; i++) {
     FileMapRegion* si = space_at(i + first_region_idx);
-    HeapShared::patch_archived_heap_embedded_pointers(
-      ranges[i],
+    HeapShared::patch_embedded_pointers(
+      regions[i],
       (address)(space_at(MetaspaceShared::bm)->mapped_base()) + si->oopmap_offset(),
       si->oopmap_size_in_bits());
   }
@@ -2049,19 +2065,19 @@ void FileMapInfo::patch_archived_heap_embedded_pointers(MemRegion* ranges, int n
 void FileMapInfo::fixup_mapped_heap_regions() {
   assert(vmClasses::Object_klass_loaded(), "must be");
   // If any closed regions were found, call the fill routine to make them parseable.
-  // Note that closed_archive_heap_ranges may be non-NULL even if no ranges were found.
-  if (num_closed_archive_heap_ranges != 0) {
-    assert(closed_archive_heap_ranges != NULL,
-           "Null closed_archive_heap_ranges array with non-zero count");
-    G1CollectedHeap::heap()->fill_archive_regions(closed_archive_heap_ranges,
-                                                  num_closed_archive_heap_ranges);
+  // Note that closed_heap_regions may be non-NULL even if no regions were found.
+  if (num_closed_heap_regions != 0) {
+    assert(closed_heap_regions != NULL,
+           "Null closed_heap_regions array with non-zero count");
+    G1CollectedHeap::heap()->fill_archive_regions(closed_heap_regions,
+                                                  num_closed_heap_regions);
   }
 
   // do the same for mapped open archive heap regions
-  if (num_open_archive_heap_ranges != 0) {
-    assert(open_archive_heap_ranges != NULL, "NULL open_archive_heap_ranges array with non-zero count");
-    G1CollectedHeap::heap()->fill_archive_regions(open_archive_heap_ranges,
-                                                  num_open_archive_heap_ranges);
+  if (num_open_heap_regions != 0) {
+    assert(open_heap_regions != NULL, "NULL open_heap_regions array with non-zero count");
+    G1CollectedHeap::heap()->fill_archive_regions(open_heap_regions,
+                                                  num_open_heap_regions);
 
     // Populate the open archive regions' G1BlockOffsetTableParts. That ensures
     // fast G1BlockOffsetTablePart::block_start operations for any given address
@@ -2072,15 +2088,15 @@ void FileMapInfo::fixup_mapped_heap_regions() {
     // regions, because objects in closed archive regions never reference objects
     // outside the closed archive regions and they are immutable. So we never
     // need their BOT during garbage collection.
-    G1CollectedHeap::heap()->populate_archive_regions_bot_part(open_archive_heap_ranges,
-                                                               num_open_archive_heap_ranges);
+    G1CollectedHeap::heap()->populate_archive_regions_bot_part(open_heap_regions,
+                                                               num_open_heap_regions);
   }
 }
 
 // dealloc the archive regions from java heap
-void FileMapInfo::dealloc_archive_heap_regions(MemRegion* regions, int num) {
+void FileMapInfo::dealloc_heap_regions(MemRegion* regions, int num) {
   if (num > 0) {
-    assert(regions != NULL, "Null archive ranges array with non-zero count");
+    assert(regions != NULL, "Null archive regions array with non-zero count");
     G1CollectedHeap::heap()->dealloc_archive_regions(regions, num);
   }
 }
@@ -2152,6 +2168,7 @@ FileMapInfo* FileMapInfo::_dynamic_archive_info = NULL;
 bool FileMapInfo::_heap_pointers_need_patching = false;
 SharedPathTable FileMapInfo::_shared_path_table;
 SharedPathTable FileMapInfo::_saved_shared_path_table;
+Array<u8>*      FileMapInfo::_saved_shared_path_table_array = NULL;
 bool FileMapInfo::_validating_shared_path_table = false;
 bool FileMapInfo::_memory_mapping_failed = false;
 GrowableArray<const char*>* FileMapInfo::_non_existent_class_paths = NULL;
@@ -2343,10 +2360,10 @@ void FileMapInfo::stop_sharing_and_unmap(const char* msg) {
     }
     // Dealloc the archive heap regions only without unmapping. The regions are part
     // of the java heap. Unmapping of the heap regions are managed by GC.
-    map_info->dealloc_archive_heap_regions(open_archive_heap_ranges,
-                                           num_open_archive_heap_ranges);
-    map_info->dealloc_archive_heap_regions(closed_archive_heap_ranges,
-                                           num_closed_archive_heap_ranges);
+    map_info->dealloc_heap_regions(open_heap_regions,
+                                   num_open_heap_regions);
+    map_info->dealloc_heap_regions(closed_heap_regions,
+                                   num_closed_heap_regions);
   } else if (DumpSharedSpaces) {
     fail_stop("%s", msg);
   }
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 13cb5f4acf2..61628df21c1 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -346,6 +346,7 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   // TODO: Probably change the following to be non-static
   static SharedPathTable       _shared_path_table;
   static SharedPathTable       _saved_shared_path_table;
+  static Array<u8>*            _saved_shared_path_table_array;  // remember the table array for cleanup
   static bool                  _validating_shared_path_table;
 
   // FileMapHeader describes the shared space data in the file to be
@@ -368,6 +369,7 @@ class FileMapInfo : public CHeapObj<mtInternal> {
     return _shared_path_table;
   }
   static SharedPathTable saved_shared_path_table() {
+    assert(_saved_shared_path_table.size() >= 0, "Sanity check");
     return _saved_shared_path_table;
   }
 
@@ -458,9 +460,9 @@ class FileMapInfo : public CHeapObj<mtInternal> {
                             GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
                             GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps,
                             size_t &size_in_bytes);
-  size_t write_archive_heap_regions(GrowableArray<MemRegion> *heap_mem,
-                                    GrowableArray<ArchiveHeapOopmapInfo> *oopmaps,
-                                    int first_region_id, int max_num_regions);
+  size_t write_heap_regions(GrowableArray<MemRegion>* regions,
+                            GrowableArray<ArchiveHeapOopmapInfo>* oopmaps,
+                            int first_region_id, int max_num_regions);
   void  write_bytes(const void* buffer, size_t count);
   void  write_bytes_aligned(const void* buffer, size_t count);
   size_t  read_bytes(void* buffer, size_t count);
@@ -468,9 +470,9 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   void  unmap_regions(int regions[], int num_regions);
   void  map_heap_regions() NOT_CDS_JAVA_HEAP_RETURN;
   void  fixup_mapped_heap_regions() NOT_CDS_JAVA_HEAP_RETURN;
-  void  patch_archived_heap_embedded_pointers() NOT_CDS_JAVA_HEAP_RETURN;
-  void  patch_archived_heap_embedded_pointers(MemRegion* ranges, int num_ranges,
-                                              int first_region_idx) NOT_CDS_JAVA_HEAP_RETURN;
+  void  patch_heap_embedded_pointers() NOT_CDS_JAVA_HEAP_RETURN;
+  void  patch_heap_embedded_pointers(MemRegion* regions, int num_regions,
+                                     int first_region_idx) NOT_CDS_JAVA_HEAP_RETURN;
   bool  has_heap_regions()  NOT_CDS_JAVA_HEAP_RETURN_(false);
   MemRegion get_heap_regions_range_with_current_oop_encoding_mode() NOT_CDS_JAVA_HEAP_RETURN_(MemRegion());
   void  unmap_region(int i);
@@ -497,6 +499,7 @@ class FileMapInfo : public CHeapObj<mtInternal> {
 
   static void allocate_shared_path_table(TRAPS);
   static void copy_shared_path_table(ClassLoaderData* loader_data, TRAPS);
+  static void clone_shared_path_table(TRAPS);
   static int add_shared_classpaths(int i, const char* which, ClassPathEntry *cpe, TRAPS);
   static void check_nonempty_dir_in_shared_path_table();
   bool validate_shared_path_table();
@@ -567,10 +570,10 @@ class FileMapInfo : public CHeapObj<mtInternal> {
                     GrowableArray<const char*>* rp_array) NOT_CDS_RETURN_(false);
   bool  validate_boot_class_paths() NOT_CDS_RETURN_(false);
   bool  validate_app_class_paths(int shared_app_paths_len) NOT_CDS_RETURN_(false);
-  bool  map_heap_data(MemRegion **heap_mem, int first, int max, int* num,
-                      bool is_open = false) NOT_CDS_JAVA_HEAP_RETURN_(false);
+  bool  map_heap_regions(int first, int max, bool is_open_archive,
+                         MemRegion** regions_ret, int* num_regions_ret) NOT_CDS_JAVA_HEAP_RETURN_(false);
   bool  region_crc_check(char* buf, size_t size, int expected_crc) NOT_CDS_RETURN_(false);
-  void  dealloc_archive_heap_regions(MemRegion* regions, int num) NOT_CDS_JAVA_HEAP_RETURN;
+  void  dealloc_heap_regions(MemRegion* regions, int num) NOT_CDS_JAVA_HEAP_RETURN;
   void  map_heap_regions_impl() NOT_CDS_JAVA_HEAP_RETURN;
   char* map_bitmap_region();
   MapArchiveResult map_region(int i, intx addr_delta, char* mapped_base_address, ReservedSpace rs);
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index bc9f6e3eb00..909abe449dd 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -67,9 +67,8 @@
 
 #if INCLUDE_CDS_JAVA_HEAP
 
-bool HeapShared::_closed_archive_heap_region_mapped = false;
-bool HeapShared::_open_archive_heap_region_mapped = false;
-bool HeapShared::_archive_heap_region_fixed = false;
+bool HeapShared::_closed_regions_mapped = false;
+bool HeapShared::_open_regions_mapped = false;
 address   HeapShared::_narrow_oop_base;
 int       HeapShared::_narrow_oop_shift;
 DumpedInternedStrings *HeapShared::_dumped_interned_strings = NULL;
@@ -117,15 +116,22 @@ GrowableArrayCHeap<oop, mtClassShared>* HeapShared::_pending_roots = NULL;
 narrowOop HeapShared::_roots_narrow;
 OopHandle HeapShared::_roots;
 
+#ifdef ASSERT
+bool HeapShared::is_archived_object_during_dumptime(oop p) {
+  assert(HeapShared::is_heap_object_archiving_allowed(), "must be");
+  assert(DumpSharedSpaces, "this function is only used with -Xshare:dump");
+  return Universe::heap()->is_archived_object(p);
+}
+#endif
+
 ////////////////////////////////////////////////////////////////
 //
 // Java heap object archiving support
 //
 ////////////////////////////////////////////////////////////////
-void HeapShared::fixup_mapped_heap_regions() {
+void HeapShared::fixup_mapped_regions() {
   FileMapInfo *mapinfo = FileMapInfo::current_info();
   mapinfo->fixup_mapped_heap_regions();
-  set_archive_heap_region_fixed();
   if (is_mapped()) {
     _roots = OopHandle(Universe::vm_global(), decode_from_archive(_roots_narrow));
     if (!MetaspaceShared::use_full_module_graph()) {
@@ -137,8 +143,8 @@ void HeapShared::fixup_mapped_heap_regions() {
 }
 
 unsigned HeapShared::oop_hash(oop const& p) {
-  assert(!UseBiasedLocking || !p->mark().has_bias_pattern(),
-         "this object should never have been locked");  // so identity_hash won't safepoin
+  assert(!EnableValhalla || !p->mark().is_inline_type(),
+         "this object should never have been locked");
   unsigned hash = (unsigned)p->identity_hash();
   return hash;
 }
@@ -215,7 +221,7 @@ objArrayOop HeapShared::roots() {
 
 void HeapShared::set_roots(narrowOop roots) {
   assert(UseSharedSpaces, "runtime only");
-  assert(open_archive_heap_region_mapped(), "must be");
+  assert(open_regions_mapped(), "must be");
   _roots_narrow = roots;
 }
 
@@ -240,7 +246,7 @@ oop HeapShared::get_root(int index, bool clear) {
 void HeapShared::clear_root(int index) {
   assert(index >= 0, "sanity");
   assert(UseSharedSpaces, "must be");
-  if (open_archive_heap_region_mapped()) {
+  if (open_regions_mapped()) {
     if (log_is_enabled(Debug, cds, heap)) {
       oop old = roots()->obj_at(index);
       log_debug(cds, heap)("Clearing root %d: was " PTR_FORMAT, index, p2i(old));
@@ -249,7 +255,7 @@ void HeapShared::clear_root(int index) {
   }
 }
 
-oop HeapShared::archive_heap_object(oop obj) {
+oop HeapShared::archive_object(oop obj) {
   assert(DumpSharedSpaces, "dump-time only");
 
   oop ao = find_archived_heap_object(obj);
@@ -275,7 +281,7 @@ oop HeapShared::archive_heap_object(oop obj) {
     // identity_hash for all shared objects, so they are less likely to be written
     // into during run time, increasing the potential of memory sharing.
     int hash_original = obj->identity_hash();
-    archived_oop->set_mark(markWord::prototype_for_klass(archived_oop->klass()).copy_set_hash(hash_original));
+    archived_oop->set_mark(archived_oop->klass()->prototype_header().copy_set_hash(hash_original));
     assert(archived_oop->mark().is_unlocked(), "sanity");
 
     DEBUG_ONLY(int hash_archived = archived_oop->identity_hash());
@@ -335,8 +341,8 @@ void HeapShared::run_full_gc_in_vm_thread() {
   }
 }
 
-void HeapShared::archive_java_heap_objects(GrowableArray<MemRegion>* closed,
-                                           GrowableArray<MemRegion>* open) {
+void HeapShared::archive_objects(GrowableArray<MemRegion>* closed_regions,
+                                 GrowableArray<MemRegion>* open_regions) {
 
   G1HeapVerifier::verify_ready_for_archiving();
 
@@ -349,10 +355,10 @@ void HeapShared::archive_java_heap_objects(GrowableArray<MemRegion>* closed,
     log_info(cds)("Heap range = [" PTR_FORMAT " - "  PTR_FORMAT "]",
                   p2i(CompressedOops::begin()), p2i(CompressedOops::end()));
     log_info(cds)("Dumping objects to closed archive heap region ...");
-    copy_closed_archive_heap_objects(closed);
+    copy_closed_objects(closed_regions);
 
     log_info(cds)("Dumping objects to open archive heap region ...");
-    copy_open_archive_heap_objects(open);
+    copy_open_objects(open_regions);
 
     destroy_archived_object_cache();
   }
@@ -360,8 +366,7 @@ void HeapShared::archive_java_heap_objects(GrowableArray<MemRegion>* closed,
   G1HeapVerifier::verify_archive_regions();
 }
 
-void HeapShared::copy_closed_archive_heap_objects(
-                                    GrowableArray<MemRegion> * closed_archive) {
+void HeapShared::copy_closed_objects(GrowableArray<MemRegion>* closed_regions) {
   assert(is_heap_object_archiving_allowed(), "Cannot archive java heap objects");
 
   G1CollectedHeap::heap()->begin_archive_alloc_range();
@@ -374,12 +379,11 @@ void HeapShared::copy_closed_archive_heap_objects(
                            true /* is_closed_archive */,
                            false /* is_full_module_graph */);
 
-  G1CollectedHeap::heap()->end_archive_alloc_range(closed_archive,
+  G1CollectedHeap::heap()->end_archive_alloc_range(closed_regions,
                                                    os::vm_allocation_granularity());
 }
 
-void HeapShared::copy_open_archive_heap_objects(
-                                    GrowableArray<MemRegion> * open_archive) {
+void HeapShared::copy_open_objects(GrowableArray<MemRegion>* open_regions) {
   assert(is_heap_object_archiving_allowed(), "Cannot archive java heap objects");
 
   G1CollectedHeap::heap()->begin_archive_alloc_range(true /* open */);
@@ -402,7 +406,7 @@ void HeapShared::copy_open_archive_heap_objects(
 
   copy_roots();
 
-  G1CollectedHeap::heap()->end_archive_alloc_range(open_archive,
+  G1CollectedHeap::heap()->end_archive_alloc_range(open_regions,
                                                    os::vm_allocation_granularity());
 }
 
@@ -416,11 +420,7 @@ void HeapShared::copy_roots() {
   memset(mem, 0, size * BytesPerWord);
   {
     // This is copied from MemAllocator::finish
-    if (UseBiasedLocking) {
-      oopDesc::set_mark(mem, k->prototype_header());
-    } else {
-      oopDesc::set_mark(mem, markWord::prototype());
-    }
+    oopDesc::set_mark(mem, k->prototype_header());
     oopDesc::release_set_klass(mem, k);
   }
   {
@@ -883,7 +883,7 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
   template <class T> void do_oop_work(T *p) {
     oop obj = RawAccess<>::oop_load(p);
     if (!CompressedOops::is_null(obj)) {
-      assert(!HeapShared::is_archived_object(obj),
+      assert(!HeapShared::is_archived_object_during_dumptime(obj),
              "original objects must not point to archived objects");
 
       size_t field_delta = pointer_delta(p, _orig_referencing_obj, sizeof(char));
@@ -902,7 +902,7 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
       oop archived = HeapShared::archive_reachable_objects_from(
           _level + 1, _subgraph_info, obj, _is_closed_archive);
       assert(archived != NULL, "VM should have exited with unarchivable objects for _level > 1");
-      assert(HeapShared::is_archived_object(archived), "must be");
+      assert(HeapShared::is_archived_object_during_dumptime(archived), "must be");
 
       if (!_record_klasses_only) {
         // Update the reference in the archived copy of the referencing object.
@@ -914,7 +914,7 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
   }
 };
 
-void HeapShared::check_closed_archive_heap_region_object(InstanceKlass* k) {
+void HeapShared::check_closed_region_object(InstanceKlass* k) {
   // Check fields in the object
   for (JavaFieldStream fs(k); !fs.done(); fs.next()) {
     if (!fs.access_flags().is_static()) {
@@ -958,7 +958,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
                                                oop orig_obj,
                                                bool is_closed_archive) {
   assert(orig_obj != NULL, "must be");
-  assert(!is_archived_object(orig_obj), "sanity");
+  assert(!is_archived_object_during_dumptime(orig_obj), "sanity");
 
   if (!JavaClasses::is_supported_for_archiving(orig_obj)) {
     // This object has injected fields that cannot be supported easily, so we disallow them for now.
@@ -996,7 +996,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
   bool record_klasses_only = (archived_obj != NULL);
   if (archived_obj == NULL) {
     ++_num_new_archived_objs;
-    archived_obj = archive_heap_object(orig_obj);
+    archived_obj = archive_object(orig_obj);
     if (archived_obj == NULL) {
       // Skip archiving the sub-graph referenced from the current entry field.
       ResourceMark rm;
@@ -1037,7 +1037,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
                                   subgraph_info, orig_obj, archived_obj);
   orig_obj->oop_iterate(&walker);
   if (is_closed_archive && orig_k->is_instance_klass()) {
-    check_closed_archive_heap_region_object(InstanceKlass::cast(orig_k));
+    check_closed_region_object(InstanceKlass::cast(orig_k));
   }
   return archived_obj;
 }
@@ -1176,10 +1176,10 @@ void HeapShared::verify_reachable_objects_from(oop obj, bool is_archived) {
     set_has_been_seen_during_subgraph_recording(obj);
 
     if (is_archived) {
-      assert(is_archived_object(obj), "must be");
+      assert(is_archived_object_during_dumptime(obj), "must be");
       assert(find_archived_heap_object(obj) == NULL, "must be");
     } else {
-      assert(!is_archived_object(obj), "must be");
+      assert(!is_archived_object_during_dumptime(obj), "must be");
       assert(find_archived_heap_object(obj) != NULL, "must be");
     }
 
@@ -1439,8 +1439,10 @@ class PatchEmbeddedPointers: public BitMapClosure {
   }
 };
 
-void HeapShared::patch_archived_heap_embedded_pointers(MemRegion region, address oopmap,
-                                                       size_t oopmap_size_in_bits) {
+// Patch all the non-null pointers that are embedded in the archived heap objects
+// in this region
+void HeapShared::patch_embedded_pointers(MemRegion region, address oopmap,
+                                         size_t oopmap_size_in_bits) {
   BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits);
 
 #ifndef PRODUCT
diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp
index 74de74d6c92..873784ab784 100644
--- a/src/hotspot/share/cds/heapShared.hpp
+++ b/src/hotspot/share/cds/heapShared.hpp
@@ -143,9 +143,8 @@ class HeapShared: AllStatic {
  private:
 
 #if INCLUDE_CDS_JAVA_HEAP
-  static bool _closed_archive_heap_region_mapped;
-  static bool _open_archive_heap_region_mapped;
-  static bool _archive_heap_region_fixed;
+  static bool _closed_regions_mapped;
+  static bool _open_regions_mapped;
   static DumpedInternedStrings *_dumped_interned_strings;
 
 public:
@@ -200,7 +199,7 @@ class HeapShared: AllStatic {
   static DumpTimeKlassSubGraphInfoTable* _dump_time_subgraph_info_table;
   static RunTimeKlassSubGraphInfoTable _run_time_subgraph_info_table;
 
-  static void check_closed_archive_heap_region_object(InstanceKlass* k);
+  static void check_closed_region_object(InstanceKlass* k);
 
   static void archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
                                        int num,
@@ -297,21 +296,14 @@ class HeapShared: AllStatic {
   }
 
   static oop find_archived_heap_object(oop obj);
-  static oop archive_heap_object(oop obj);
+  static oop archive_object(oop obj);
 
   static void archive_klass_objects();
 
-  static void set_archive_heap_region_fixed() {
-    _archive_heap_region_fixed = true;
-  }
-  static bool archive_heap_region_fixed() {
-    return _archive_heap_region_fixed;
-  }
-
-  static void archive_java_heap_objects(GrowableArray<MemRegion> *closed,
-                                        GrowableArray<MemRegion> *open);
-  static void copy_closed_archive_heap_objects(GrowableArray<MemRegion> * closed_archive);
-  static void copy_open_archive_heap_objects(GrowableArray<MemRegion> * open_archive);
+  static void archive_objects(GrowableArray<MemRegion>* closed_regions,
+                              GrowableArray<MemRegion>* open_regions);
+  static void copy_closed_objects(GrowableArray<MemRegion>* closed_regions);
+  static void copy_open_objects(GrowableArray<MemRegion>* open_regions);
 
   static oop archive_reachable_objects_from(int level,
                                             KlassSubGraphInfo* subgraph_info,
@@ -357,34 +349,34 @@ class HeapShared: AllStatic {
   }
 
   static bool is_heap_region(int idx) {
-    CDS_JAVA_HEAP_ONLY(return (idx >= MetaspaceShared::first_closed_archive_heap_region &&
-                               idx <= MetaspaceShared::last_open_archive_heap_region);)
+    CDS_JAVA_HEAP_ONLY(return (idx >= MetaspaceShared::first_closed_heap_region &&
+                               idx <= MetaspaceShared::last_open_heap_region);)
     NOT_CDS_JAVA_HEAP_RETURN_(false);
   }
 
-  static void set_closed_archive_heap_region_mapped() {
-    CDS_JAVA_HEAP_ONLY(_closed_archive_heap_region_mapped = true;)
+  static void set_closed_regions_mapped() {
+    CDS_JAVA_HEAP_ONLY(_closed_regions_mapped = true;)
     NOT_CDS_JAVA_HEAP_RETURN;
   }
-  static bool closed_archive_heap_region_mapped() {
-    CDS_JAVA_HEAP_ONLY(return _closed_archive_heap_region_mapped;)
+  static bool closed_regions_mapped() {
+    CDS_JAVA_HEAP_ONLY(return _closed_regions_mapped;)
     NOT_CDS_JAVA_HEAP_RETURN_(false);
   }
-  static void set_open_archive_heap_region_mapped() {
-    CDS_JAVA_HEAP_ONLY(_open_archive_heap_region_mapped = true;)
+  static void set_open_regions_mapped() {
+    CDS_JAVA_HEAP_ONLY(_open_regions_mapped = true;)
     NOT_CDS_JAVA_HEAP_RETURN;
   }
-  static bool open_archive_heap_region_mapped() {
-    CDS_JAVA_HEAP_ONLY(return _open_archive_heap_region_mapped;)
+  static bool open_regions_mapped() {
+    CDS_JAVA_HEAP_ONLY(return _open_regions_mapped;)
     NOT_CDS_JAVA_HEAP_RETURN_(false);
   }
   static bool is_mapped() {
-    return closed_archive_heap_region_mapped() && open_archive_heap_region_mapped();
+    return closed_regions_mapped() && open_regions_mapped();
   }
 
-  static void fixup_mapped_heap_regions() NOT_CDS_JAVA_HEAP_RETURN;
+  static void fixup_mapped_regions() NOT_CDS_JAVA_HEAP_RETURN;
 
-  inline static bool is_archived_object(oop p) NOT_CDS_JAVA_HEAP_RETURN_(false);
+  static bool is_archived_object_during_dumptime(oop p) NOT_CDS_JAVA_HEAP_RETURN_(false);
 
   static void resolve_classes(JavaThread* THREAD) NOT_CDS_JAVA_HEAP_RETURN;
   static void initialize_from_archived_subgraph(Klass* k, JavaThread* THREAD) NOT_CDS_JAVA_HEAP_RETURN;
@@ -397,8 +389,8 @@ class HeapShared: AllStatic {
 
   static void init_narrow_oop_decoding(address base, int shift) NOT_CDS_JAVA_HEAP_RETURN;
 
-  static void patch_archived_heap_embedded_pointers(MemRegion mem, address  oopmap,
-                                                    size_t oopmap_in_bits) NOT_CDS_JAVA_HEAP_RETURN;
+  static void patch_embedded_pointers(MemRegion region, address oopmap,
+                                      size_t oopmap_in_bits) NOT_CDS_JAVA_HEAP_RETURN;
 
   static void init_for_dumping(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
   static void write_subgraph_info_table() NOT_CDS_JAVA_HEAP_RETURN;
diff --git a/src/hotspot/share/cds/heapShared.inline.hpp b/src/hotspot/share/cds/heapShared.inline.hpp
index c3eeee060d4..1f0c6697959 100644
--- a/src/hotspot/share/cds/heapShared.inline.hpp
+++ b/src/hotspot/share/cds/heapShared.inline.hpp
@@ -26,17 +26,11 @@
 #define SHARE_CDS_HEAPSHARED_INLINE_HPP
 
 #include "cds/heapShared.hpp"
-
-#include "gc/shared/collectedHeap.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "utilities/align.hpp"
 
 #if INCLUDE_CDS_JAVA_HEAP
 
-bool HeapShared::is_archived_object(oop p) {
-  return Universe::heap()->is_archived_object(p);
-}
-
 inline oop HeapShared::decode_from_archive(narrowOop v) {
   assert(!CompressedOops::is_null(v), "narrow oop value can never be zero");
   oop result = cast_to_oop((uintptr_t)_narrow_oop_base + ((uintptr_t)v << _narrow_oop_shift));
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
new file mode 100644
index 00000000000..78917ebaae3
--- /dev/null
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "cds/archiveBuilder.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
+#include "classfile/systemDictionaryShared.hpp"
+
+DumpTimeLambdaProxyClassInfo DumpTimeLambdaProxyClassInfo::clone() {
+  DumpTimeLambdaProxyClassInfo res;
+  res._proxy_klasses = NULL;
+  if (_proxy_klasses != NULL && _proxy_klasses->length() > 0) {
+    int num_proxy_klasses = _proxy_klasses->length();
+    res._proxy_klasses = new (ResourceObj::C_HEAP, mtClassShared) GrowableArray<InstanceKlass*>(num_proxy_klasses, mtClassShared);
+    for (int i = 0; i < num_proxy_klasses; i++) {
+      res._proxy_klasses->append(_proxy_klasses->at(i));
+    }
+  }
+  return res;
+}
+
+void LambdaProxyClassKey::mark_pointers() {
+  ArchivePtrMarker::mark_pointer(&_caller_ik);
+  ArchivePtrMarker::mark_pointer(&_instantiated_method_type);
+  ArchivePtrMarker::mark_pointer(&_invoked_name);
+  ArchivePtrMarker::mark_pointer(&_invoked_type);
+  ArchivePtrMarker::mark_pointer(&_member_method);
+  ArchivePtrMarker::mark_pointer(&_method_type);
+}
+
+unsigned int LambdaProxyClassKey::hash() const {
+  return SystemDictionaryShared::hash_for_shared_dictionary((address)_caller_ik) +
+         SystemDictionaryShared::hash_for_shared_dictionary((address)_invoked_name) +
+         SystemDictionaryShared::hash_for_shared_dictionary((address)_invoked_type) +
+         SystemDictionaryShared::hash_for_shared_dictionary((address)_method_type) +
+         SystemDictionaryShared::hash_for_shared_dictionary((address)_instantiated_method_type);
+}
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
new file mode 100644
index 00000000000..e7153aa450e
--- /dev/null
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARED_CDS_LAMBDAPROXYCLASSINFO_HPP
+#define SHARED_CDS_LAMBDAPROXYCLASSINFO_HPP
+#include "cds/metaspaceShared.hpp"
+#include "classfile/javaClasses.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/resourceHash.hpp"
+
+class InstanceKlass;
+class Method;
+class Symbol;
+
+class LambdaProxyClassKey {
+  InstanceKlass* _caller_ik;
+  Symbol*        _invoked_name;
+  Symbol*        _invoked_type;
+  Symbol*        _method_type;
+  Method*        _member_method;
+  Symbol*        _instantiated_method_type;
+
+public:
+  LambdaProxyClassKey(InstanceKlass* caller_ik,
+                      Symbol*        invoked_name,
+                      Symbol*        invoked_type,
+                      Symbol*        method_type,
+                      Method*        member_method,
+                      Symbol*        instantiated_method_type) :
+    _caller_ik(caller_ik),
+    _invoked_name(invoked_name),
+    _invoked_type(invoked_type),
+    _method_type(method_type),
+    _member_method(member_method),
+    _instantiated_method_type(instantiated_method_type) {}
+
+  void metaspace_pointers_do(MetaspaceClosure* it) {
+    it->push(&_caller_ik);
+    it->push(&_invoked_name);
+    it->push(&_invoked_type);
+    it->push(&_method_type);
+    it->push(&_member_method);
+    it->push(&_instantiated_method_type);
+  }
+
+  bool equals(LambdaProxyClassKey const& other) const {
+    return _caller_ik == other._caller_ik &&
+           _invoked_name == other._invoked_name &&
+           _invoked_type == other._invoked_type &&
+           _method_type == other._method_type &&
+           _member_method == other._member_method &&
+           _instantiated_method_type == other._instantiated_method_type;
+  }
+
+  void mark_pointers();
+  unsigned int hash() const;
+
+  static unsigned int dumptime_hash(Symbol* sym)  {
+    if (sym == NULL) {
+      // _invoked_name maybe NULL
+      return 0;
+    }
+    return java_lang_String::hash_code((const jbyte*)sym->bytes(), sym->utf8_length());
+  }
+
+  unsigned int dumptime_hash() const {
+    return dumptime_hash(_caller_ik->name()) +
+           dumptime_hash(_invoked_name) +
+           dumptime_hash(_invoked_type) +
+           dumptime_hash(_method_type) +
+           dumptime_hash(_instantiated_method_type);
+  }
+
+  static inline unsigned int DUMPTIME_HASH(LambdaProxyClassKey const& key) {
+    return (key.dumptime_hash());
+  }
+
+  static inline bool DUMPTIME_EQUALS(
+      LambdaProxyClassKey const& k1, LambdaProxyClassKey const& k2) {
+    return (k1.equals(k2));
+  }
+
+  InstanceKlass* caller_ik() const { return _caller_ik; }
+};
+
+class DumpTimeLambdaProxyClassInfo {
+public:
+  GrowableArray<InstanceKlass*>* _proxy_klasses;
+  DumpTimeLambdaProxyClassInfo() : _proxy_klasses(NULL) {}
+  void add_proxy_klass(InstanceKlass* proxy_klass) {
+    if (_proxy_klasses == NULL) {
+      _proxy_klasses = new (ResourceObj::C_HEAP, mtClassShared) GrowableArray<InstanceKlass*>(5, mtClassShared);
+    }
+    assert(_proxy_klasses != NULL, "sanity");
+    _proxy_klasses->append(proxy_klass);
+  }
+
+  void metaspace_pointers_do(MetaspaceClosure* it) {
+    for (int i=0; i<_proxy_klasses->length(); i++) {
+      it->push(_proxy_klasses->adr_at(i));
+    }
+  }
+  DumpTimeLambdaProxyClassInfo clone(); // copy ctor will cause implicitly-declared
+};
+
+class RunTimeLambdaProxyClassInfo {
+  LambdaProxyClassKey _key;
+  InstanceKlass* _proxy_klass_head;
+public:
+  RunTimeLambdaProxyClassInfo(LambdaProxyClassKey key, InstanceKlass* proxy_klass_head) :
+    _key(key), _proxy_klass_head(proxy_klass_head) {}
+
+  InstanceKlass* proxy_klass_head() const { return _proxy_klass_head; }
+
+  // Used by LambdaProxyClassDictionary to implement OffsetCompactHashtable::EQUALS
+  static inline bool EQUALS(
+       const RunTimeLambdaProxyClassInfo* value, LambdaProxyClassKey* key, int len_unused) {
+    return (value->_key.equals(*key));
+  }
+  void init(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+    _key = key;
+    _key.mark_pointers();
+    _proxy_klass_head = info._proxy_klasses->at(0);
+    ArchivePtrMarker::mark_pointer(&_proxy_klass_head);
+  }
+
+  unsigned int hash() const {
+    return _key.hash();
+  }
+  LambdaProxyClassKey key() const {
+    return _key;
+  }
+};
+
+class DumpTimeLambdaProxyClassDictionary
+  : public ResourceHashtable<LambdaProxyClassKey,
+                             DumpTimeLambdaProxyClassInfo,
+                             LambdaProxyClassKey::DUMPTIME_HASH,
+                             LambdaProxyClassKey::DUMPTIME_EQUALS,
+                             137, // prime number
+                             ResourceObj::C_HEAP> {
+public:
+  DumpTimeLambdaProxyClassDictionary() : _count(0) {}
+  int _count;
+};
+
+class LambdaProxyClassDictionary : public OffsetCompactHashtable<
+  LambdaProxyClassKey*,
+  const RunTimeLambdaProxyClassInfo*,
+  RunTimeLambdaProxyClassInfo::EQUALS> {};
+
+#endif // SHARED_CDS_LAMBDAPROXYCLASSINFO_HPP
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index 1b174ebbc9d..2c73ba06e26 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "jvm_io.h"
 #include "cds/archiveBuilder.hpp"
+#include "cds/cdsProtectionDomain.hpp"
 #include "cds/classListParser.hpp"
 #include "cds/cppVtables.hpp"
 #include "cds/dumpAllocStats.hpp"
@@ -71,6 +72,7 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/vmThread.hpp"
 #include "runtime/vmOperations.hpp"
+#include "services/memTracker.hpp"
 #include "utilities/align.hpp"
 #include "utilities/bitMap.inline.hpp"
 #include "utilities/ostream.hpp"
@@ -114,7 +116,7 @@ bool MetaspaceShared::_use_full_module_graph = true;
 // [5] SymbolTable, StringTable, SystemDictionary, and a few other read-only data
 //     are copied into the ro region as read-only tables.
 //
-// The ca0/ca1 and oa0/oa1 regions are populated inside HeapShared::archive_java_heap_objects.
+// The ca0/ca1 and oa0/oa1 regions are populated inside HeapShared::archive_objects.
 // Their layout is independent of the rw/ro regions.
 
 static DumpRegion _symbol_region("symbols");
@@ -248,7 +250,7 @@ void MetaspaceShared::post_initialize(TRAPS) {
   if (UseSharedSpaces) {
     int size = FileMapInfo::get_number_of_shared_paths();
     if (size > 0) {
-      SystemDictionaryShared::allocate_shared_data_arrays(size, CHECK);
+      CDSProtectionDomain::allocate_shared_data_arrays(size, CHECK);
       if (!DynamicDumpSharedSpaces) {
         FileMapInfo* info;
         if (FileMapInfo::dynamic_info() == NULL) {
@@ -392,7 +394,7 @@ static void rewrite_nofast_bytecode(const methodHandle& method) {
 void MetaspaceShared::rewrite_nofast_bytecodes_and_calculate_fingerprints(Thread* thread, InstanceKlass* ik) {
   for (int i = 0; i < ik->methods()->length(); i++) {
     methodHandle m(thread, ik->methods()->at(i));
-    if (ik->can_be_verified_at_dumptime()) {
+    if (ik->can_be_verified_at_dumptime() && ik->is_linked()) {
       rewrite_nofast_bytecode(m);
     }
     Fingerprinter fp(m);
@@ -403,15 +405,15 @@ void MetaspaceShared::rewrite_nofast_bytecodes_and_calculate_fingerprints(Thread
 
 class VM_PopulateDumpSharedSpace : public VM_GC_Operation {
 private:
-  GrowableArray<MemRegion> *_closed_archive_heap_regions;
-  GrowableArray<MemRegion> *_open_archive_heap_regions;
+  GrowableArray<MemRegion> *_closed_heap_regions;
+  GrowableArray<MemRegion> *_open_heap_regions;
 
-  GrowableArray<ArchiveHeapOopmapInfo> *_closed_archive_heap_oopmaps;
-  GrowableArray<ArchiveHeapOopmapInfo> *_open_archive_heap_oopmaps;
+  GrowableArray<ArchiveHeapOopmapInfo> *_closed_heap_oopmaps;
+  GrowableArray<ArchiveHeapOopmapInfo> *_open_heap_oopmaps;
 
   void dump_java_heap_objects(GrowableArray<Klass*>* klasses) NOT_CDS_JAVA_HEAP_RETURN;
-  void dump_archive_heap_oopmaps() NOT_CDS_JAVA_HEAP_RETURN;
-  void dump_archive_heap_oopmaps(GrowableArray<MemRegion>* regions,
+  void dump_heap_oopmaps() NOT_CDS_JAVA_HEAP_RETURN;
+  void dump_heap_oopmaps(GrowableArray<MemRegion>* regions,
                                  GrowableArray<ArchiveHeapOopmapInfo>* oopmaps);
   void dump_shared_symbol_table(GrowableArray<Symbol*>* symbols) {
     log_info(cds)("Dumping symbol table ...");
@@ -423,10 +425,10 @@ class VM_PopulateDumpSharedSpace : public VM_GC_Operation {
 
   VM_PopulateDumpSharedSpace() :
     VM_GC_Operation(0 /* total collections, ignored */, GCCause::_archive_time_gc),
-    _closed_archive_heap_regions(NULL),
-    _open_archive_heap_regions(NULL),
-    _closed_archive_heap_oopmaps(NULL),
-    _open_archive_heap_oopmaps(NULL) {}
+    _closed_heap_regions(NULL),
+    _open_heap_regions(NULL),
+    _closed_heap_oopmaps(NULL),
+    _open_heap_oopmaps(NULL) {}
 
   bool skip_operation() const { return false; }
 
@@ -472,7 +474,7 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() {
   MetaspaceShared::serialize(&wc);
 
   // Write the bitmaps for patching the archive heap regions
-  dump_archive_heap_oopmaps();
+  dump_heap_oopmaps();
 
   return start;
 }
@@ -486,12 +488,10 @@ void VM_PopulateDumpSharedSpace::doit() {
 
   NOT_PRODUCT(SystemDictionary::verify();)
 
-  // At this point, many classes have been loaded.
-  // Gather systemDictionary classes in a global array and do everything to
-  // that so we don't have to walk the SystemDictionary again.
+  // Block concurrent class unloading from changing the _dumptime_table
+  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
   SystemDictionaryShared::check_excluded_classes();
 
-  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
   StaticArchiveBuilder builder;
   builder.gather_source_objs();
   builder.reserve_buffer();
@@ -532,10 +532,10 @@ void VM_PopulateDumpSharedSpace::doit() {
   mapinfo->set_cloned_vtables(cloned_vtables);
   mapinfo->open_for_write();
   builder.write_archive(mapinfo,
-                        _closed_archive_heap_regions,
-                        _open_archive_heap_regions,
-                        _closed_archive_heap_oopmaps,
-                        _open_archive_heap_oopmaps);
+                        _closed_heap_regions,
+                        _open_heap_regions,
+                        _closed_heap_oopmaps,
+                        _open_heap_oopmaps);
 
   if (PrintSystemDictionaryAtExit) {
     SystemDictionary::print();
@@ -573,10 +573,26 @@ class CollectCLDClosure : public CLDClosure {
   ClassLoaderData* cld_at(int index) { return _loaded_cld.at(index); }
 };
 
-bool MetaspaceShared::linking_required(InstanceKlass* ik) {
-  // For static CDS dump, do not link old classes.
-  // For dynamic CDS dump, only link classes loaded by the builtin class loaders.
-  return DumpSharedSpaces ? ik->can_be_verified_at_dumptime() : !ik->is_shared_unregistered_class();
+// Check if we can eagerly link this class at dump time, so we can avoid the
+// runtime linking overhead (especially verification)
+bool MetaspaceShared::may_be_eagerly_linked(InstanceKlass* ik) {
+  if (!ik->can_be_verified_at_dumptime()) {
+    // For old classes, try to leave them in the unlinked state, so
+    // we can still store them in the archive. They must be
+    // linked/verified at runtime.
+    return false;
+  }
+  if (DynamicDumpSharedSpaces && ik->is_shared_unregistered_class()) {
+    // Linking of unregistered classes at this stage may cause more
+    // classes to be resolved, resulting in calls to ClassLoader.loadClass()
+    // that may not be expected by custom class loaders.
+    //
+    // It's OK to do this for the built-in loaders as we know they can
+    // tolerate this. (Note that unregistered classes are loaded by the NULL
+    // loader during DumpSharedSpaces).
+    return false;
+  }
+  return true;
 }
 
 bool MetaspaceShared::link_class_for_cds(InstanceKlass* ik, TRAPS) {
@@ -594,7 +610,7 @@ bool MetaspaceShared::link_class_for_cds(InstanceKlass* ik, TRAPS) {
   return res;
 }
 
-void MetaspaceShared::link_and_cleanup_shared_classes(TRAPS) {
+void MetaspaceShared::link_shared_classes(TRAPS) {
   // Collect all loaded ClassLoaderData.
   ResourceMark rm;
 
@@ -616,7 +632,7 @@ void MetaspaceShared::link_and_cleanup_shared_classes(TRAPS) {
       for (Klass* klass = cld->klasses(); klass != NULL; klass = klass->next_link()) {
         if (klass->is_instance_klass()) {
           InstanceKlass* ik = InstanceKlass::cast(klass);
-          if (linking_required(ik)) {
+          if (may_be_eagerly_linked(ik)) {
             has_linked |= link_class_for_cds(ik, CHECK);
           }
         }
@@ -704,7 +720,7 @@ void MetaspaceShared::preload_classes(TRAPS) {
   // Exercise the manifest processing code to ensure classes used by CDS at runtime
   // are always archived
   const char* dummy = "Manifest-Version: 1.0\n";
-  SystemDictionaryShared::create_jar_manifest(dummy, strlen(dummy), CHECK);
+  CDSProtectionDomain::create_jar_manifest(dummy, strlen(dummy), CHECK);
 
   log_info(cds)("Loading classes to share: done.");
   log_info(cds)("Shared spaces: preloaded %d classes", class_count);
@@ -728,7 +744,7 @@ void MetaspaceShared::preload_and_dump_impl(TRAPS) {
   // were not explicitly specified in the classlist. E.g., if an interface implemented by class K
   // fails verification, all other interfaces that were not specified in the classlist but
   // are implemented by K are not verified.
-  link_and_cleanup_shared_classes(CHECK);
+  link_shared_classes(CHECK);
   log_info(cds)("Rewriting and linking classes: done");
 
 #if INCLUDE_CDS_JAVA_HEAP
@@ -811,27 +827,26 @@ void VM_PopulateDumpSharedSpace::dump_java_heap_objects(GrowableArray<Klass*>* k
   }
 
   // The closed and open archive heap space has maximum two regions.
-  // See FileMapInfo::write_archive_heap_regions() for details.
-  _closed_archive_heap_regions = new GrowableArray<MemRegion>(2);
-  _open_archive_heap_regions = new GrowableArray<MemRegion>(2);
-  HeapShared::archive_java_heap_objects(_closed_archive_heap_regions,
-                                        _open_archive_heap_regions);
+  // See FileMapInfo::write_heap_regions() for details.
+  _closed_heap_regions = new GrowableArray<MemRegion>(2);
+  _open_heap_regions = new GrowableArray<MemRegion>(2);
+  HeapShared::archive_objects(_closed_heap_regions, _open_heap_regions);
   ArchiveBuilder::OtherROAllocMark mark;
   HeapShared::write_subgraph_info_table();
 }
 
-void VM_PopulateDumpSharedSpace::dump_archive_heap_oopmaps() {
+void VM_PopulateDumpSharedSpace::dump_heap_oopmaps() {
   if (HeapShared::is_heap_object_archiving_allowed()) {
-    _closed_archive_heap_oopmaps = new GrowableArray<ArchiveHeapOopmapInfo>(2);
-    dump_archive_heap_oopmaps(_closed_archive_heap_regions, _closed_archive_heap_oopmaps);
+    _closed_heap_oopmaps = new GrowableArray<ArchiveHeapOopmapInfo>(2);
+    dump_heap_oopmaps(_closed_heap_regions, _closed_heap_oopmaps);
 
-    _open_archive_heap_oopmaps = new GrowableArray<ArchiveHeapOopmapInfo>(2);
-    dump_archive_heap_oopmaps(_open_archive_heap_regions, _open_archive_heap_oopmaps);
+    _open_heap_oopmaps = new GrowableArray<ArchiveHeapOopmapInfo>(2);
+    dump_heap_oopmaps(_open_heap_regions, _open_heap_oopmaps);
   }
 }
 
-void VM_PopulateDumpSharedSpace::dump_archive_heap_oopmaps(GrowableArray<MemRegion>* regions,
-                                                           GrowableArray<ArchiveHeapOopmapInfo>* oopmaps) {
+void VM_PopulateDumpSharedSpace::dump_heap_oopmaps(GrowableArray<MemRegion>* regions,
+                                                   GrowableArray<ArchiveHeapOopmapInfo>* oopmaps) {
   for (int i=0; i<regions->length(); i++) {
     ResourceBitMap oopmap = HeapShared::calculate_oopmap(regions->at(i));
     size_t size_in_bits = oopmap.size();
@@ -1386,7 +1401,7 @@ void MetaspaceShared::initialize_shared_spaces() {
   // Initialize the run-time symbol table.
   SymbolTable::create_table();
 
-  static_mapinfo->patch_archived_heap_embedded_pointers();
+  static_mapinfo->patch_heap_embedded_pointers();
 
   // Close the mapinfo file
   static_mapinfo->close();
diff --git a/src/hotspot/share/cds/metaspaceShared.hpp b/src/hotspot/share/cds/metaspaceShared.hpp
index 553facc6134..34ff4f32452 100644
--- a/src/hotspot/share/cds/metaspaceShared.hpp
+++ b/src/hotspot/share/cds/metaspaceShared.hpp
@@ -58,21 +58,21 @@ class MetaspaceShared : AllStatic {
  public:
   enum {
     // core archive spaces
-    rw = 0,  // read-write shared space in the heap
-    ro = 1,  // read-only shared space in the heap
+    rw = 0,  // read-write shared space
+    ro = 1,  // read-only shared space
     bm = 2,  // relocation bitmaps (freed after file mapping is finished)
     num_core_region = 2,       // rw and ro
     num_non_heap_spaces = 3,   // rw and ro and bm
 
     // mapped java heap regions
-    first_closed_archive_heap_region = bm + 1,
-    max_closed_archive_heap_region = 2,
-    last_closed_archive_heap_region = first_closed_archive_heap_region + max_closed_archive_heap_region - 1,
-    first_open_archive_heap_region = last_closed_archive_heap_region + 1,
-    max_open_archive_heap_region = 2,
-    last_open_archive_heap_region = first_open_archive_heap_region + max_open_archive_heap_region - 1,
-
-    last_valid_region = last_open_archive_heap_region,
+    first_closed_heap_region = bm + 1,
+    max_closed_heap_region = 2,
+    last_closed_heap_region = first_closed_heap_region + max_closed_heap_region - 1,
+    first_open_heap_region = last_closed_heap_region + 1,
+    max_open_heap_region = 2,
+    last_open_heap_region = first_open_heap_region + max_open_heap_region - 1,
+
+    last_valid_region = last_open_heap_region,
     n_regions =  last_valid_region + 1 // total number of regions
   };
 
@@ -101,19 +101,14 @@ class MetaspaceShared : AllStatic {
     _archive_loading_failed = true;
   }
 
-  static bool map_shared_spaces(FileMapInfo* mapinfo) NOT_CDS_RETURN_(false);
   static void initialize_shared_spaces() NOT_CDS_RETURN;
 
   // Return true if given address is in the shared metaspace regions (i.e., excluding any
-  // mapped shared heap regions.)
+  // mapped heap regions.)
   static bool is_in_shared_metaspace(const void* p) {
     return MetaspaceObj::is_shared((const MetaspaceObj*)p);
   }
 
-  static address shared_metaspace_top() {
-    return (address)MetaspaceObj::shared_metaspace_top();
-  }
-
   static void set_shared_metaspace_range(void* base, void *static_top, void* top) NOT_CDS_RETURN;
 
   // Return true if given address is in the shared region corresponding to the idx
@@ -134,9 +129,9 @@ class MetaspaceShared : AllStatic {
   }
 
   static bool try_link_class(JavaThread* current, InstanceKlass* ik);
-  static void link_and_cleanup_shared_classes(TRAPS) NOT_CDS_RETURN;
+  static void link_shared_classes(TRAPS) NOT_CDS_RETURN;
   static bool link_class_for_cds(InstanceKlass* ik, TRAPS) NOT_CDS_RETURN_(false);
-  static bool linking_required(InstanceKlass* ik) NOT_CDS_RETURN_(false);
+  static bool may_be_eagerly_linked(InstanceKlass* ik) NOT_CDS_RETURN_(false);
 
 #if INCLUDE_CDS
   // Alignment for the 2 core CDS regions (RW/RO) only.
diff --git a/src/hotspot/share/cds/runTimeClassInfo.cpp b/src/hotspot/share/cds/runTimeClassInfo.cpp
new file mode 100644
index 00000000000..52fa94c119d
--- /dev/null
+++ b/src/hotspot/share/cds/runTimeClassInfo.cpp
@@ -0,0 +1,76 @@
+
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "cds/archiveBuilder.hpp"
+#include "cds/runTimeClassInfo.hpp"
+
+void RunTimeClassInfo::init(DumpTimeClassInfo& info) {
+  ArchiveBuilder* builder = ArchiveBuilder::current();
+  assert(builder->is_in_buffer_space(info._klass), "must be");
+  _klass = info._klass;
+  if (!SystemDictionaryShared::is_builtin(_klass)) {
+    CrcInfo* c = crc();
+    c->_clsfile_size = info._clsfile_size;
+    c->_clsfile_crc32 = info._clsfile_crc32;
+  }
+  _num_verifier_constraints = info.num_verifier_constraints();
+  _num_loader_constraints   = info.num_loader_constraints();
+  int i;
+  if (_num_verifier_constraints > 0) {
+    RTVerifierConstraint* vf_constraints = verifier_constraints();
+    char* flags = verifier_constraint_flags();
+    for (i = 0; i < _num_verifier_constraints; i++) {
+      vf_constraints[i]._name      = builder->any_to_offset_u4(info._verifier_constraints->at(i)._name);
+      vf_constraints[i]._from_name = builder->any_to_offset_u4(info._verifier_constraints->at(i)._from_name);
+    }
+    for (i = 0; i < _num_verifier_constraints; i++) {
+      flags[i] = info._verifier_constraint_flags->at(i);
+    }
+  }
+
+  if (_num_loader_constraints > 0) {
+    RTLoaderConstraint* ld_constraints = loader_constraints();
+    for (i = 0; i < _num_loader_constraints; i++) {
+      ld_constraints[i]._name = builder->any_to_offset_u4(info._loader_constraints->at(i)._name);
+      ld_constraints[i]._loader_type1 = info._loader_constraints->at(i)._loader_type1;
+      ld_constraints[i]._loader_type2 = info._loader_constraints->at(i)._loader_type2;
+    }
+  }
+
+  if (_klass->is_hidden()) {
+    InstanceKlass* n_h = info.nest_host();
+    set_nest_host(n_h);
+  }
+  ArchivePtrMarker::mark_pointer(&_klass);
+}
+
+size_t RunTimeClassInfo::crc_size(InstanceKlass* klass) {
+  if (!SystemDictionaryShared::is_builtin(klass)) {
+    return sizeof(CrcInfo);
+  } else {
+    return 0;
+  }
+}
diff --git a/src/hotspot/share/cds/runTimeClassInfo.hpp b/src/hotspot/share/cds/runTimeClassInfo.hpp
new file mode 100644
index 00000000000..adc828c4f88
--- /dev/null
+++ b/src/hotspot/share/cds/runTimeClassInfo.hpp
@@ -0,0 +1,226 @@
+
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARED_CDS_SHAREDCLASSINFO_HPP
+#define SHARED_CDS_SHAREDCLASSINFO_HPP
+#include "classfile/compactHashtable.hpp"
+#include "classfile/javaClasses.hpp"
+#include "classfile/systemDictionaryShared.hpp"
+#include "cds/archiveBuilder.hpp"
+#include "cds/archiveUtils.hpp"
+#include "cds/metaspaceShared.hpp"
+#include "memory/metaspaceClosure.hpp"
+#include "oops/instanceKlass.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/resourceHash.hpp"
+
+class Method;
+class Symbol;
+
+class RunTimeClassInfo {
+public:
+  struct CrcInfo {
+    int _clsfile_size;
+    int _clsfile_crc32;
+  };
+
+  // This is different than  DumpTimeClassInfo::DTVerifierConstraint. We use
+  // u4 instead of Symbol* to save space on 64-bit CPU.
+  struct RTVerifierConstraint {
+    u4 _name;
+    u4 _from_name;
+    Symbol* name() { return (Symbol*)(SharedBaseAddress + _name);}
+    Symbol* from_name() { return (Symbol*)(SharedBaseAddress + _from_name); }
+  };
+
+  struct RTLoaderConstraint {
+    u4   _name;
+    char _loader_type1;
+    char _loader_type2;
+    Symbol* constraint_name() {
+      return (Symbol*)(SharedBaseAddress + _name);
+    }
+  };
+
+  InstanceKlass* _klass;
+  int _num_verifier_constraints;
+  int _num_loader_constraints;
+
+  // optional CrcInfo              _crc;  (only for UNREGISTERED classes)
+  // optional InstanceKlass*       _nest_host
+  // optional RTLoaderConstraint   _loader_constraint_types[_num_loader_constraints]
+  // optional RTVerifierConstraint _verifier_constraints[_num_verifier_constraints]
+  // optional char                 _verifier_constraint_flags[_num_verifier_constraints]
+
+private:
+  static size_t header_size_size() {
+    return sizeof(RunTimeClassInfo);
+  }
+  static size_t verifier_constraints_size(int num_verifier_constraints) {
+    return sizeof(RTVerifierConstraint) * num_verifier_constraints;
+  }
+  static size_t verifier_constraint_flags_size(int num_verifier_constraints) {
+    return sizeof(char) * num_verifier_constraints;
+  }
+  static size_t loader_constraints_size(int num_loader_constraints) {
+    return sizeof(RTLoaderConstraint) * num_loader_constraints;
+  }
+  static size_t nest_host_size(InstanceKlass* klass) {
+    if (klass->is_hidden()) {
+      return sizeof(InstanceKlass*);
+    } else {
+      return 0;
+    }
+  }
+
+  static size_t crc_size(InstanceKlass* klass);
+public:
+  static size_t byte_size(InstanceKlass* klass, int num_verifier_constraints, int num_loader_constraints) {
+    return header_size_size() +
+           crc_size(klass) +
+           nest_host_size(klass) +
+           loader_constraints_size(num_loader_constraints) +
+           verifier_constraints_size(num_verifier_constraints) +
+           verifier_constraint_flags_size(num_verifier_constraints);
+  }
+
+private:
+  size_t crc_offset() const {
+    return header_size_size();
+  }
+
+  size_t nest_host_offset() const {
+      return crc_offset() + crc_size(_klass);
+  }
+
+  size_t loader_constraints_offset() const  {
+    return nest_host_offset() + nest_host_size(_klass);
+  }
+  size_t verifier_constraints_offset() const {
+    return loader_constraints_offset() + loader_constraints_size(_num_loader_constraints);
+  }
+  size_t verifier_constraint_flags_offset() const {
+    return verifier_constraints_offset() + verifier_constraints_size(_num_verifier_constraints);
+  }
+
+  void check_verifier_constraint_offset(int i) const {
+    assert(0 <= i && i < _num_verifier_constraints, "sanity");
+  }
+
+  void check_loader_constraint_offset(int i) const {
+    assert(0 <= i && i < _num_loader_constraints, "sanity");
+  }
+
+public:
+  CrcInfo* crc() const {
+    assert(crc_size(_klass) > 0, "must be");
+    return (CrcInfo*)(address(this) + crc_offset());
+  }
+  RTVerifierConstraint* verifier_constraints() {
+    assert(_num_verifier_constraints > 0, "sanity");
+    return (RTVerifierConstraint*)(address(this) + verifier_constraints_offset());
+  }
+  RTVerifierConstraint* verifier_constraint_at(int i) {
+    check_verifier_constraint_offset(i);
+    return verifier_constraints() + i;
+  }
+
+  char* verifier_constraint_flags() {
+    assert(_num_verifier_constraints > 0, "sanity");
+    return (char*)(address(this) + verifier_constraint_flags_offset());
+  }
+
+  InstanceKlass** nest_host_addr() {
+    assert(_klass->is_hidden(), "sanity");
+    return (InstanceKlass**)(address(this) + nest_host_offset());
+  }
+  InstanceKlass* nest_host() {
+    return *nest_host_addr();
+  }
+  void set_nest_host(InstanceKlass* k) {
+    *nest_host_addr() = k;
+    ArchivePtrMarker::mark_pointer((address*)nest_host_addr());
+  }
+
+  RTLoaderConstraint* loader_constraints() {
+    assert(_num_loader_constraints > 0, "sanity");
+    return (RTLoaderConstraint*)(address(this) + loader_constraints_offset());
+  }
+
+  RTLoaderConstraint* loader_constraint_at(int i) {
+    check_loader_constraint_offset(i);
+    return loader_constraints() + i;
+  }
+
+  void init(DumpTimeClassInfo& info);
+
+  bool matches(int clsfile_size, int clsfile_crc32) const {
+    return crc()->_clsfile_size  == clsfile_size &&
+           crc()->_clsfile_crc32 == clsfile_crc32;
+  }
+
+  char verifier_constraint_flag(int i) {
+    check_verifier_constraint_offset(i);
+    return verifier_constraint_flags()[i];
+  }
+
+private:
+  // ArchiveBuilder::make_shallow_copy() has reserved a pointer immediately
+  // before archived InstanceKlasses. We can use this slot to do a quick
+  // lookup of InstanceKlass* -> RunTimeClassInfo* without
+  // building a new hashtable.
+  //
+  //  info_pointer_addr(klass) --> 0x0100   RunTimeClassInfo*
+  //  InstanceKlass* klass     --> 0x0108   <C++ vtbl>
+  //                               0x0110   fields from Klass ...
+  static RunTimeClassInfo** info_pointer_addr(InstanceKlass* klass) {
+    return &((RunTimeClassInfo**)klass)[-1];
+  }
+
+public:
+  static RunTimeClassInfo* get_for(InstanceKlass* klass) {
+    assert(klass->is_shared(), "don't call for non-shared class");
+    return *info_pointer_addr(klass);
+  }
+  static void set_for(InstanceKlass* klass, RunTimeClassInfo* record) {
+    assert(ArchiveBuilder::current()->is_in_buffer_space(klass), "must be");
+    assert(ArchiveBuilder::current()->is_in_buffer_space(record), "must be");
+    *info_pointer_addr(klass) = record;
+    ArchivePtrMarker::mark_pointer(info_pointer_addr(klass));
+  }
+
+  // Used by RunTimeSharedDictionary to implement OffsetCompactHashtable::EQUALS
+  static inline bool EQUALS(
+       const RunTimeClassInfo* value, Symbol* key, int len_unused) {
+    return (value->_klass->name() == key);
+  }
+};
+
+class RunTimeSharedDictionary : public OffsetCompactHashtable<
+  Symbol*,
+  const RunTimeClassInfo*,
+  RunTimeClassInfo::EQUALS> {};
+#endif // SHARED_CDS_SHAREDCLASSINFO_HPP
diff --git a/src/hotspot/share/ci/ciConstant.cpp b/src/hotspot/share/ci/ciConstant.cpp
index fb8a856b172..2e34ba02fed 100644
--- a/src/hotspot/share/ci/ciConstant.cpp
+++ b/src/hotspot/share/ci/ciConstant.cpp
@@ -56,7 +56,6 @@ void ciConstant::print() {
   case T_DOUBLE:
     tty->print("%lf", _value._double);
     break;
-  case T_INLINE_TYPE:
   default:
     if (is_reference_type(basic_type())) {
       _value._object->print();
diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp
index a9c79be9ce0..5092bd959f9 100644
--- a/src/hotspot/share/ci/ciEnv.cpp
+++ b/src/hotspot/share/ci/ciEnv.cpp
@@ -693,7 +693,15 @@ ciConstant ciEnv::get_constant_by_index_impl(const constantPoolHandle& cpool,
     }
     assert (klass->is_instance_klass() || klass->is_array_klass(),
             "must be an instance or array klass ");
-    return ciConstant(T_OBJECT, klass->java_mirror());
+    if (!klass->is_loaded()) {
+      return ciConstant(T_OBJECT, get_unloaded_klass_mirror(klass));
+    } else {
+      if (tag.is_Qdescriptor_klass()) {
+        return ciConstant(T_OBJECT, klass->as_inline_klass()->val_mirror());
+      } else {
+        return ciConstant(T_OBJECT, klass->java_mirror());
+      }
+    }
   } else if (tag.is_method_type()) {
     // must execute Java code to link this CP entry into cache[i].f1
     ciSymbol* signature = get_symbol(cpool->method_type_signature_at(index));
diff --git a/src/hotspot/share/ci/ciInlineKlass.cpp b/src/hotspot/share/ci/ciInlineKlass.cpp
index 1c27884a6f1..429f5cfeac2 100644
--- a/src/hotspot/share/ci/ciInlineKlass.cpp
+++ b/src/hotspot/share/ci/ciInlineKlass.cpp
@@ -121,6 +121,20 @@ ciInstance* ciInlineKlass::default_instance() const {
   )
 }
 
+ciInstance* ciInlineKlass::ref_instance() const {
+  GUARDED_VM_ENTRY(
+    oop ref_mirror = to_InlineKlass()->ref_mirror();
+    return CURRENT_ENV->get_instance(ref_mirror);
+  )
+}
+
+ciInstance* ciInlineKlass::val_instance() const {
+  GUARDED_VM_ENTRY(
+    oop val_mirror = to_InlineKlass()->val_mirror();
+    return CURRENT_ENV->get_instance(val_mirror);
+  )
+}
+
 bool ciInlineKlass::contains_oops() const {
   GUARDED_VM_ENTRY(return get_InlineKlass()->contains_oops();)
 }
@@ -140,3 +154,11 @@ address ciInlineKlass::unpack_handler() const {
 InlineKlass* ciInlineKlass::get_InlineKlass() const {
   GUARDED_VM_ENTRY(return to_InlineKlass();)
 }
+
+ciInstance* ciInlineKlass::ref_mirror() {
+  GUARDED_VM_ENTRY(return CURRENT_ENV->get_instance(to_InlineKlass()->ref_mirror());)
+}
+
+ciInstance* ciInlineKlass::val_mirror() {
+  GUARDED_VM_ENTRY(return CURRENT_ENV->get_instance(to_InlineKlass()->val_mirror());)
+}
diff --git a/src/hotspot/share/ci/ciInlineKlass.hpp b/src/hotspot/share/ci/ciInlineKlass.hpp
index 21934174e5e..c172d487097 100644
--- a/src/hotspot/share/ci/ciInlineKlass.hpp
+++ b/src/hotspot/share/ci/ciInlineKlass.hpp
@@ -85,11 +85,15 @@ class ciInlineKlass : public ciInstanceKlass {
   int inline_arg_slots();
   int default_value_offset() const;
   ciInstance* default_instance() const;
+  ciInstance* ref_instance() const;
+  ciInstance* val_instance() const;
   bool contains_oops() const;
   int oop_count() const;
   address pack_handler() const;
   address unpack_handler() const;
   InlineKlass* get_InlineKlass() const;
+  ciInstance* ref_mirror();
+  ciInstance* val_mirror();
 };
 
 #endif // SHARE_VM_CI_CIINLINEKLASS_HPP
diff --git a/src/hotspot/share/ci/ciInstance.cpp b/src/hotspot/share/ci/ciInstance.cpp
index 460a829c79f..3cafdf45e57 100644
--- a/src/hotspot/share/ci/ciInstance.cpp
+++ b/src/hotspot/share/ci/ciInstance.cpp
@@ -39,7 +39,7 @@
 
 // ------------------------------------------------------------------
 // ciObject::java_mirror_type
-ciType* ciInstance::java_mirror_type() {
+ciType* ciInstance::java_mirror_type(bool* is_val_mirror) {
   VM_ENTRY_MARK;
   oop m = get_oop();
   // Return NULL if it is not java.lang.Class.
@@ -52,6 +52,9 @@ ciType* ciInstance::java_mirror_type() {
   } else {
     Klass* k = java_lang_Class::as_Klass(m);
     assert(k != NULL, "");
+    if (is_val_mirror != NULL) {
+      *is_val_mirror = java_lang_Class::is_secondary_mirror(m);
+    }
     return CURRENT_THREAD_ENV->get_klass(k);
   }
 }
diff --git a/src/hotspot/share/ci/ciInstance.hpp b/src/hotspot/share/ci/ciInstance.hpp
index aa328391ad4..075ac421191 100644
--- a/src/hotspot/share/ci/ciInstance.hpp
+++ b/src/hotspot/share/ci/ciInstance.hpp
@@ -55,7 +55,7 @@ class ciInstance : public ciObject {
   // If this object is a java mirror, return the corresponding type.
   // Otherwise, return NULL.
   // (Remember that a java mirror is an instance of java.lang.Class.)
-  ciType* java_mirror_type();
+  ciType* java_mirror_type(bool* is_val_mirror = NULL);
 
   // What kind of ciObject is this?
   bool is_instance()     { return true; }
diff --git a/src/hotspot/share/ci/ciObjectFactory.cpp b/src/hotspot/share/ci/ciObjectFactory.cpp
index 23ae4818b4b..693a432ab28 100644
--- a/src/hotspot/share/ci/ciObjectFactory.cpp
+++ b/src/hotspot/share/ci/ciObjectFactory.cpp
@@ -630,7 +630,7 @@ ciReturnAddress* ciObjectFactory::get_return_address(int bci) {
 }
 
 ciWrapper* ciObjectFactory::make_null_free_wrapper(ciType* type) {
-  ciWrapper* wrapper = new (arena()) ciWrapper(type, /* null_free */ true);
+  ciWrapper* wrapper = new (arena()) ciWrapper(type);
   init_ident_of(wrapper);
   return wrapper;
 }
diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp
index 5f6e4366ed7..585d59ae126 100644
--- a/src/hotspot/share/ci/ciReplay.cpp
+++ b/src/hotspot/share/ci/ciReplay.cpp
@@ -841,7 +841,7 @@ class CompileReplay : public StackObj {
       }
       case T_ARRAY:
       case T_OBJECT: {
-        JavaThread* THREAD = Thread::current()->as_Java_thread();
+        JavaThread* THREAD = JavaThread::current();
         bool res = _replay->process_staticfield_reference(string_value, _vt, fd, THREAD);
         assert(res, "should succeed for arrays & objects");
         break;
@@ -854,7 +854,7 @@ class CompileReplay : public StackObj {
           InlineTypeFieldInitializer init_fields(obj, _replay);
           vk->do_nonstatic_fields(&init_fields);
         } else {
-          oop value = vk->allocate_instance(Thread::current()->as_Java_thread());
+          oop value = vk->allocate_instance(JavaThread::current());
           _vt->obj_field_put(fd->offset(), value);
         }
         break;
diff --git a/src/hotspot/share/ci/ciType.hpp b/src/hotspot/share/ci/ciType.hpp
index 3313c9cb388..2df869600ba 100644
--- a/src/hotspot/share/ci/ciType.hpp
+++ b/src/hotspot/share/ci/ciType.hpp
@@ -72,7 +72,7 @@ class ciType : public ciMetadata {
   bool is_type() const                      { return true; }
   bool is_classless() const                 { return is_primitive_type(); }
 
-  virtual ciType*     unwrap()              { return this; }
+  virtual ciType* unwrap()                  { return this; }
   virtual bool is_null_free() const         { return false; }
 
   const char* name();
@@ -112,22 +112,20 @@ class ciReturnAddress : public ciType {
 
 // ciWrapper
 //
-// This class wraps another type to carry additional information like nullability.
-// Should only be instantiated and used by ciTypeFlow and ciSignature.
+// This class wraps another type to carry additional information.
+// Currently it is only used to mark inline klasses as null-free.
 class ciWrapper : public ciType {
   CI_PACKAGE_ACCESS
 
 private:
   ciType* _type;
-  bool _null_free;
 
-  ciWrapper(ciType* type, bool null_free) : ciType(type->basic_type()) {
+  ciWrapper(ciType* type) : ciType(type->basic_type()) {
     assert(type->is_inlinetype()
           // An unloaded inline type is an instance_klass (see ciEnv::get_klass_by_name_impl())
           || (type->is_instance_klass() && !type->is_loaded()),
           "should only be used for inline types");
     _type = type;
-    _null_free = null_free;
   }
 
   const char* type_string() { return "ciWrapper"; }
@@ -135,10 +133,9 @@ class ciWrapper : public ciType {
   void print_impl(outputStream* st) { _type->print_impl(st); }
 
 public:
-  bool    is_wrapper() const { return true; }
-
-  ciType*     unwrap()       { return _type; }
-  bool is_null_free() const { return _null_free; }
+  bool is_wrapper()   const { return true; }
+  ciType* unwrap()          { return _type; }
+  bool is_null_free() const { return true; }
 };
 
 #endif // SHARE_CI_CITYPE_HPP
diff --git a/src/hotspot/share/ci/ciTypeFlow.cpp b/src/hotspot/share/ci/ciTypeFlow.cpp
index e0d22a5ffc5..06a9bd3eb0b 100644
--- a/src/hotspot/share/ci/ciTypeFlow.cpp
+++ b/src/hotspot/share/ci/ciTypeFlow.cpp
@@ -350,7 +350,7 @@ ciType* ciTypeFlow::StateVector::type_meet_internal(ciType* t1, ciType* t2, ciTy
     assert(k1->is_instance_klass(), "previous cases handle non-instances");
     assert(k2->is_instance_klass(), "previous cases handle non-instances");
     ciType* result = k1->least_common_ancestor(k2);
-    if (null_free1 && null_free2) {
+    if (null_free1 && null_free2 && result->is_inlinetype()) {
       result = analyzer->mark_as_null_free(result);
     }
     return result;
@@ -637,6 +637,10 @@ void ciTypeFlow::StateVector::do_checkcast(ciBytecodeStream* str) {
     }
   } else {
     ciType* type = pop_value();
+    if (type->unwrap() != klass && klass->is_loaded() && type->unwrap()->is_subtype_of(klass)) {
+      // Useless cast, propagate more precise type of object
+      klass = type->unwrap()->as_klass();
+    }
     if (klass->is_inlinetype() && (null_free || type->is_null_free())) {
       push(outer()->mark_as_null_free(klass));
     } else {
@@ -1579,7 +1583,7 @@ bool ciTypeFlow::StateVector::apply_one_bytecode(ciBytecodeStream* str) {
 // ------------------------------------------------------------------
 // ciTypeFlow::StateVector::print_cell_on
 void ciTypeFlow::StateVector::print_cell_on(outputStream* st, Cell c) const {
-  ciType* type = type_at(c);
+  ciType* type = type_at(c)->unwrap();
   if (type == top_type()) {
     st->print("top");
   } else if (type == bottom_type()) {
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index dc5564d13b9..54dc3256eaa 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -705,22 +705,30 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
           }
         } else {
           if (_need_verify) {
-            // Method name and signature are verified above, when iterating NameAndType_info.
-            // Need only to be sure signature is non-zero length and the right type.
+            // Method name and signature are individually verified above, when iterating
+            // NameAndType_info.  Need to check here that signature is non-zero length and
+            // the right type.
             if (!Signature::is_method(signature)) {
               throwIllegalSignature("Method", name, signature, CHECK);
             }
           }
-          // 4509014: If a class method name begins with '<', it must be "<init>"
+          // If a class method name begins with '<', it must be "<init>" and have void signature
+          // unless it's an inline type.
           const unsigned int name_len = name->utf8_length();
-          if (tag == JVM_CONSTANT_Methodref &&
-              name_len != 0 &&
-              name->char_at(0) == JVM_SIGNATURE_SPECIAL &&
-              name != vmSymbols::object_initializer_name()) {
-            classfile_parse_error(
-              "Bad method name at constant pool index %u in class file %s",
-              name_ref_index, THREAD);
-            return;
+          if (tag == JVM_CONSTANT_Methodref && name_len != 0 &&
+              name->char_at(0) == JVM_SIGNATURE_SPECIAL) {
+            if (name != vmSymbols::object_initializer_name()) {
+              classfile_parse_error(
+                "Bad method name at constant pool index %u in class file %s",
+                name_ref_index, THREAD);
+              return;
+            } else if (!Signature::is_void_method(signature)) {
+              // if return type is non-void then it cannot be a basic primitive
+              // and primitve types must be supported.
+              if (!signature->ends_with(JVM_SIGNATURE_ENDCLASS) || !EnableValhalla) {
+                throwIllegalSignature("Method", name, signature, CHECK);
+              }
+            }
           }
         }
         break;
@@ -2183,7 +2191,6 @@ void ClassFileParser::ClassAnnotationCollector::apply_to(InstanceKlass* ik) {
     ik->set_has_value_based_class_annotation();
     if (DiagnoseSyncOnValueBasedClasses) {
       ik->set_is_value_based();
-      ik->set_prototype_header(markWord::prototype());
     }
   }
 }
@@ -2442,6 +2449,7 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
 
   int args_size = -1;  // only used when _need_verify is true
   if (_need_verify) {
+    verify_legal_name_with_signature(name, signature, CHECK_NULL);
     args_size = ((flags & JVM_ACC_STATIC) ? 0 : 1) +
                  verify_legal_method_signature(name, signature, CHECK_NULL);
     if (args_size > MAX_ARGS_SIZE) {
@@ -4155,10 +4163,8 @@ const InstanceKlass* ClassFileParser::parse_super_class(ConstantPool* const cp,
   const InstanceKlass* super_klass = NULL;
 
   if (super_class_index == 0) {
-    check_property(_class_name == vmSymbols::java_lang_Object()
-                   || (_access_flags.get_flags() & JVM_ACC_INLINE),
-                   "Invalid superclass index %u in class file %s",
-                   super_class_index,
+    check_property(_class_name == vmSymbols::java_lang_Object(),
+                   "Invalid superclass index 0 in class file %s",
                    CHECK_NULL);
   } else {
     check_property(valid_klass_reference_at(super_class_index),
@@ -4352,7 +4358,7 @@ void ClassFileParser::set_precomputed_flags(InstanceKlass* ik) {
   if (vmClasses::Cloneable_klass_loaded()) {
     if (ik->is_subtype_of(vmClasses::Cloneable_klass())) {
       if (ik->is_inline_klass()) {
-        JavaThread *THREAD = Thread::current()->as_Java_thread();
+        JavaThread *THREAD = JavaThread::current();
         throwInlineTypeLimitation(THREAD_AND_LOCATION, "Inline Types do not support Cloneable");
         return;
       }
@@ -5177,7 +5183,8 @@ void ClassFileParser::verify_legal_class_name(const Symbol* name, TRAPS) const {
         p = skip_over_field_name(bytes, true, length);
         legal = (p != NULL) && ((p - bytes) == (int)length);
       }
-    } else if (_major_version >= CONSTANT_CLASS_DESCRIPTORS && bytes[length - 1] == ';' ) {
+    } else if ((_major_version >= CONSTANT_CLASS_DESCRIPTORS || _class_name->starts_with("jdk/internal/reflect/"))
+                   && bytes[length - 1] == ';' ) {
       // Support for L...; and Q...; descriptors
       legal = verify_unqualified_name(bytes + 1, length - 2, LegalClass);
     } else {
@@ -5290,6 +5297,34 @@ void ClassFileParser::verify_legal_field_signature(const Symbol* name,
   }
 }
 
+// Check that the signature is compatible with the method name.  For example,
+// check that <init> has a void signature.
+void ClassFileParser::verify_legal_name_with_signature(const Symbol* name,
+                                                       const Symbol* signature,
+                                                       TRAPS) const {
+  if (!_need_verify) {
+    return;
+  }
+
+  // Class initializers cannot have args for class format version >= 51.
+  if (name == vmSymbols::class_initializer_name() &&
+      signature != vmSymbols::void_method_signature() &&
+      _major_version >= JAVA_7_VERSION) {
+    throwIllegalSignature("Method", name, signature, THREAD);
+    return;
+  }
+
+  if (!is_inline_type()) {
+    int sig_length = signature->utf8_length();
+    if (name->utf8_length() > 0 &&
+      name->char_at(0) == JVM_SIGNATURE_SPECIAL &&
+      sig_length > 0 &&
+      signature->char_at(sig_length - 1) != JVM_SIGNATURE_VOID) {
+      throwIllegalSignature("Method", name, signature, THREAD);
+    }
+  }
+}
+
 // Checks if signature is a legal method signature.
 // Returns number of parameters
 int ClassFileParser::verify_legal_method_signature(const Symbol* name,
@@ -5301,14 +5336,6 @@ int ClassFileParser::verify_legal_method_signature(const Symbol* name,
     return -2;
   }
 
-  // Class initializers cannot have args for class format version >= 51.
-  if (name == vmSymbols::class_initializer_name() &&
-      signature != vmSymbols::void_method_signature() &&
-      _major_version >= JAVA_7_VERSION) {
-    throwIllegalSignature("Method", name, signature, CHECK_0);
-    return 0;
-  }
-
   unsigned int args_size = 0;
   const char* p = (const char*)signature->bytes();
   unsigned int length = signature->utf8_length();
@@ -5331,38 +5358,15 @@ int ClassFileParser::verify_legal_method_signature(const Symbol* name,
     // The first non-signature thing better be a ')'
     if ((length > 0) && (*p++ == JVM_SIGNATURE_ENDFUNC)) {
       length--;
-      if (name->utf8_length() > 0 && name->char_at(0) == JVM_SIGNATURE_SPECIAL) {
-        // All constructor methods must return void
-        if ((length == 1) && (p[0] == JVM_SIGNATURE_VOID)) {
-          return args_size;
-        }
-        // All static init methods must return the current class
-        if ((length >= 3) && (p[length-1] == JVM_SIGNATURE_ENDCLASS)
-            && name == vmSymbols::object_initializer_name()) {
-          nextp = skip_over_field_signature(p, true, length, CHECK_0);
-          if (nextp && ((int)length == (nextp - p))) {
-            // The actual class will be checked against current class
-            // when the method is defined (see parse_method).
-            // A reference to a static init with a bad return type
-            // will load and verify OK, but will fail to link.
-            return args_size;
-          }
-        }
-        // The distinction between static factory methods and
-        // constructors depends on the JVM_ACC_STATIC modifier.
-        // This distinction must be reflected in a void or non-void
-        // return. For declared methods, the check is in parse_method.
-      } else {
-        // Now we better just have a return value
-        nextp = skip_over_field_signature(p, true, length, CHECK_0);
-        if (nextp && ((int)length == (nextp - p))) {
-          return args_size;
-        }
+      // Now we better just have a return value
+      nextp = skip_over_field_signature(p, true, length, CHECK_0);
+      if (nextp && ((int)length == (nextp - p))) {
+        return args_size;
       }
     }
   }
   // Report error
-  throwIllegalSignature("Method", name, signature, CHECK_0);
+  throwIllegalSignature("Method", name, signature, THREAD);
   return 0;
 }
 
diff --git a/src/hotspot/share/classfile/classFileParser.hpp b/src/hotspot/share/classfile/classFileParser.hpp
index 61acb18c88e..d953e7a2ab0 100644
--- a/src/hotspot/share/classfile/classFileParser.hpp
+++ b/src/hotspot/share/classfile/classFileParser.hpp
@@ -490,6 +490,9 @@ class ClassFileParser {
   int  verify_legal_method_signature(const Symbol* methodname,
                                      const Symbol* signature,
                                      TRAPS) const;
+  void verify_legal_name_with_signature(const Symbol* name,
+                                        const Symbol* signature,
+                                        TRAPS) const;
 
   void verify_class_version(u2 major, u2 minor, Symbol* class_name, TRAPS);
 
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index ff0c2376679..e7ec9e393ca 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -563,6 +563,21 @@ void ClassLoaderData::unload() {
   // after erroneous classes are released.
   classes_do(InstanceKlass::unload_class);
 
+  // Method::clear_jmethod_ids only sets the jmethod_ids to NULL without
+  // releasing the memory for related JNIMethodBlocks and JNIMethodBlockNodes.
+  // This is done intentionally because native code (e.g. JVMTI agent) holding
+  // jmethod_ids may access them after the associated classes and class loader
+  // are unloaded. The Java Native Interface Specification says "method ID
+  // does not prevent the VM from unloading the class from which the ID has
+  // been derived. After the class is unloaded, the method or field ID becomes
+  // invalid". In real world usages, the native code may rely on jmethod_ids
+  // being NULL after class unloading. Hence, it is unsafe to free the memory
+  // from the VM side without knowing when native code is going to stop using
+  // them.
+  if (_jmethod_ids != NULL) {
+    Method::clear_jmethod_ids(this);
+  }
+
   // Clean up global class iterator for compiler
   ClassLoaderDataGraph::adjust_saved_class(this);
 }
@@ -707,20 +722,7 @@ ClassLoaderData::~ClassLoaderData() {
     _metaspace = NULL;
     delete m;
   }
-  // Method::clear_jmethod_ids only sets the jmethod_ids to NULL without
-  // releasing the memory for related JNIMethodBlocks and JNIMethodBlockNodes.
-  // This is done intentionally because native code (e.g. JVMTI agent) holding
-  // jmethod_ids may access them after the associated classes and class loader
-  // are unloaded. The Java Native Interface Specification says "method ID
-  // does not prevent the VM from unloading the class from which the ID has
-  // been derived. After the class is unloaded, the method or field ID becomes
-  // invalid". In real world usages, the native code may rely on jmethod_ids
-  // being NULL after class unloading. Hence, it is unsafe to free the memory
-  // from the VM side without knowing when native code is going to stop using
-  // them.
-  if (_jmethod_ids != NULL) {
-    Method::clear_jmethod_ids(this);
-  }
+
   // Delete lock
   delete _metaspace_lock;
 
@@ -817,6 +819,7 @@ void ClassLoaderData::init_handle_locked(OopHandle& dest, Handle h) {
   if (dest.resolve() != NULL) {
     return;
   } else {
+    record_modified_oops();
     dest = _handles.add(h());
   }
 }
diff --git a/src/hotspot/share/classfile/classLoaderData.hpp b/src/hotspot/share/classfile/classLoaderData.hpp
index 5ead297a9c0..484cf19af23 100644
--- a/src/hotspot/share/classfile/classLoaderData.hpp
+++ b/src/hotspot/share/classfile/classLoaderData.hpp
@@ -324,7 +324,7 @@ class ClassLoaderData : public CHeapObj<mtClass> {
   Symbol* name_and_id() const { return _name_and_id; }
 
   unsigned identity_hash() const {
-    return (unsigned)((uintptr_t)this >> 3);
+    return (unsigned)((uintptr_t)this >> LogBytesPerWord);
   }
 
   JFR_ONLY(DEFINE_TRACE_ID_METHODS;)
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index dc06c78ce28..39920b12727 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -643,7 +643,7 @@ void FieldLayoutBuilder::regular_field_sorting() {
         // Flattening decision to be taken here
         // This code assumes all verification already have been performed
         // (field's type has been loaded and it is an inline klass)
-        JavaThread* THREAD = Thread::current()->as_Java_thread();
+        JavaThread* THREAD = JavaThread::current();
         Klass* klass =
             SystemDictionary::resolve_inline_type_field_or_fail(&fs,
                                                                 Handle(THREAD, _class_loader_data->class_loader()),
@@ -744,7 +744,7 @@ void FieldLayoutBuilder::inline_class_field_sorting(TRAPS) {
         // Flattening decision to be taken here
         // This code assumes all verifications have already been performed
         // (field's type has been loaded and it is an inline klass)
-        JavaThread* THREAD = Thread::current()->as_Java_thread();
+        JavaThread* THREAD = JavaThread::current();
         Klass* klass =
             SystemDictionary::resolve_inline_type_field_or_fail(&fs,
                 Handle(THREAD, _class_loader_data->class_loader()),
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index 6b593ff4423..3a319649d8e 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -40,6 +40,7 @@
 #include "code/debugInfo.hpp"
 #include "code/dependencyContext.hpp"
 #include "code/pcDesc.hpp"
+#include "gc/shared/collectedHeap.inline.hpp"
 #include "interpreter/interpreter.hpp"
 #include "interpreter/linkResolver.hpp"
 #include "logging/log.hpp"
@@ -504,7 +505,7 @@ jchar* java_lang_String::as_unicode_string_or_null(oop java_string, int& length)
   return result;
 }
 
-unsigned int java_lang_String::hash_code(oop java_string) {
+inline unsigned int java_lang_String::hash_code_impl(oop java_string, bool update) {
   // The hash and hashIsZero fields are subject to a benign data race,
   // making it crucial to ensure that any observable result of the
   // calculation in this method stays correct under any possible read of
@@ -531,14 +532,25 @@ unsigned int java_lang_String::hash_code(oop java_string) {
     }
   }
 
-  if (hash != 0) {
-    java_string->int_field_put(_hash_offset, hash);
-  } else {
-    java_string->bool_field_put(_hashIsZero_offset, true);
+  if (update) {
+    if (hash != 0) {
+      java_string->int_field_put(_hash_offset, hash);
+    } else {
+      java_string->bool_field_put(_hashIsZero_offset, true);
+    }
   }
   return hash;
 }
 
+unsigned int java_lang_String::hash_code(oop java_string) {
+  return hash_code_impl(java_string, /*update=*/true);
+}
+
+unsigned int java_lang_String::hash_code_noupdate(oop java_string) {
+  return hash_code_impl(java_string, /*update=*/false);
+}
+
+
 char* java_lang_String::as_quoted_ascii(oop java_string) {
   typeArrayOop value  = java_lang_String::value(java_string);
   int          length = java_lang_String::length(java_string, value);
@@ -809,14 +821,11 @@ static void initialize_static_string_field(fieldDescriptor* fd, Handle mirror, T
 static void initialize_static_string_field_for_dump(fieldDescriptor* fd, Handle mirror) {
   DEBUG_ONLY(assert_valid_static_string_field(fd);)
   assert(DumpSharedSpaces, "must be");
-  if (HeapShared::is_archived_object(mirror())) {
-    // Archive the String field and update the pointer.
-    oop s = mirror()->obj_field(fd->offset());
-    oop archived_s = StringTable::create_archived_string(s);
-    mirror()->obj_field_put(fd->offset(), archived_s);
-  } else {
-    guarantee(false, "Unexpected");
-  }
+  assert(HeapShared::is_archived_object_during_dumptime(mirror()), "must be");
+  // Archive the String field and update the pointer.
+  oop s = mirror()->obj_field(fd->offset());
+  oop archived_s = StringTable::create_archived_string(s);
+  mirror()->obj_field_put(fd->offset(), archived_s);
 }
 #endif
 
@@ -898,7 +907,7 @@ void java_lang_Class::fixup_mirror(Klass* k, TRAPS) {
   }
 
   if (k->is_shared() && k->has_archived_mirror_index()) {
-    if (HeapShared::open_archive_heap_region_mapped()) {
+    if (HeapShared::open_regions_mapped()) {
       bool present = restore_archived_mirror(k, Handle(), Handle(), Handle(), CHECK);
       assert(present, "Missing archived mirror for %s", k->external_name());
       return;
@@ -1187,7 +1196,7 @@ void java_lang_Class::archive_basic_type_mirrors() {
     oop m = Universe::_mirrors[t].resolve();
     if (m != NULL) {
       // Update the field at _array_klass_offset to point to the relocated array klass.
-      oop archived_m = HeapShared::archive_heap_object(m);
+      oop archived_m = HeapShared::archive_object(m);
       assert(archived_m != NULL, "sanity");
       Klass *ak = (Klass*)(archived_m->metadata_field(_array_klass_offset));
       assert(ak != NULL || t == T_VOID, "should not be NULL");
@@ -1252,7 +1261,7 @@ oop java_lang_Class::archive_mirror(Klass* k) {
   }
 
   // Now start archiving the mirror object
-  oop archived_mirror = HeapShared::archive_heap_object(mirror);
+  oop archived_mirror = HeapShared::archive_object(mirror);
   if (archived_mirror == NULL) {
     return NULL;
   }
@@ -1379,7 +1388,7 @@ bool java_lang_Class::restore_archived_mirror(Klass *k,
 
   // mirror is archived, restore
   log_debug(cds, mirror)("Archived mirror is: " PTR_FORMAT, p2i(m));
-  assert(HeapShared::is_archived_object(m), "must be archived mirror object");
+  assert(Universe::heap()->is_archived_object(m), "must be archived mirror object");
   assert(as_Klass(m) == k, "must be");
   Handle mirror(THREAD, m);
 
diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp
index 0d9cc84eaa2..748dcb99e98 100644
--- a/src/hotspot/share/classfile/javaClasses.hpp
+++ b/src/hotspot/share/classfile/javaClasses.hpp
@@ -127,6 +127,8 @@ class java_lang_String : AllStatic {
   // returning true if the bit was already set.
   static bool test_and_set_flag(oop java_string, uint8_t flag_mask);
 
+  static inline unsigned int hash_code_impl(oop java_string, bool update);
+
  public:
 
   // Coders
@@ -222,6 +224,7 @@ class java_lang_String : AllStatic {
   }
 
   static unsigned int hash_code(oop java_string);
+  static unsigned int hash_code_noupdate(oop java_string);
 
   static bool equals(oop java_string, const jchar* chars, int len);
   static bool equals(oop str1, oop str2);
@@ -345,6 +348,8 @@ class java_lang_Class : AllStatic {
   static int klass_offset()                { CHECK_INIT(_klass_offset); }
   static int array_klass_offset()          { CHECK_INIT(_array_klass_offset); }
   static int component_mirror_offset()     { CHECK_INIT(_component_mirror_offset); }
+  static int primary_mirror_offset()       { CHECK_INIT(_primary_mirror_offset); }
+  static int secondary_mirror_offset()     { CHECK_INIT(_secondary_mirror_offset); }
   // Support for classRedefinedCount field
   static int classRedefinedCount(oop the_class_mirror);
   static void set_classRedefinedCount(oop the_class_mirror, int value);
diff --git a/src/hotspot/share/classfile/placeholders.cpp b/src/hotspot/share/classfile/placeholders.cpp
index be1a7890a93..f92ce57257b 100644
--- a/src/hotspot/share/classfile/placeholders.cpp
+++ b/src/hotspot/share/classfile/placeholders.cpp
@@ -45,21 +45,21 @@
 // class circularity checking.
 class SeenThread: public CHeapObj<mtInternal> {
 private:
-   Thread *_thread;
+   JavaThread* _thread;
    SeenThread* _stnext;
    SeenThread* _stprev;
 public:
-   SeenThread(Thread *thread) {
+   SeenThread(JavaThread* thread) {
        _thread = thread;
        _stnext = NULL;
        _stprev = NULL;
    }
-   Thread* thread()                const { return _thread;}
-   void set_thread(Thread *thread) { _thread = thread; }
+   JavaThread* thread()          const { return _thread;}
+   void set_thread(JavaThread* thread) { _thread = thread; }
 
-   SeenThread* next()              const { return _stnext;}
-   void set_next(SeenThread *seen) { _stnext = seen; }
-   void set_prev(SeenThread *seen) { _stprev = seen; }
+   SeenThread* next()        const { return _stnext;}
+   void set_next(SeenThread* seen) { _stnext = seen; }
+   void set_prev(SeenThread* seen) { _stprev = seen; }
 
   void print_action_queue(outputStream* st) {
     SeenThread* seen = this;
@@ -115,7 +115,7 @@ void PlaceholderEntry::set_threadQ(SeenThread* seenthread, PlaceholderTable::cla
 // bootstrap loader support:  links in a thread before load_instance_class
 // definers: use as queue of define requestors, including owner of
 // define token. Appends for debugging of requestor order
-void PlaceholderEntry::add_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) {
+void PlaceholderEntry::add_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action) {
   assert_lock_strong(SystemDictionary_lock);
   SeenThread* threadEntry = new SeenThread(thread);
   SeenThread* seen = actionToQueue(action);
@@ -136,7 +136,7 @@ void PlaceholderEntry::add_seen_thread(Thread* thread, PlaceholderTable::classlo
   return;
 }
 
-bool PlaceholderEntry::check_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) {
+bool PlaceholderEntry::check_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action) {
   assert_lock_strong(SystemDictionary_lock);
   SeenThread* threadQ = actionToQueue(action);
   SeenThread* seen = threadQ;
@@ -154,7 +154,7 @@ bool PlaceholderEntry::check_seen_thread(Thread* thread, PlaceholderTable::class
 // SystemDictionary_lock
 // ignores if cleanup has already been done
 // if found, deletes SeenThread
-bool PlaceholderEntry::remove_seen_thread(Thread* thread, PlaceholderTable::classloadAction action) {
+bool PlaceholderEntry::remove_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action) {
   assert_lock_strong(SystemDictionary_lock);
   SeenThread* threadQ = actionToQueue(action);
   SeenThread* seen = threadQ;
@@ -298,7 +298,7 @@ PlaceholderEntry* PlaceholderTable::find_and_add(unsigned int hash,
                                                  ClassLoaderData* loader_data,
                                                  classloadAction action,
                                                  Symbol* supername,
-                                                 Thread* thread) {
+                                                 JavaThread* thread) {
   assert(action != LOAD_SUPER || supername != NULL, "must have a super class name");
   PlaceholderEntry* probe = get_entry(hash, name, loader_data);
   if (probe == NULL) {
@@ -331,7 +331,7 @@ PlaceholderEntry* PlaceholderTable::find_and_add(unsigned int hash,
 void PlaceholderTable::find_and_remove(unsigned int hash,
                                        Symbol* name, ClassLoaderData* loader_data,
                                        classloadAction action,
-                                       Thread* thread) {
+                                       JavaThread* thread) {
     assert_locked_or_safepoint(SystemDictionary_lock);
     PlaceholderEntry *probe = get_entry(hash, name, loader_data);
     if (probe != NULL) {
diff --git a/src/hotspot/share/classfile/placeholders.hpp b/src/hotspot/share/classfile/placeholders.hpp
index fdf0a51857a..e58788781fd 100644
--- a/src/hotspot/share/classfile/placeholders.hpp
+++ b/src/hotspot/share/classfile/placeholders.hpp
@@ -87,7 +87,7 @@ class PlaceholderTable : public Hashtable<Symbol*, mtClass> {
   PlaceholderEntry* find_and_add(unsigned int hash,
                                  Symbol* name, ClassLoaderData* loader_data,
                                  classloadAction action, Symbol* supername,
-                                 Thread* thread);
+                                 JavaThread* thread);
 
   void remove_entry(unsigned int hash,
                     Symbol* name, ClassLoaderData* loader_data);
@@ -96,7 +96,7 @@ class PlaceholderTable : public Hashtable<Symbol*, mtClass> {
   // If all queues are empty and definer is null, remove the PlacheholderEntry completely
   void find_and_remove(unsigned int hash,
                        Symbol* name, ClassLoaderData* loader_data,
-                       classloadAction action, Thread* thread);
+                       classloadAction action, JavaThread* thread);
 
   void print_on(outputStream* st) const;
   void print() const;
@@ -118,7 +118,7 @@ class PlaceholderEntry : public HashtableEntry<Symbol*, mtClass> {
  private:
   ClassLoaderData*  _loader_data;   // initiating loader
   Symbol*           _supername;
-  Thread*           _definer;       // owner of define token
+  JavaThread*       _definer;       // owner of define token
   InstanceKlass*    _instanceKlass; // InstanceKlass from successful define
   SeenThread*       _superThreadQ;  // doubly-linked queue of Threads loading a superclass for this class
   SeenThread*       _loadInstanceThreadQ;  // loadInstance thread
@@ -133,8 +133,8 @@ class PlaceholderEntry : public HashtableEntry<Symbol*, mtClass> {
 
   SeenThread* actionToQueue(PlaceholderTable::classloadAction action);
   void set_threadQ(SeenThread* seenthread, PlaceholderTable::classloadAction action);
-  void add_seen_thread(Thread* thread, PlaceholderTable::classloadAction action);
-  bool remove_seen_thread(Thread* thread, PlaceholderTable::classloadAction action);
+  void add_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action);
+  bool remove_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action);
 
  public:
   // Simple accessors, used only by SystemDictionary
@@ -149,8 +149,8 @@ class PlaceholderEntry : public HashtableEntry<Symbol*, mtClass> {
     if (_supername != NULL) _supername->increment_refcount();
   }
 
-  Thread*            definer()             const {return _definer; }
-  void               set_definer(Thread* definer) { _definer = definer; }
+  JavaThread*        definer()             const {return _definer; }
+  void               set_definer(JavaThread* definer) { _definer = definer; }
 
   InstanceKlass*     instance_klass()      const {return _instanceKlass; }
   void               set_instance_klass(InstanceKlass* ik) { _instanceKlass = ik; }
@@ -161,7 +161,7 @@ class PlaceholderEntry : public HashtableEntry<Symbol*, mtClass> {
   SeenThread*        loadInstanceThreadQ() const { return _loadInstanceThreadQ; }
   void               set_loadInstanceThreadQ(SeenThread* SeenThread) { _loadInstanceThreadQ = SeenThread; }
 
-  SeenThread*        defineThreadQ()        const { return _defineThreadQ; }
+  SeenThread*        defineThreadQ()       const { return _defineThreadQ; }
   void               set_defineThreadQ(SeenThread* SeenThread) { _defineThreadQ = SeenThread; }
 
   SeenThread*        inlineTypeFieldQ()    const { return _inlineTypeFieldQ; }
@@ -198,7 +198,7 @@ class PlaceholderEntry : public HashtableEntry<Symbol*, mtClass> {
   }
 
   // Used for ClassCircularityError checking
-  bool check_seen_thread(Thread* thread, PlaceholderTable::classloadAction action);
+  bool check_seen_thread(JavaThread* thread, PlaceholderTable::classloadAction action);
 
   // Print method doesn't append a cr
   void print_entry(outputStream* st) const;
diff --git a/src/hotspot/share/classfile/stackMapTableFormat.hpp b/src/hotspot/share/classfile/stackMapTableFormat.hpp
index 59fa03cdeb4..eac9c8fd19c 100644
--- a/src/hotspot/share/classfile/stackMapTableFormat.hpp
+++ b/src/hotspot/share/classfile/stackMapTableFormat.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,11 +41,11 @@ class verification_type_info {
   address tag_addr() const { return (address)this; }
   address cpool_index_addr() const { return tag_addr() + sizeof(u1); }
   address bci_addr() const { return cpool_index_addr(); }
+  NONCOPYABLE(verification_type_info);
 
  protected:
   // No constructors  - should be 'private', but GCC issues a warning if it is
   verification_type_info() {}
-  verification_type_info(const verification_type_info&) {}
 
  public:
 
@@ -169,12 +169,13 @@ FOR_EACH_STACKMAP_FRAME_TYPE(SM_FORWARD_DECL, x, x)
 #undef SM_FORWARD_DECL
 
 class stack_map_frame {
+  NONCOPYABLE(stack_map_frame);
+
  protected:
   address frame_type_addr() const { return (address)this; }
 
   // No constructors  - should be 'private', but GCC issues a warning if it is
   stack_map_frame() {}
-  stack_map_frame(const stack_map_frame&) {}
 
  public:
 
@@ -901,11 +902,11 @@ class stack_map_table {
   address entries_addr() const {
     return number_of_entries_addr() + sizeof(u2);
   }
+  NONCOPYABLE(stack_map_table);
 
  protected:
   // No constructors  - should be 'private', but GCC issues a warning if it is
   stack_map_table() {}
-  stack_map_table(const stack_map_table&) {}
 
  public:
 
@@ -933,11 +934,11 @@ class stack_map_table_attribute {
       return name_index_addr() + sizeof(u2); }
   address stack_map_table_addr() const {
       return attribute_length_addr() + sizeof(u4); }
+  NONCOPYABLE(stack_map_table_attribute);
 
  protected:
   // No constructors  - should be 'private', but GCC issues a warning if it is
   stack_map_table_attribute() {}
-  stack_map_table_attribute(const stack_map_table_attribute&) {}
 
  public:
 
diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp
index ab74df425a7..695e0d12cca 100644
--- a/src/hotspot/share/classfile/stringTable.cpp
+++ b/src/hotspot/share/classfile/stringTable.cpp
@@ -53,6 +53,7 @@
 #include "utilities/concurrentHashTable.inline.hpp"
 #include "utilities/concurrentHashTableTasks.inline.hpp"
 #include "utilities/macros.hpp"
+#include "utilities/resizeableResourceHash.hpp"
 #include "utilities/utf8.hpp"
 
 // We prefer short chains of avg 2
@@ -597,39 +598,40 @@ void StringTable::verify() {
 
 // Verification and comp
 class VerifyCompStrings : StackObj {
-  GrowableArray<oop>* _oops;
+  static unsigned string_hash(oop const& str) {
+    return java_lang_String::hash_code_noupdate(str);
+  }
+  static bool string_equals(oop const& a, oop const& b) {
+    return java_lang_String::equals(a, b);
+  }
+
+  ResizeableResourceHashtable<oop, bool,
+                              ResourceObj::C_HEAP, mtInternal,
+                              string_hash, string_equals> _table;
  public:
   size_t _errors;
-  VerifyCompStrings(GrowableArray<oop>* oops) : _oops(oops), _errors(0) {}
+  VerifyCompStrings() : _table(unsigned(_items_count / 8) + 1), _errors(0) {}
   bool operator()(WeakHandle* val) {
     oop s = val->resolve();
     if (s == NULL) {
       return true;
     }
-    int len = _oops->length();
-    for (int i = 0; i < len; i++) {
-      bool eq = java_lang_String::equals(s, _oops->at(i));
-      assert(!eq, "Duplicate strings");
-      if (eq) {
-        _errors++;
-      }
+    bool created;
+    _table.put_if_absent(s, true, &created);
+    assert(created, "Duplicate strings");
+    if (!created) {
+      _errors++;
     }
-    _oops->push(s);
     return true;
   };
 };
 
 size_t StringTable::verify_and_compare_entries() {
   Thread* thr = Thread::current();
-  GrowableArray<oop>* oops =
-    new (ResourceObj::C_HEAP, mtInternal)
-      GrowableArray<oop>((int)_current_size, mtInternal);
-
-  VerifyCompStrings vcs(oops);
+  VerifyCompStrings vcs;
   if (!_local_table->try_scan(thr, vcs)) {
     log_info(stringtable)("verify unavailable at this moment");
   }
-  delete oops;
   return vcs._errors;
 }
 
@@ -718,15 +720,15 @@ oop StringTable::lookup_shared(const jchar* name, int len) {
 oop StringTable::create_archived_string(oop s) {
   assert(DumpSharedSpaces, "this function is only used with -Xshare:dump");
   assert(java_lang_String::is_instance(s), "sanity");
-  assert(!HeapShared::is_archived_object(s), "sanity");
+  assert(!HeapShared::is_archived_object_during_dumptime(s), "sanity");
 
   oop new_s = NULL;
   typeArrayOop v = java_lang_String::value_no_keepalive(s);
-  typeArrayOop new_v = (typeArrayOop)HeapShared::archive_heap_object(v);
+  typeArrayOop new_v = (typeArrayOop)HeapShared::archive_object(v);
   if (new_v == NULL) {
     return NULL;
   }
-  new_s = HeapShared::archive_heap_object(s);
+  new_s = HeapShared::archive_object(s);
   if (new_s == NULL) {
     return NULL;
   }
@@ -777,7 +779,7 @@ void StringTable::serialize_shared_table_header(SerializeClosure* soc) {
   if (soc->writing()) {
     // Sanity. Make sure we don't use the shared table at dump time
     _shared_table.reset();
-  } else if (!HeapShared::closed_archive_heap_region_mapped()) {
+  } else if (!HeapShared::closed_regions_mapped()) {
     _shared_table.reset();
   }
 }
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 150000f2fac..abe7d5104a7 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -70,7 +70,6 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/biasedLocking.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
 #include "runtime/javaCalls.hpp"
@@ -158,7 +157,6 @@ ClassLoaderData* SystemDictionary::register_loader(Handle class_loader, bool cre
 
 bool is_parallelCapable(Handle class_loader) {
   if (class_loader.is_null()) return true;
-  if (AlwaysLockClassLoader) return false;
   return java_lang_ClassLoader::parallelCapable(class_loader());
 }
 // ----------------------------------------------------------------------------
diff --git a/src/hotspot/share/classfile/systemDictionary.hpp b/src/hotspot/share/classfile/systemDictionary.hpp
index 9f6728072ee..b5393b052f4 100644
--- a/src/hotspot/share/classfile/systemDictionary.hpp
+++ b/src/hotspot/share/classfile/systemDictionary.hpp
@@ -209,15 +209,14 @@ class SystemDictionary : AllStatic {
   // Initialization
   static void initialize(TRAPS);
 
-protected:
-  // Returns the class loader data to be used when looking up/updating the
-  // system dictionary.
-  static ClassLoaderData *class_loader_data(Handle class_loader);
-
 public:
   // Returns java system loader
   static oop java_system_loader();
 
+  // Returns the class loader data to be used when looking up/updating the
+  // system dictionary.
+  static ClassLoaderData *class_loader_data(Handle class_loader);
+
   // Returns java platform loader
   static oop java_platform_loader();
 
@@ -227,7 +226,6 @@ class SystemDictionary : AllStatic {
   // Register a new class loader
   static ClassLoaderData* register_loader(Handle class_loader, bool create_mirror_cld = false);
 
-public:
   static Symbol* check_signature_loaders(Symbol* signature, Klass* klass_being_linked,
                                          Handle loader1, Handle loader2, bool is_method);
 
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index 811feb30f54..38d14136be8 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -29,7 +29,10 @@
 #include "cds/dynamicArchive.hpp"
 #include "cds/filemap.hpp"
 #include "cds/heapShared.hpp"
+#include "cds/cdsProtectionDomain.hpp"
+#include "cds/dumpTimeClassInfo.hpp"
 #include "cds/metaspaceShared.hpp"
+#include "cds/runTimeClassInfo.hpp"
 #include "classfile/classFileStream.hpp"
 #include "classfile/classLoader.hpp"
 #include "classfile/classLoaderData.inline.hpp"
@@ -60,7 +63,6 @@
 #include "oops/oop.inline.hpp"
 #include "oops/oopHandle.inline.hpp"
 #include "oops/typeArrayOop.inline.hpp"
-#include "prims/jvmtiExport.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
@@ -70,870 +72,264 @@
 #include "utilities/resourceHash.hpp"
 #include "utilities/stringUtils.hpp"
 
+DumpTimeSharedClassTable* SystemDictionaryShared::_dumptime_table = NULL;
+DumpTimeSharedClassTable* SystemDictionaryShared::_cloned_dumptime_table = NULL;
+DumpTimeLambdaProxyClassDictionary* SystemDictionaryShared::_dumptime_lambda_proxy_class_dictionary = NULL;
+DumpTimeLambdaProxyClassDictionary* SystemDictionaryShared::_cloned_dumptime_lambda_proxy_class_dictionary = NULL;
+// SystemDictionaries in the base layer static archive
+RunTimeSharedDictionary SystemDictionaryShared::_builtin_dictionary;
+RunTimeSharedDictionary SystemDictionaryShared::_unregistered_dictionary;
+// SystemDictionaries in the top layer dynamic archive
+RunTimeSharedDictionary SystemDictionaryShared::_dynamic_builtin_dictionary;
+RunTimeSharedDictionary SystemDictionaryShared::_dynamic_unregistered_dictionary;
+
+LambdaProxyClassDictionary SystemDictionaryShared::_lambda_proxy_class_dictionary;
+LambdaProxyClassDictionary SystemDictionaryShared::_dynamic_lambda_proxy_class_dictionary;
 
-OopHandle SystemDictionaryShared::_shared_protection_domains;
-OopHandle SystemDictionaryShared::_shared_jar_urls;
-OopHandle SystemDictionaryShared::_shared_jar_manifests;
 DEBUG_ONLY(bool SystemDictionaryShared::_no_class_loading_should_happen = false;)
 bool SystemDictionaryShared::_dump_in_progress = false;
 
-class DumpTimeSharedClassInfo: public CHeapObj<mtClass> {
-  bool                         _excluded;
-  bool                         _is_early_klass;
-  bool                         _has_checked_exclusion;
-public:
-  struct DTLoaderConstraint {
-    Symbol* _name;
-    char _loader_type1;
-    char _loader_type2;
-    DTLoaderConstraint(Symbol* name, char l1, char l2) : _name(name), _loader_type1(l1), _loader_type2(l2) {
-      _name->increment_refcount();
-    }
-    DTLoaderConstraint() : _name(NULL), _loader_type1('0'), _loader_type2('0') {}
-    bool equals(const DTLoaderConstraint& t) {
-      return t._name == _name &&
-             ((t._loader_type1 == _loader_type1 && t._loader_type2 == _loader_type2) ||
-              (t._loader_type2 == _loader_type1 && t._loader_type1 == _loader_type2));
-    }
-  };
-
-  struct DTVerifierConstraint {
-    Symbol* _name;
-    Symbol* _from_name;
-    DTVerifierConstraint() : _name(NULL), _from_name(NULL) {}
-    DTVerifierConstraint(Symbol* n, Symbol* fn) : _name(n), _from_name(fn) {
-      _name->increment_refcount();
-      _from_name->increment_refcount();
-    }
-  };
-
-  InstanceKlass*               _klass;
-  InstanceKlass*               _nest_host;
-  bool                         _failed_verification;
-  bool                         _is_archived_lambda_proxy;
-  int                          _id;
-  int                          _clsfile_size;
-  int                          _clsfile_crc32;
-  GrowableArray<DTVerifierConstraint>* _verifier_constraints;
-  GrowableArray<char>*                 _verifier_constraint_flags;
-  GrowableArray<DTLoaderConstraint>* _loader_constraints;
-
-  DumpTimeSharedClassInfo() {
-    _klass = NULL;
-    _nest_host = NULL;
-    _failed_verification = false;
-    _is_archived_lambda_proxy = false;
-    _has_checked_exclusion = false;
-    _id = -1;
-    _clsfile_size = -1;
-    _clsfile_crc32 = -1;
-    _excluded = false;
-    _is_early_klass = JvmtiExport::is_early_phase();
-    _verifier_constraints = NULL;
-    _verifier_constraint_flags = NULL;
-    _loader_constraints = NULL;
-  }
-
-  void add_verification_constraint(InstanceKlass* k, Symbol* name,
-         Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object);
-  void record_linking_constraint(Symbol* name, Handle loader1, Handle loader2);
-
-  bool is_builtin() {
-    return SystemDictionaryShared::is_builtin(_klass);
-  }
-
-  int num_verifier_constraints() {
-    if (_verifier_constraint_flags != NULL) {
-      return _verifier_constraint_flags->length();
-    } else {
-      return 0;
-    }
-  }
-
-  int num_loader_constraints() {
-    if (_loader_constraints != NULL) {
-      return _loader_constraints->length();
-    } else {
-      return 0;
-    }
-  }
+InstanceKlass* SystemDictionaryShared::load_shared_class_for_builtin_loader(
+                 Symbol* class_name, Handle class_loader, TRAPS) {
+  assert(UseSharedSpaces, "must be");
+  InstanceKlass* ik = find_builtin_class(class_name);
 
-  void metaspace_pointers_do(MetaspaceClosure* it) {
-    it->push(&_klass);
-    it->push(&_nest_host);
-    if (_verifier_constraints != NULL) {
-      for (int i = 0; i < _verifier_constraints->length(); i++) {
-        DTVerifierConstraint* cons = _verifier_constraints->adr_at(i);
-        it->push(&cons->_name);
-        it->push(&cons->_from_name);
-      }
-    }
-    if (_loader_constraints != NULL) {
-      for (int i = 0; i < _loader_constraints->length(); i++) {
-        DTLoaderConstraint* lc = _loader_constraints->adr_at(i);
-        it->push(&lc->_name);
-      }
+  if (ik != NULL && !ik->shared_loading_failed()) {
+    if ((SystemDictionary::is_system_class_loader(class_loader()) && ik->is_shared_app_class())  ||
+        (SystemDictionary::is_platform_class_loader(class_loader()) && ik->is_shared_platform_class())) {
+      SharedClassLoadingMark slm(THREAD, ik);
+      PackageEntry* pkg_entry = CDSProtectionDomain::get_package_entry_from_class(ik, class_loader);
+      Handle protection_domain =
+        CDSProtectionDomain::init_security_info(class_loader, ik, pkg_entry, CHECK_NULL);
+      return load_shared_class(ik, class_loader, protection_domain, NULL, pkg_entry, THREAD);
     }
   }
-
-  bool is_excluded() {
-    // _klass may become NULL due to DynamicArchiveBuilder::set_to_null
-    return _excluded || _failed_verification || _klass == NULL;
-  }
-
-  // Was this class loaded while JvmtiExport::is_early_phase()==true
-  bool is_early_klass() {
-    return _is_early_klass;
-  }
-
-  // simple accessors
-  void set_excluded()                               { _excluded = true; }
-  bool has_checked_exclusion() const                { return _has_checked_exclusion; }
-  void set_has_checked_exclusion()                  { _has_checked_exclusion = true; }
-  bool failed_verification() const                  { return _failed_verification; }
-  void set_failed_verification()                    { _failed_verification = true; }
-  InstanceKlass* nest_host() const                  { return _nest_host; }
-  void set_nest_host(InstanceKlass* nest_host)      { _nest_host = nest_host; }
-};
-
-inline unsigned DumpTimeSharedClassTable_hash(InstanceKlass* const& k) {
-  if (DumpSharedSpaces) {
-    // Deterministic archive contents
-    uintx delta = k->name() - MetaspaceShared::symbol_rs_base();
-    return primitive_hash<uintx>(delta);
-  } else {
-    // Deterministic archive is not possible because classes can be loaded
-    // in multiple threads.
-    return primitive_hash<InstanceKlass*>(k);
-  }
+  return NULL;
 }
 
-class DumpTimeSharedClassTable: public ResourceHashtable<
-  InstanceKlass*,
-  DumpTimeSharedClassInfo,
-  &DumpTimeSharedClassTable_hash,
-  primitive_equals<InstanceKlass*>,
-  15889, // prime number
-  ResourceObj::C_HEAP>
-{
-  int _builtin_count;
-  int _unregistered_count;
-public:
-  DumpTimeSharedClassInfo* find_or_allocate_info_for(InstanceKlass* k, bool dump_in_progress) {
-    bool created = false;
-    DumpTimeSharedClassInfo* p;
-    if (!dump_in_progress) {
-      p = put_if_absent(k, &created);
-    } else {
-      p = get(k);
-    }
-    if (created) {
-      assert(!SystemDictionaryShared::no_class_loading_should_happen(),
-             "no new classes can be loaded while dumping archive");
-      p->_klass = k;
-    } else {
-      if (!dump_in_progress) {
-        assert(p->_klass == k, "Sanity");
-      }
-    }
-    return p;
-  }
-
-  class CountClassByCategory : StackObj {
-    DumpTimeSharedClassTable* _table;
-  public:
-    CountClassByCategory(DumpTimeSharedClassTable* table) : _table(table) {}
-    bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
-      if (!info.is_excluded()) {
-        if (info.is_builtin()) {
-          ++ _table->_builtin_count;
-        } else {
-          ++ _table->_unregistered_count;
-        }
-      }
-      return true; // keep on iterating
-    }
-  };
-
-  void update_counts() {
-    _builtin_count = 0;
-    _unregistered_count = 0;
-    CountClassByCategory counter(this);
-    iterate(&counter);
+// This function is called for loading only UNREGISTERED classes
+InstanceKlass* SystemDictionaryShared::lookup_from_stream(Symbol* class_name,
+                                                          Handle class_loader,
+                                                          Handle protection_domain,
+                                                          const ClassFileStream* cfs,
+                                                          TRAPS) {
+  if (!UseSharedSpaces) {
+    return NULL;
   }
-
-  int count_of(bool is_builtin) const {
-    if (is_builtin) {
-      return _builtin_count;
-    } else {
-      return _unregistered_count;
-    }
+  if (class_name == NULL) {  // don't do this for hidden classes
+    return NULL;
   }
-};
-
-class LambdaProxyClassKey {
-  InstanceKlass* _caller_ik;
-  Symbol*        _invoked_name;
-  Symbol*        _invoked_type;
-  Symbol*        _method_type;
-  Method*        _member_method;
-  Symbol*        _instantiated_method_type;
-
-public:
-  LambdaProxyClassKey(InstanceKlass* caller_ik,
-                      Symbol*        invoked_name,
-                      Symbol*        invoked_type,
-                      Symbol*        method_type,
-                      Method*        member_method,
-                      Symbol*        instantiated_method_type) :
-    _caller_ik(caller_ik),
-    _invoked_name(invoked_name),
-    _invoked_type(invoked_type),
-    _method_type(method_type),
-    _member_method(member_method),
-    _instantiated_method_type(instantiated_method_type) {}
-
-  void metaspace_pointers_do(MetaspaceClosure* it) {
-    it->push(&_caller_ik);
-    it->push(&_invoked_name);
-    it->push(&_invoked_type);
-    it->push(&_method_type);
-    it->push(&_member_method);
-    it->push(&_instantiated_method_type);
-  }
-
-  void mark_pointers() {
-    ArchivePtrMarker::mark_pointer(&_caller_ik);
-    ArchivePtrMarker::mark_pointer(&_instantiated_method_type);
-    ArchivePtrMarker::mark_pointer(&_invoked_name);
-    ArchivePtrMarker::mark_pointer(&_invoked_type);
-    ArchivePtrMarker::mark_pointer(&_member_method);
-    ArchivePtrMarker::mark_pointer(&_method_type);
-  }
-
-  bool equals(LambdaProxyClassKey const& other) const {
-    return _caller_ik == other._caller_ik &&
-           _invoked_name == other._invoked_name &&
-           _invoked_type == other._invoked_type &&
-           _method_type == other._method_type &&
-           _member_method == other._member_method &&
-           _instantiated_method_type == other._instantiated_method_type;
-  }
-
-  unsigned int hash() const {
-    return SystemDictionaryShared::hash_for_shared_dictionary((address)_caller_ik) +
-           SystemDictionaryShared::hash_for_shared_dictionary((address)_invoked_name) +
-           SystemDictionaryShared::hash_for_shared_dictionary((address)_invoked_type) +
-           SystemDictionaryShared::hash_for_shared_dictionary((address)_method_type) +
-           SystemDictionaryShared::hash_for_shared_dictionary((address)_instantiated_method_type);
-  }
-
-  static unsigned int dumptime_hash(Symbol* sym)  {
-    if (sym == NULL) {
-      // _invoked_name maybe NULL
-      return 0;
-    }
-    return java_lang_String::hash_code((const jbyte*)sym->bytes(), sym->utf8_length());
+  if (class_loader.is_null() ||
+      SystemDictionary::is_system_class_loader(class_loader()) ||
+      SystemDictionary::is_platform_class_loader(class_loader())) {
+    // Do nothing for the BUILTIN loaders.
+    return NULL;
   }
 
-  unsigned int dumptime_hash() const {
-    return dumptime_hash(_caller_ik->name()) +
-           dumptime_hash(_invoked_name) +
-           dumptime_hash(_invoked_type) +
-           dumptime_hash(_method_type) +
-           dumptime_hash(_instantiated_method_type);
+  const RunTimeClassInfo* record = find_record(&_unregistered_dictionary, &_dynamic_unregistered_dictionary, class_name);
+  if (record == NULL) {
+    return NULL;
   }
 
-  static inline unsigned int DUMPTIME_HASH(LambdaProxyClassKey const& key) {
-    return (key.dumptime_hash());
-  }
+  int clsfile_size  = cfs->length();
+  int clsfile_crc32 = ClassLoader::crc32(0, (const char*)cfs->buffer(), cfs->length());
 
-  static inline bool DUMPTIME_EQUALS(
-      LambdaProxyClassKey const& k1, LambdaProxyClassKey const& k2) {
-    return (k1.equals(k2));
+  if (!record->matches(clsfile_size, clsfile_crc32)) {
+    return NULL;
   }
 
-  InstanceKlass* caller_ik() const { return _caller_ik; }
-};
+  return acquire_class_for_current_thread(record->_klass, class_loader,
+                                          protection_domain, cfs,
+                                          THREAD);
+}
 
+InstanceKlass* SystemDictionaryShared::acquire_class_for_current_thread(
+                   InstanceKlass *ik,
+                   Handle class_loader,
+                   Handle protection_domain,
+                   const ClassFileStream *cfs,
+                   TRAPS) {
+  ClassLoaderData* loader_data = ClassLoaderData::class_loader_data(class_loader());
 
-class DumpTimeLambdaProxyClassInfo {
-public:
-  GrowableArray<InstanceKlass*>* _proxy_klasses;
-  DumpTimeLambdaProxyClassInfo() : _proxy_klasses(NULL) {}
-  void add_proxy_klass(InstanceKlass* proxy_klass) {
-    if (_proxy_klasses == NULL) {
-      _proxy_klasses = new (ResourceObj::C_HEAP, mtClassShared)GrowableArray<InstanceKlass*>(5, mtClassShared);
+  {
+    MutexLocker mu(THREAD, SharedDictionary_lock);
+    if (ik->class_loader_data() != NULL) {
+      //    ik is already loaded (by this loader or by a different loader)
+      // or ik is being loaded by a different thread (by this loader or by a different loader)
+      return NULL;
     }
-    assert(_proxy_klasses != NULL, "sanity");
-    _proxy_klasses->append(proxy_klass);
-  }
 
-  void metaspace_pointers_do(MetaspaceClosure* it) {
-    for (int i=0; i<_proxy_klasses->length(); i++) {
-      it->push(_proxy_klasses->adr_at(i));
-    }
+    // No other thread has acquired this yet, so give it to *this thread*
+    ik->set_class_loader_data(loader_data);
   }
-};
 
-class RunTimeLambdaProxyClassInfo {
-  LambdaProxyClassKey _key;
-  InstanceKlass* _proxy_klass_head;
-public:
-  RunTimeLambdaProxyClassInfo(LambdaProxyClassKey key, InstanceKlass* proxy_klass_head) :
-    _key(key), _proxy_klass_head(proxy_klass_head) {}
-
-  InstanceKlass* proxy_klass_head() const { return _proxy_klass_head; }
+  // No longer holding SharedDictionary_lock
+  // No need to lock, as <ik> can be held only by a single thread.
+  loader_data->add_class(ik);
 
-  // Used by LambdaProxyClassDictionary to implement OffsetCompactHashtable::EQUALS
-  static inline bool EQUALS(
-       const RunTimeLambdaProxyClassInfo* value, LambdaProxyClassKey* key, int len_unused) {
-    return (value->_key.equals(*key));
-  }
-  void init(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
-    _key = key;
-    _key.mark_pointers();
-    _proxy_klass_head = info._proxy_klasses->at(0);
-    ArchivePtrMarker::mark_pointer(&_proxy_klass_head);
-  }
+  // Get the package entry.
+  PackageEntry* pkg_entry = CDSProtectionDomain::get_package_entry_from_class(ik, class_loader);
 
-  unsigned int hash() const {
-    return _key.hash();
-  }
-  LambdaProxyClassKey key() const {
-    return _key;
+  // Load and check super/interfaces, restore unsharable info
+  InstanceKlass* shared_klass = load_shared_class(ik, class_loader, protection_domain,
+                                                  cfs, pkg_entry, THREAD);
+  if (shared_klass == NULL || HAS_PENDING_EXCEPTION) {
+    // TODO: clean up <ik> so it can be used again
+    return NULL;
   }
-};
-
-class LambdaProxyClassDictionary : public OffsetCompactHashtable<
-  LambdaProxyClassKey*,
-  const RunTimeLambdaProxyClassInfo*,
-  RunTimeLambdaProxyClassInfo::EQUALS> {};
 
-LambdaProxyClassDictionary _lambda_proxy_class_dictionary;
-
-LambdaProxyClassDictionary _dynamic_lambda_proxy_class_dictionary;
+  return shared_klass;
+}
 
-class DumpTimeLambdaProxyClassDictionary
-  : public ResourceHashtable<LambdaProxyClassKey,
-                             DumpTimeLambdaProxyClassInfo,
-                             LambdaProxyClassKey::DUMPTIME_HASH,
-                             LambdaProxyClassKey::DUMPTIME_EQUALS,
-                             137, // prime number
-                             ResourceObj::C_HEAP> {
-public:
-  int _count;
-};
+void SystemDictionaryShared::start_dumping() {
+  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+  _dump_in_progress = true;
+}
 
-DumpTimeLambdaProxyClassDictionary* _dumptime_lambda_proxy_class_dictionary = NULL;
+DumpTimeClassInfo* SystemDictionaryShared::find_or_allocate_info_for(InstanceKlass* k) {
+  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+  return find_or_allocate_info_for_locked(k);
+}
 
-static void add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey key,
-                                                           InstanceKlass* proxy_klass) {
+DumpTimeClassInfo* SystemDictionaryShared::find_or_allocate_info_for_locked(InstanceKlass* k) {
   assert_lock_strong(DumpTimeTable_lock);
-  if (_dumptime_lambda_proxy_class_dictionary == NULL) {
-    _dumptime_lambda_proxy_class_dictionary =
-      new (ResourceObj::C_HEAP, mtClass)DumpTimeLambdaProxyClassDictionary();
-  }
-  DumpTimeLambdaProxyClassInfo* lambda_info = _dumptime_lambda_proxy_class_dictionary->get(key);
-  if (lambda_info == NULL) {
-    DumpTimeLambdaProxyClassInfo info;
-    info.add_proxy_klass(proxy_klass);
-    _dumptime_lambda_proxy_class_dictionary->put(key, info);
-    //lambda_info = _dumptime_lambda_proxy_class_dictionary->get(key);
-    //assert(lambda_info->_proxy_klass == proxy_klass, "must be"); // debug only -- remove
-    ++_dumptime_lambda_proxy_class_dictionary->_count;
-  } else {
-    lambda_info->add_proxy_klass(proxy_klass);
+  if (_dumptime_table == NULL) {
+    _dumptime_table = new (ResourceObj::C_HEAP, mtClass) DumpTimeSharedClassTable;
   }
+  return _dumptime_table->find_or_allocate_info_for(k, _dump_in_progress);
 }
 
-class RunTimeSharedClassInfo {
-public:
-  struct CrcInfo {
-    int _clsfile_size;
-    int _clsfile_crc32;
-  };
-
-  // This is different than  DumpTimeSharedClassInfo::DTVerifierConstraint. We use
-  // u4 instead of Symbol* to save space on 64-bit CPU.
-  struct RTVerifierConstraint {
-    u4 _name;
-    u4 _from_name;
-    Symbol* name() { return (Symbol*)(SharedBaseAddress + _name);}
-    Symbol* from_name() { return (Symbol*)(SharedBaseAddress + _from_name); }
-  };
-
-  struct RTLoaderConstraint {
-    u4   _name;
-    char _loader_type1;
-    char _loader_type2;
-    Symbol* constraint_name() {
-      return (Symbol*)(SharedBaseAddress + _name);
-    }
-  };
-
-  InstanceKlass* _klass;
-  int _num_verifier_constraints;
-  int _num_loader_constraints;
-
-  // optional CrcInfo              _crc;  (only for UNREGISTERED classes)
-  // optional InstanceKlass*       _nest_host
-  // optional RTLoaderConstraint   _loader_constraint_types[_num_loader_constraints]
-  // optional RTVerifierConstraint _verifier_constraints[_num_verifier_constraints]
-  // optional char                 _verifier_constraint_flags[_num_verifier_constraints]
-
-private:
-  static size_t header_size_size() {
-    return sizeof(RunTimeSharedClassInfo);
-  }
-  static size_t crc_size(InstanceKlass* klass) {
-    if (!SystemDictionaryShared::is_builtin(klass)) {
-      return sizeof(CrcInfo);
-    } else {
-      return 0;
-    }
-  }
-  static size_t verifier_constraints_size(int num_verifier_constraints) {
-    return sizeof(RTVerifierConstraint) * num_verifier_constraints;
-  }
-  static size_t verifier_constraint_flags_size(int num_verifier_constraints) {
-    return sizeof(char) * num_verifier_constraints;
+bool SystemDictionaryShared::check_for_exclusion(InstanceKlass* k, DumpTimeClassInfo* info) {
+  if (MetaspaceShared::is_in_shared_metaspace(k)) {
+    // We have reached a super type that's already in the base archive. Treat it
+    // as "not excluded".
+    assert(DynamicDumpSharedSpaces, "must be");
+    return false;
   }
-  static size_t loader_constraints_size(int num_loader_constraints) {
-    return sizeof(RTLoaderConstraint) * num_loader_constraints;
+
+  if (info == NULL) {
+    info = _dumptime_table->get(k);
+    assert(info != NULL, "supertypes of any classes in _dumptime_table must either be shared, or must also be in _dumptime_table");
   }
-  static size_t nest_host_size(InstanceKlass* klass) {
-    if (klass->is_hidden()) {
-      return sizeof(InstanceKlass*);
-    } else {
-      return 0;
+
+  if (!info->has_checked_exclusion()) {
+    if (check_for_exclusion_impl(k)) {
+      info->set_excluded();
     }
+    info->set_has_checked_exclusion();
   }
 
-public:
-  static size_t byte_size(InstanceKlass* klass, int num_verifier_constraints, int num_loader_constraints) {
-    return header_size_size() +
-           crc_size(klass) +
-           nest_host_size(klass) +
-           loader_constraints_size(num_loader_constraints) +
-           verifier_constraints_size(num_verifier_constraints) +
-           verifier_constraint_flags_size(num_verifier_constraints);
-  }
+  return info->is_excluded();
+}
 
-private:
-  size_t crc_offset() const {
-    return header_size_size();
-  }
+// Returns true so the caller can do:    return warn_excluded(".....");
+bool SystemDictionaryShared::warn_excluded(InstanceKlass* k, const char* reason) {
+  ResourceMark rm;
+  log_warning(cds)("Skipping %s: %s", k->name()->as_C_string(), reason);
+  return true;
+}
 
-  size_t nest_host_offset() const {
-      return crc_offset() + crc_size(_klass);
+bool SystemDictionaryShared::is_jfr_event_class(InstanceKlass *k) {
+  while (k) {
+    if (k->name()->equals("jdk/internal/event/Event")) {
+      return true;
+    }
+    k = k->java_super();
   }
+  return false;
+}
 
-  size_t loader_constraints_offset() const  {
-    return nest_host_offset() + nest_host_size(_klass);
-  }
-  size_t verifier_constraints_offset() const {
-    return loader_constraints_offset() + loader_constraints_size(_num_loader_constraints);
-  }
-  size_t verifier_constraint_flags_offset() const {
-    return verifier_constraints_offset() + verifier_constraints_size(_num_verifier_constraints);
-  }
+bool SystemDictionaryShared::is_registered_lambda_proxy_class(InstanceKlass* ik) {
+  DumpTimeClassInfo* info = _dumptime_table->get(ik);
+  return (info != NULL) ? info->_is_archived_lambda_proxy : false;
+}
 
-  void check_verifier_constraint_offset(int i) const {
-    assert(0 <= i && i < _num_verifier_constraints, "sanity");
-  }
+bool SystemDictionaryShared::is_early_klass(InstanceKlass* ik) {
+  DumpTimeClassInfo* info = _dumptime_table->get(ik);
+  return (info != NULL) ? info->is_early_klass() : false;
+}
 
-  void check_loader_constraint_offset(int i) const {
-    assert(0 <= i && i < _num_loader_constraints, "sanity");
+bool SystemDictionaryShared::is_hidden_lambda_proxy(InstanceKlass* ik) {
+  assert(ik->is_shared(), "applicable to only a shared class");
+  if (ik->is_hidden()) {
+    return true;
+  } else {
+    return false;
   }
+}
 
-public:
-  CrcInfo* crc() const {
-    assert(crc_size(_klass) > 0, "must be");
-    return (CrcInfo*)(address(this) + crc_offset());
-  }
-  RTVerifierConstraint* verifier_constraints() {
-    assert(_num_verifier_constraints > 0, "sanity");
-    return (RTVerifierConstraint*)(address(this) + verifier_constraints_offset());
-  }
-  RTVerifierConstraint* verifier_constraint_at(int i) {
-    check_verifier_constraint_offset(i);
-    return verifier_constraints() + i;
+bool SystemDictionaryShared::check_for_exclusion_impl(InstanceKlass* k) {
+  if (k->is_in_error_state()) {
+    return warn_excluded(k, "In error state");
   }
-
-  char* verifier_constraint_flags() {
-    assert(_num_verifier_constraints > 0, "sanity");
-    return (char*)(address(this) + verifier_constraint_flags_offset());
+  if (k->is_scratch_class()) {
+    return warn_excluded(k, "A scratch class");
   }
-
-  InstanceKlass** nest_host_addr() {
-    assert(_klass->is_hidden(), "sanity");
-    return (InstanceKlass**)(address(this) + nest_host_offset());
+  if (!k->is_loaded()) {
+    return warn_excluded(k, "Not in loaded state");
   }
-  InstanceKlass* nest_host() {
-    return *nest_host_addr();
+  if (has_been_redefined(k)) {
+    return warn_excluded(k, "Has been redefined");
   }
-  void set_nest_host(InstanceKlass* k) {
-    *nest_host_addr() = k;
-    ArchivePtrMarker::mark_pointer((address*)nest_host_addr());
+  if (!k->is_hidden() && k->shared_classpath_index() < 0 && is_builtin(k)) {
+    // These are classes loaded from unsupported locations (such as those loaded by JVMTI native
+    // agent during dump time).
+    return warn_excluded(k, "Unsupported location");
   }
-
-  RTLoaderConstraint* loader_constraints() {
-    assert(_num_loader_constraints > 0, "sanity");
-    return (RTLoaderConstraint*)(address(this) + loader_constraints_offset());
+  if (k->signers() != NULL) {
+    // We cannot include signed classes in the archive because the certificates
+    // used during dump time may be different than those used during
+    // runtime (due to expiration, etc).
+    return warn_excluded(k, "Signed JAR");
   }
-
-  RTLoaderConstraint* loader_constraint_at(int i) {
-    check_loader_constraint_offset(i);
-    return loader_constraints() + i;
+  if (is_jfr_event_class(k)) {
+    // We cannot include JFR event classes because they need runtime-specific
+    // instrumentation in order to work with -XX:FlightRecorderOptions:retransform=false.
+    // There are only a small number of these classes, so it's not worthwhile to
+    // support them and make CDS more complicated.
+    return warn_excluded(k, "JFR event class");
   }
 
-  void init(DumpTimeSharedClassInfo& info) {
-    ArchiveBuilder* builder = ArchiveBuilder::current();
-    assert(builder->is_in_buffer_space(info._klass), "must be");
-    _klass = info._klass;
-    if (!SystemDictionaryShared::is_builtin(_klass)) {
-      CrcInfo* c = crc();
-      c->_clsfile_size = info._clsfile_size;
-      c->_clsfile_crc32 = info._clsfile_crc32;
-    }
-    _num_verifier_constraints = info.num_verifier_constraints();
-    _num_loader_constraints   = info.num_loader_constraints();
-    int i;
-    if (_num_verifier_constraints > 0) {
-      RTVerifierConstraint* vf_constraints = verifier_constraints();
-      char* flags = verifier_constraint_flags();
-      for (i = 0; i < _num_verifier_constraints; i++) {
-        vf_constraints[i]._name      = builder->any_to_offset_u4(info._verifier_constraints->at(i)._name);
-        vf_constraints[i]._from_name = builder->any_to_offset_u4(info._verifier_constraints->at(i)._from_name);
-      }
-      for (i = 0; i < _num_verifier_constraints; i++) {
-        flags[i] = info._verifier_constraint_flags->at(i);
-      }
-    }
-
-    if (_num_loader_constraints > 0) {
-      RTLoaderConstraint* ld_constraints = loader_constraints();
-      for (i = 0; i < _num_loader_constraints; i++) {
-        ld_constraints[i]._name = builder->any_to_offset_u4(info._loader_constraints->at(i)._name);
-        ld_constraints[i]._loader_type1 = info._loader_constraints->at(i)._loader_type1;
-        ld_constraints[i]._loader_type2 = info._loader_constraints->at(i)._loader_type2;
-      }
+  if (!k->is_linked()) {
+    if (has_class_failed_verification(k)) {
+      return warn_excluded(k, "Failed verification");
     }
-
-    if (_klass->is_hidden()) {
-      InstanceKlass* n_h = info.nest_host();
-      set_nest_host(n_h);
+  } else {
+    if (!k->can_be_verified_at_dumptime()) {
+      // We have an old class that has been linked (e.g., it's been executed during
+      // dump time). This class has been verified using the old verifier, which
+      // doesn't save the verification constraints, so check_verification_constraints()
+      // won't work at runtime.
+      // As a result, we cannot store this class. It must be loaded and fully verified
+      // at runtime.
+      return warn_excluded(k, "Old class has been linked");
     }
-    ArchivePtrMarker::mark_pointer(&_klass);
-  }
-
-  bool matches(int clsfile_size, int clsfile_crc32) const {
-    return crc()->_clsfile_size  == clsfile_size &&
-           crc()->_clsfile_crc32 == clsfile_crc32;
-  }
-
-  char verifier_constraint_flag(int i) {
-    check_verifier_constraint_offset(i);
-    return verifier_constraint_flags()[i];
-  }
-
-private:
-  // ArchiveBuilder::make_shallow_copy() has reserved a pointer immediately
-  // before archived InstanceKlasses. We can use this slot to do a quick
-  // lookup of InstanceKlass* -> RunTimeSharedClassInfo* without
-  // building a new hashtable.
-  //
-  //  info_pointer_addr(klass) --> 0x0100   RunTimeSharedClassInfo*
-  //  InstanceKlass* klass     --> 0x0108   <C++ vtbl>
-  //                               0x0110   fields from Klass ...
-  static RunTimeSharedClassInfo** info_pointer_addr(InstanceKlass* klass) {
-    return &((RunTimeSharedClassInfo**)klass)[-1];
   }
 
-public:
-  static RunTimeSharedClassInfo* get_for(InstanceKlass* klass) {
-    assert(klass->is_shared(), "don't call for non-shared class");
-    return *info_pointer_addr(klass);
-  }
-  static void set_for(InstanceKlass* klass, RunTimeSharedClassInfo* record) {
-    assert(ArchiveBuilder::current()->is_in_buffer_space(klass), "must be");
-    assert(ArchiveBuilder::current()->is_in_buffer_space(record), "must be");
-    *info_pointer_addr(klass) = record;
-    ArchivePtrMarker::mark_pointer(info_pointer_addr(klass));
+  if (k->is_hidden() && !is_registered_lambda_proxy_class(k)) {
+    ResourceMark rm;
+    log_debug(cds)("Skipping %s: Hidden class", k->name()->as_C_string());
+    return true;
   }
 
-  // Used by RunTimeSharedDictionary to implement OffsetCompactHashtable::EQUALS
-  static inline bool EQUALS(
-       const RunTimeSharedClassInfo* value, Symbol* key, int len_unused) {
-    return (value->_klass->name() == key);
+  InstanceKlass* super = k->java_super();
+  if (super != NULL && check_for_exclusion(super, NULL)) {
+    ResourceMark rm;
+    log_warning(cds)("Skipping %s: super class %s is excluded", k->name()->as_C_string(), super->name()->as_C_string());
+    return true;
   }
-};
-
-class RunTimeSharedDictionary : public OffsetCompactHashtable<
-  Symbol*,
-  const RunTimeSharedClassInfo*,
-  RunTimeSharedClassInfo::EQUALS> {};
-
-static DumpTimeSharedClassTable* _dumptime_table = NULL;
-// SystemDictionaries in the base layer static archive
-static RunTimeSharedDictionary _builtin_dictionary;
-static RunTimeSharedDictionary _unregistered_dictionary;
-// SystemDictionaries in the top layer dynamic archive
-static RunTimeSharedDictionary _dynamic_builtin_dictionary;
-static RunTimeSharedDictionary _dynamic_unregistered_dictionary;
 
-void SystemDictionaryShared::atomic_set_array_index(OopHandle array, int index, oop o) {
-  // Benign race condition:  array.obj_at(index) may already be filled in.
-  // The important thing here is that all threads pick up the same result.
-  // It doesn't matter which racing thread wins, as long as only one
-  // result is used by all threads, and all future queries.
-  ((objArrayOop)array.resolve())->atomic_compare_exchange_oop(index, o, NULL);
-}
-
-Handle SystemDictionaryShared::create_jar_manifest(const char* manifest_chars, size_t size, TRAPS) {
-  typeArrayOop buf = oopFactory::new_byteArray((int)size, CHECK_NH);
-  typeArrayHandle bufhandle(THREAD, buf);
-  ArrayAccess<>::arraycopy_from_native(reinterpret_cast<const jbyte*>(manifest_chars),
-                                         buf, typeArrayOopDesc::element_offset<jbyte>(0), size);
-  Handle bais = JavaCalls::construct_new_instance(vmClasses::ByteArrayInputStream_klass(),
-                      vmSymbols::byte_array_void_signature(),
-                      bufhandle, CHECK_NH);
-  // manifest = new Manifest(ByteArrayInputStream)
-  Handle manifest = JavaCalls::construct_new_instance(vmClasses::Jar_Manifest_klass(),
-                      vmSymbols::input_stream_void_signature(),
-                      bais, CHECK_NH);
-  return manifest;
-}
-
-oop SystemDictionaryShared::shared_protection_domain(int index) {
-  return ((objArrayOop)_shared_protection_domains.resolve())->obj_at(index);
-}
-
-oop SystemDictionaryShared::shared_jar_url(int index) {
-  return ((objArrayOop)_shared_jar_urls.resolve())->obj_at(index);
-}
-
-oop SystemDictionaryShared::shared_jar_manifest(int index) {
-  return ((objArrayOop)_shared_jar_manifests.resolve())->obj_at(index);
-}
-
-Handle SystemDictionaryShared::get_shared_jar_manifest(int shared_path_index, TRAPS) {
-  Handle manifest ;
-  if (shared_jar_manifest(shared_path_index) == NULL) {
-    SharedClassPathEntry* ent = FileMapInfo::shared_path(shared_path_index);
-    size_t size = (size_t)ent->manifest_size();
-    if (size == 0) {
-      return Handle();
-    }
-
-    // ByteArrayInputStream bais = new ByteArrayInputStream(buf);
-    const char* src = ent->manifest();
-    assert(src != NULL, "No Manifest data");
-    manifest = create_jar_manifest(src, size, CHECK_NH);
-    atomic_set_shared_jar_manifest(shared_path_index, manifest());
-  }
-  manifest = Handle(THREAD, shared_jar_manifest(shared_path_index));
-  assert(manifest.not_null(), "sanity");
-  return manifest;
-}
-
-Handle SystemDictionaryShared::get_shared_jar_url(int shared_path_index, TRAPS) {
-  Handle url_h;
-  if (shared_jar_url(shared_path_index) == NULL) {
-    JavaValue result(T_OBJECT);
-    const char* path = FileMapInfo::shared_path_name(shared_path_index);
-    Handle path_string = java_lang_String::create_from_str(path, CHECK_(url_h));
-    Klass* classLoaders_klass =
-        vmClasses::jdk_internal_loader_ClassLoaders_klass();
-    JavaCalls::call_static(&result, classLoaders_klass,
-                           vmSymbols::toFileURL_name(),
-                           vmSymbols::toFileURL_signature(),
-                           path_string, CHECK_(url_h));
-
-    atomic_set_shared_jar_url(shared_path_index, result.get_oop());
-  }
-
-  url_h = Handle(THREAD, shared_jar_url(shared_path_index));
-  assert(url_h.not_null(), "sanity");
-  return url_h;
-}
-
-Handle SystemDictionaryShared::get_package_name(Symbol* class_name, TRAPS) {
-  ResourceMark rm(THREAD);
-  Handle pkgname_string;
-  TempNewSymbol pkg = ClassLoader::package_from_class_name(class_name);
-  if (pkg != NULL) { // Package prefix found
-    const char* pkgname = pkg->as_klass_external_name();
-    pkgname_string = java_lang_String::create_from_str(pkgname,
-                                                       CHECK_(pkgname_string));
-  }
-  return pkgname_string;
-}
-
-// Define Package for shared app classes from JAR file and also checks for
-// package sealing (all done in Java code)
-// See http://docs.oracle.com/javase/tutorial/deployment/jar/sealman.html
-void SystemDictionaryShared::define_shared_package(Symbol*  class_name,
-                                                   Handle class_loader,
-                                                   Handle manifest,
-                                                   Handle url,
-                                                   TRAPS) {
-  assert(SystemDictionary::is_system_class_loader(class_loader()), "unexpected class loader");
-  // get_package_name() returns a NULL handle if the class is in unnamed package
-  Handle pkgname_string = get_package_name(class_name, CHECK);
-  if (pkgname_string.not_null()) {
-    Klass* app_classLoader_klass = vmClasses::jdk_internal_loader_ClassLoaders_AppClassLoader_klass();
-    JavaValue result(T_OBJECT);
-    JavaCallArguments args(3);
-    args.set_receiver(class_loader);
-    args.push_oop(pkgname_string);
-    args.push_oop(manifest);
-    args.push_oop(url);
-    JavaCalls::call_virtual(&result, app_classLoader_klass,
-                            vmSymbols::defineOrCheckPackage_name(),
-                            vmSymbols::defineOrCheckPackage_signature(),
-                            &args,
-                            CHECK);
-  }
-}
-
-// Get the ProtectionDomain associated with the CodeSource from the classloader.
-Handle SystemDictionaryShared::get_protection_domain_from_classloader(Handle class_loader,
-                                                                      Handle url, TRAPS) {
-  // CodeSource cs = new CodeSource(url, null);
-  Handle cs = JavaCalls::construct_new_instance(vmClasses::CodeSource_klass(),
-                  vmSymbols::url_code_signer_array_void_signature(),
-                  url, Handle(), CHECK_NH);
-
-  // protection_domain = SecureClassLoader.getProtectionDomain(cs);
-  Klass* secureClassLoader_klass = vmClasses::SecureClassLoader_klass();
-  JavaValue obj_result(T_OBJECT);
-  JavaCalls::call_virtual(&obj_result, class_loader, secureClassLoader_klass,
-                          vmSymbols::getProtectionDomain_name(),
-                          vmSymbols::getProtectionDomain_signature(),
-                          cs, CHECK_NH);
-  return Handle(THREAD, obj_result.get_oop());
-}
-
-// Returns the ProtectionDomain associated with the JAR file identified by the url.
-Handle SystemDictionaryShared::get_shared_protection_domain(Handle class_loader,
-                                                            int shared_path_index,
-                                                            Handle url,
-                                                            TRAPS) {
-  Handle protection_domain;
-  if (shared_protection_domain(shared_path_index) == NULL) {
-    Handle pd = get_protection_domain_from_classloader(class_loader, url, THREAD);
-    atomic_set_shared_protection_domain(shared_path_index, pd());
-  }
-
-  // Acquire from the cache because if another thread beats the current one to
-  // set the shared protection_domain and the atomic_set fails, the current thread
-  // needs to get the updated protection_domain from the cache.
-  protection_domain = Handle(THREAD, shared_protection_domain(shared_path_index));
-  assert(protection_domain.not_null(), "sanity");
-  return protection_domain;
-}
-
-// Returns the ProtectionDomain associated with the moduleEntry.
-Handle SystemDictionaryShared::get_shared_protection_domain(Handle class_loader,
-                                                            ModuleEntry* mod, TRAPS) {
-  ClassLoaderData *loader_data = mod->loader_data();
-  if (mod->shared_protection_domain() == NULL) {
-    Symbol* location = mod->location();
-    if (location != NULL) {
-      Handle location_string = java_lang_String::create_from_symbol(
-                                     location, CHECK_NH);
-      Handle url;
-      JavaValue result(T_OBJECT);
-      if (location->starts_with("jrt:/")) {
-        url = JavaCalls::construct_new_instance(vmClasses::URL_klass(),
-                                                vmSymbols::string_void_signature(),
-                                                location_string, CHECK_NH);
-      } else {
-        Klass* classLoaders_klass =
-          vmClasses::jdk_internal_loader_ClassLoaders_klass();
-        JavaCalls::call_static(&result, classLoaders_klass, vmSymbols::toFileURL_name(),
-                               vmSymbols::toFileURL_signature(),
-                               location_string, CHECK_NH);
-        url = Handle(THREAD, result.get_oop());
-      }
-
-      Handle pd = get_protection_domain_from_classloader(class_loader, url,
-                                                         CHECK_NH);
-      mod->set_shared_protection_domain(loader_data, pd);
+  Array<InstanceKlass*>* interfaces = k->local_interfaces();
+  int len = interfaces->length();
+  for (int i = 0; i < len; i++) {
+    InstanceKlass* intf = interfaces->at(i);
+    if (check_for_exclusion(intf, NULL)) {
+      log_warning(cds)("Skipping %s: interface %s is excluded", k->name()->as_C_string(), intf->name()->as_C_string());
+      return true;
     }
   }
 
-  Handle protection_domain(THREAD, mod->shared_protection_domain());
-  assert(protection_domain.not_null(), "sanity");
-  return protection_domain;
-}
-
-// Initializes the java.lang.Package and java.security.ProtectionDomain objects associated with
-// the given InstanceKlass.
-// Returns the ProtectionDomain for the InstanceKlass.
-Handle SystemDictionaryShared::init_security_info(Handle class_loader, InstanceKlass* ik, PackageEntry* pkg_entry, TRAPS) {
-  Handle pd;
-
-  if (ik != NULL) {
-    int index = ik->shared_classpath_index();
-    assert(index >= 0, "Sanity");
-    SharedClassPathEntry* ent = FileMapInfo::shared_path(index);
-    Symbol* class_name = ik->name();
-
-    if (ent->is_modules_image()) {
-      // For shared app/platform classes originated from the run-time image:
-      //   The ProtectionDomains are cached in the corresponding ModuleEntries
-      //   for fast access by the VM.
-      // all packages from module image are already created during VM bootstrap in
-      // Modules::define_module().
-      assert(pkg_entry != NULL, "archived class in module image cannot be from unnamed package");
-      ModuleEntry* mod_entry = pkg_entry->module();
-      pd = get_shared_protection_domain(class_loader, mod_entry, CHECK_(pd));
-    } else {
-      // For shared app/platform classes originated from JAR files on the class path:
-      //   Each of the 3 SystemDictionaryShared::_shared_xxx arrays has the same length
-      //   as the shared classpath table in the shared archive (see
-      //   FileMap::_shared_path_table in filemap.hpp for details).
-      //
-      //   If a shared InstanceKlass k is loaded from the class path, let
-      //
-      //     index = k->shared_classpath_index():
-      //
-      //   FileMap::_shared_path_table[index] identifies the JAR file that contains k.
-      //
-      //   k's protection domain is:
-      //
-      //     ProtectionDomain pd = _shared_protection_domains[index];
-      //
-      //   and k's Package is initialized using
-      //
-      //     manifest = _shared_jar_manifests[index];
-      //     url = _shared_jar_urls[index];
-      //     define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
-      //
-      //   Note that if an element of these 3 _shared_xxx arrays is NULL, it will be initialized by
-      //   the corresponding SystemDictionaryShared::get_shared_xxx() function.
-      Handle manifest = get_shared_jar_manifest(index, CHECK_(pd));
-      Handle url = get_shared_jar_url(index, CHECK_(pd));
-      int index_offset = index - ClassLoaderExt::app_class_paths_start_index();
-      if (index_offset < PackageEntry::max_index_for_defined_in_class_path()) {
-        if (pkg_entry == NULL || !pkg_entry->is_defined_by_cds_in_class_path(index_offset)) {
-          // define_shared_package only needs to be called once for each package in a jar specified
-          // in the shared class path.
-          define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
-          if (pkg_entry != NULL) {
-            pkg_entry->set_defined_by_cds_in_class_path(index_offset);
-          }
-        }
-      } else {
-        define_shared_package(class_name, class_loader, manifest, url, CHECK_(pd));
-      }
-      pd = get_shared_protection_domain(class_loader, index, url, CHECK_(pd));
-    }
-  }
-  return pd;
+  return false; // false == k should NOT be excluded
 }
 
 bool SystemDictionaryShared::is_sharing_possible(ClassLoaderData* loader_data) {
@@ -1018,176 +414,51 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class(
 
       k = load_shared_class_for_builtin_loader(name, class_loader, THREAD);
       if (k != NULL) {
-        SharedClassLoadingMark slm(THREAD, k);
-        k = find_or_define_instance_class(name, class_loader, k, CHECK_NULL);
-      }
-    }
-  }
-  return k;
-}
-
-PackageEntry* SystemDictionaryShared::get_package_entry_from_class(InstanceKlass* ik, Handle class_loader) {
-  PackageEntry* pkg_entry = ik->package();
-  if (MetaspaceShared::use_full_module_graph() && ik->is_shared() && pkg_entry != NULL) {
-    assert(MetaspaceShared::is_in_shared_metaspace(pkg_entry), "must be");
-    assert(!ik->is_shared_unregistered_class(), "unexpected archived package entry for an unregistered class");
-    assert(ik->module()->is_named(), "unexpected archived package entry for a class in an unnamed module");
-    return pkg_entry;
-  }
-  TempNewSymbol pkg_name = ClassLoader::package_from_class_name(ik->name());
-  if (pkg_name != NULL) {
-    pkg_entry = class_loader_data(class_loader)->packages()->lookup_only(pkg_name);
-  } else {
-    pkg_entry = NULL;
-  }
-  return pkg_entry;
-}
-
-InstanceKlass* SystemDictionaryShared::load_shared_class_for_builtin_loader(
-                 Symbol* class_name, Handle class_loader, TRAPS) {
-  assert(UseSharedSpaces, "must be");
-  InstanceKlass* ik = find_builtin_class(class_name);
-
-  if (ik != NULL && !ik->shared_loading_failed()) {
-    if ((SystemDictionary::is_system_class_loader(class_loader()) && ik->is_shared_app_class())  ||
-        (SystemDictionary::is_platform_class_loader(class_loader()) && ik->is_shared_platform_class())) {
-      SharedClassLoadingMark slm(THREAD, ik);
-      PackageEntry* pkg_entry = get_package_entry_from_class(ik, class_loader);
-      Handle protection_domain =
-        SystemDictionaryShared::init_security_info(class_loader, ik, pkg_entry, CHECK_NULL);
-      return load_shared_class(ik, class_loader, protection_domain, NULL, pkg_entry, THREAD);
-    }
-  }
-  return NULL;
-}
-
-void SystemDictionaryShared::allocate_shared_protection_domain_array(int size, TRAPS) {
-  if (_shared_protection_domains.resolve() == NULL) {
-    oop spd = oopFactory::new_objArray(
-        vmClasses::ProtectionDomain_klass(), size, CHECK);
-    _shared_protection_domains = OopHandle(Universe::vm_global(), spd);
-  }
-}
-
-void SystemDictionaryShared::allocate_shared_jar_url_array(int size, TRAPS) {
-  if (_shared_jar_urls.resolve() == NULL) {
-    oop sju = oopFactory::new_objArray(
-        vmClasses::URL_klass(), size, CHECK);
-    _shared_jar_urls = OopHandle(Universe::vm_global(), sju);
-  }
-}
-
-void SystemDictionaryShared::allocate_shared_jar_manifest_array(int size, TRAPS) {
-  if (_shared_jar_manifests.resolve() == NULL) {
-    oop sjm = oopFactory::new_objArray(
-        vmClasses::Jar_Manifest_klass(), size, CHECK);
-    _shared_jar_manifests = OopHandle(Universe::vm_global(), sjm);
-  }
-}
-
-void SystemDictionaryShared::allocate_shared_data_arrays(int size, TRAPS) {
-  allocate_shared_protection_domain_array(size, CHECK);
-  allocate_shared_jar_url_array(size, CHECK);
-  allocate_shared_jar_manifest_array(size, CHECK);
-}
-
-// This function is called for loading only UNREGISTERED classes
-InstanceKlass* SystemDictionaryShared::lookup_from_stream(Symbol* class_name,
-                                                          Handle class_loader,
-                                                          Handle protection_domain,
-                                                          const ClassFileStream* cfs,
-                                                          TRAPS) {
-  if (!UseSharedSpaces) {
-    return NULL;
-  }
-  if (class_name == NULL) {  // don't do this for hidden classes
-    return NULL;
-  }
-  if (class_loader.is_null() ||
-      SystemDictionary::is_system_class_loader(class_loader()) ||
-      SystemDictionary::is_platform_class_loader(class_loader())) {
-    // Do nothing for the BUILTIN loaders.
-    return NULL;
-  }
-
-  const RunTimeSharedClassInfo* record = find_record(&_unregistered_dictionary, &_dynamic_unregistered_dictionary, class_name);
-  if (record == NULL) {
-    return NULL;
-  }
-
-  int clsfile_size  = cfs->length();
-  int clsfile_crc32 = ClassLoader::crc32(0, (const char*)cfs->buffer(), cfs->length());
-
-  if (!record->matches(clsfile_size, clsfile_crc32)) {
-    return NULL;
-  }
-
-  return acquire_class_for_current_thread(record->_klass, class_loader,
-                                          protection_domain, cfs,
-                                          THREAD);
-}
-
-InstanceKlass* SystemDictionaryShared::acquire_class_for_current_thread(
-                   InstanceKlass *ik,
-                   Handle class_loader,
-                   Handle protection_domain,
-                   const ClassFileStream *cfs,
-                   TRAPS) {
-  ClassLoaderData* loader_data = ClassLoaderData::class_loader_data(class_loader());
-
-  {
-    MutexLocker mu(THREAD, SharedDictionary_lock);
-    if (ik->class_loader_data() != NULL) {
-      //    ik is already loaded (by this loader or by a different loader)
-      // or ik is being loaded by a different thread (by this loader or by a different loader)
-      return NULL;
-    }
-
-    // No other thread has acquired this yet, so give it to *this thread*
-    ik->set_class_loader_data(loader_data);
-  }
-
-  // No longer holding SharedDictionary_lock
-  // No need to lock, as <ik> can be held only by a single thread.
-  loader_data->add_class(ik);
-
-  // Get the package entry.
-  PackageEntry* pkg_entry = get_package_entry_from_class(ik, class_loader);
-
-  // Load and check super/interfaces, restore unsharable info
-  InstanceKlass* shared_klass = load_shared_class(ik, class_loader, protection_domain,
-                                                  cfs, pkg_entry, THREAD);
-  if (shared_klass == NULL || HAS_PENDING_EXCEPTION) {
-    // TODO: clean up <ik> so it can be used again
-    return NULL;
+        SharedClassLoadingMark slm(THREAD, k);
+        k = find_or_define_instance_class(name, class_loader, k, CHECK_NULL);
+      }
+    }
   }
-
-  return shared_klass;
+  return k;
 }
 
-class LoadedUnregisteredClassesTable : public ResourceHashtable<
-  Symbol*, bool,
+class UnregisteredClassesTable : public ResourceHashtable<
+  Symbol*, InstanceKlass*,
   primitive_hash<Symbol*>,
   primitive_equals<Symbol*>,
-  6661,                             // prime number
+  15889, // prime number
   ResourceObj::C_HEAP> {};
 
-static LoadedUnregisteredClassesTable* _loaded_unregistered_classes = NULL;
+static UnregisteredClassesTable* _unregistered_classes_table = NULL;
 
-bool SystemDictionaryShared::add_unregistered_class(Thread* current, InstanceKlass* k) {
-  // We don't allow duplicated unregistered classes of the same name.
-  assert(DumpSharedSpaces, "only when dumping");
-  Symbol* name = k->name();
-  if (_loaded_unregistered_classes == NULL) {
-    _loaded_unregistered_classes = new (ResourceObj::C_HEAP, mtClass)LoadedUnregisteredClassesTable();
+bool SystemDictionaryShared::add_unregistered_class(Thread* current, InstanceKlass* klass) {
+  // We don't allow duplicated unregistered classes with the same name.
+  // We only archive the first class with that name that succeeds putting
+  // itself into the table.
+  Arguments::assert_is_dumping_archive();
+  MutexLocker ml(current, UnregisteredClassesTable_lock);
+  Symbol* name = klass->name();
+  if (_unregistered_classes_table == NULL) {
+    _unregistered_classes_table = new (ResourceObj::C_HEAP, mtClass)UnregisteredClassesTable();
   }
-  bool created = false;
-  _loaded_unregistered_classes->put_if_absent(name, true, &created);
+  bool created;
+  InstanceKlass** v = _unregistered_classes_table->put_if_absent(name, klass, &created);
   if (created) {
+    name->increment_refcount();
+  }
+  return (klass == *v);
+}
+
+// true == class was successfully added; false == a duplicated class (with the same name) already exists.
+bool SystemDictionaryShared::add_unregistered_class_for_static_archive(Thread* current, InstanceKlass* k) {
+  assert(DumpSharedSpaces, "only when dumping");
+  if (add_unregistered_class(current, k)) {
     MutexLocker mu_r(current, Compile_lock); // add_to_hierarchy asserts this.
     SystemDictionary::add_to_hierarchy(k);
+    return true;
+  } else {
+    return false;
   }
-  return created;
 }
 
 // This function is called to lookup the super/interfaces of shared classes for
@@ -1229,28 +500,10 @@ InstanceKlass* SystemDictionaryShared::lookup_super_for_unregistered_class(
   }
 }
 
-void SystemDictionaryShared::start_dumping() {
-  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
-  _dump_in_progress = true;
-}
-
-DumpTimeSharedClassInfo* SystemDictionaryShared::find_or_allocate_info_for(InstanceKlass* k) {
-  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
-  return find_or_allocate_info_for_locked(k);
-}
-
-DumpTimeSharedClassInfo* SystemDictionaryShared::find_or_allocate_info_for_locked(InstanceKlass* k) {
-  assert_lock_strong(DumpTimeTable_lock);
-  if (_dumptime_table == NULL) {
-    _dumptime_table = new (ResourceObj::C_HEAP, mtClass)DumpTimeSharedClassTable();
-  }
-  return _dumptime_table->find_or_allocate_info_for(k, _dump_in_progress);
-}
-
 void SystemDictionaryShared::set_shared_class_misc_info(InstanceKlass* k, ClassFileStream* cfs) {
   Arguments::assert_is_dumping_archive();
   assert(!is_builtin(k), "must be unregistered class");
-  DumpTimeSharedClassInfo* info = find_or_allocate_info_for(k);
+  DumpTimeClassInfo* info = find_or_allocate_info_for(k);
   if (info != NULL) {
     info->_clsfile_size  = cfs->length();
     info->_clsfile_crc32 = ClassLoader::crc32(0, (const char*)cfs->buffer(), cfs->length());
@@ -1263,13 +516,13 @@ void SystemDictionaryShared::init_dumptime_info(InstanceKlass* k) {
 
 void SystemDictionaryShared::remove_dumptime_info(InstanceKlass* k) {
   MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
-  DumpTimeSharedClassInfo* p = _dumptime_table->get(k);
+  DumpTimeClassInfo* p = _dumptime_table->get(k);
   if (p == NULL) {
     return;
   }
   if (p->_verifier_constraints != NULL) {
     for (int i = 0; i < p->_verifier_constraints->length(); i++) {
-      DumpTimeSharedClassInfo::DTVerifierConstraint constraint = p->_verifier_constraints->at(i);
+      DumpTimeClassInfo::DTVerifierConstraint constraint = p->_verifier_constraints->at(i);
       if (constraint._name != NULL ) {
         constraint._name->decrement_refcount();
       }
@@ -1277,81 +530,37 @@ void SystemDictionaryShared::remove_dumptime_info(InstanceKlass* k) {
         constraint._from_name->decrement_refcount();
       }
     }
-    FREE_C_HEAP_ARRAY(DumpTimeSharedClassInfo::DTVerifierConstraint, p->_verifier_constraints);
+    FREE_C_HEAP_ARRAY(DumpTimeClassInfo::DTVerifierConstraint, p->_verifier_constraints);
     p->_verifier_constraints = NULL;
     FREE_C_HEAP_ARRAY(char, p->_verifier_constraint_flags);
     p->_verifier_constraint_flags = NULL;
   }
   if (p->_loader_constraints != NULL) {
     for (int i = 0; i < p->_loader_constraints->length(); i++) {
-      DumpTimeSharedClassInfo::DTLoaderConstraint ld =  p->_loader_constraints->at(i);
+      DumpTimeClassInfo::DTLoaderConstraint ld =  p->_loader_constraints->at(i);
       if (ld._name != NULL) {
         ld._name->decrement_refcount();
       }
     }
-    FREE_C_HEAP_ARRAY(DumpTimeSharedClassInfo::DTLoaderConstraint, p->_loader_constraints);
+    FREE_C_HEAP_ARRAY(DumpTimeClassInfo::DTLoaderConstraint, p->_loader_constraints);
     p->_loader_constraints = NULL;
   }
   _dumptime_table->remove(k);
 }
 
-bool SystemDictionaryShared::is_jfr_event_class(InstanceKlass *k) {
-  while (k) {
-    if (k->name()->equals("jdk/internal/event/Event")) {
-      return true;
-    }
-    k = k->java_super();
-  }
-  return false;
-}
-
-bool SystemDictionaryShared::is_registered_lambda_proxy_class(InstanceKlass* ik) {
-  DumpTimeSharedClassInfo* info = _dumptime_table->get(ik);
-  return (info != NULL) ? info->_is_archived_lambda_proxy : false;
-}
-
-bool SystemDictionaryShared::is_hidden_lambda_proxy(InstanceKlass* ik) {
-  assert(ik->is_shared(), "applicable to only a shared class");
-  if (ik->is_hidden()) {
-    return true;
-  } else {
-    return false;
-  }
-}
-
-bool SystemDictionaryShared::is_early_klass(InstanceKlass* ik) {
-  DumpTimeSharedClassInfo* info = _dumptime_table->get(ik);
-  return (info != NULL) ? info->is_early_klass() : false;
-}
-
-// Returns true so the caller can do:    return warn_excluded(".....");
-bool SystemDictionaryShared::warn_excluded(InstanceKlass* k, const char* reason) {
-  ResourceMark rm;
-  log_warning(cds)("Skipping %s: %s", k->name()->as_C_string(), reason);
-  return true;
-}
-
-bool SystemDictionaryShared::check_for_exclusion(InstanceKlass* k, DumpTimeSharedClassInfo* info) {
-  if (MetaspaceShared::is_in_shared_metaspace(k)) {
-    // We have reached a super type that's already in the base archive. Treat it
-    // as "not excluded".
-    assert(DynamicDumpSharedSpaces, "must be");
-    return false;
-  }
-
-  if (info == NULL) {
-    info = _dumptime_table->get(k);
-    assert(info != NULL, "supertypes of any classes in _dumptime_table must either be shared, or must also be in _dumptime_table");
-  }
+void SystemDictionaryShared::handle_class_unloading(InstanceKlass* klass) {
+  remove_dumptime_info(klass);
 
-  if (!info->has_checked_exclusion()) {
-    if (check_for_exclusion_impl(k)) {
-      info->set_excluded();
+  if (_unregistered_classes_table != NULL) {
+    // Remove the class from _unregistered_classes_table: keep the entry but
+    // set it to NULL. This ensure no classes with the same name can be
+    // added again.
+    MutexLocker ml(Thread::current(), UnregisteredClassesTable_lock);
+    InstanceKlass** v = _unregistered_classes_table->get(klass->name());
+    if (v != NULL) {
+      *v = NULL;
     }
-    info->set_has_checked_exclusion();
   }
-
-  return info->is_excluded();
 }
 
 // Check if a class or any of its supertypes has been redefined.
@@ -1372,94 +581,11 @@ bool SystemDictionaryShared::has_been_redefined(InstanceKlass* k) {
   return false;
 }
 
-bool SystemDictionaryShared::check_for_exclusion_impl(InstanceKlass* k) {
-  if (k->is_in_error_state()) {
-    return warn_excluded(k, "In error state");
-  }
-  if (has_been_redefined(k)) {
-    return warn_excluded(k, "Has been redefined");
-  }
-  if (!k->is_hidden() && k->shared_classpath_index() < 0 && is_builtin(k)) {
-    // These are classes loaded from unsupported locations (such as those loaded by JVMTI native
-    // agent during dump time).
-    return warn_excluded(k, "Unsupported location");
-  }
-  if (k->signers() != NULL) {
-    // We cannot include signed classes in the archive because the certificates
-    // used during dump time may be different than those used during
-    // runtime (due to expiration, etc).
-    return warn_excluded(k, "Signed JAR");
-  }
-  if (is_jfr_event_class(k)) {
-    // We cannot include JFR event classes because they need runtime-specific
-    // instrumentation in order to work with -XX:FlightRecorderOptions:retransform=false.
-    // There are only a small number of these classes, so it's not worthwhile to
-    // support them and make CDS more complicated.
-    return warn_excluded(k, "JFR event class");
-  }
-  if (k->init_state() < InstanceKlass::linked) {
-    // In CDS dumping, we will attempt to link all classes. Those that fail to link will
-    // be recorded in DumpTimeSharedClassInfo.
-    Arguments::assert_is_dumping_archive();
-
-    // TODO -- rethink how this can be handled.
-    // We should try to link ik, however, we can't do it here because
-    // 1. We are at VM exit
-    // 2. linking a class may cause other classes to be loaded, which means
-    //    a custom ClassLoader.loadClass() may be called, at a point where the
-    //    class loader doesn't expect it.
-    if (has_class_failed_verification(k)) {
-      return warn_excluded(k, "Failed verification");
-    } else {
-      if (k->can_be_verified_at_dumptime()) {
-        return warn_excluded(k, "Not linked");
-      }
-    }
-  }
-  if (DynamicDumpSharedSpaces && k->major_version() < 50 /*JAVA_6_VERSION*/) {
-    // In order to support old classes during dynamic dump, class rewriting needs to
-    // be reverted. This would result in more complex code and testing but not much gain.
-    ResourceMark rm;
-    log_warning(cds)("Pre JDK 6 class not supported by CDS: %u.%u %s",
-                     k->major_version(),  k->minor_version(), k->name()->as_C_string());
-    return true;
-  }
-
-  if (!k->can_be_verified_at_dumptime() && k->is_linked()) {
-    return warn_excluded(k, "Old class has been linked");
-  }
-
-  if (k->is_hidden() && !is_registered_lambda_proxy_class(k)) {
-    ResourceMark rm;
-    log_debug(cds)("Skipping %s: Hidden class", k->name()->as_C_string());
-    return true;
-  }
-
-  InstanceKlass* super = k->java_super();
-  if (super != NULL && check_for_exclusion(super, NULL)) {
-    ResourceMark rm;
-    log_warning(cds)("Skipping %s: super class %s is excluded", k->name()->as_C_string(), super->name()->as_C_string());
-    return true;
-  }
-
-  Array<InstanceKlass*>* interfaces = k->local_interfaces();
-  int len = interfaces->length();
-  for (int i = 0; i < len; i++) {
-    InstanceKlass* intf = interfaces->at(i);
-    if (check_for_exclusion(intf, NULL)) {
-      log_warning(cds)("Skipping %s: interface %s is excluded", k->name()->as_C_string(), intf->name()->as_C_string());
-      return true;
-    }
-  }
-
-  return false; // false == k should NOT be excluded
-}
-
 // k is a class before relocating by ArchiveBuilder
 void SystemDictionaryShared::validate_before_archiving(InstanceKlass* k) {
   ResourceMark rm;
   const char* name = k->name()->as_C_string();
-  DumpTimeSharedClassInfo* info = _dumptime_table->get(k);
+  DumpTimeClassInfo* info = _dumptime_table->get(k);
   assert(_no_class_loading_should_happen, "class loading must be disabled");
   guarantee(info != NULL, "Class %s must be entered into _dumptime_table", name);
   guarantee(!info->is_excluded(), "Should not attempt to archive excluded class %s", name);
@@ -1476,15 +602,69 @@ void SystemDictionaryShared::validate_before_archiving(InstanceKlass* k) {
   }
 }
 
+class UnregisteredClassesDuplicationChecker : StackObj {
+  GrowableArray<InstanceKlass*> _list;
+  Thread* _thread;
+public:
+  UnregisteredClassesDuplicationChecker() : _thread(Thread::current()) {}
+
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
+    if (!SystemDictionaryShared::is_builtin(k)) {
+      _list.append(k);
+    }
+    return true;  // keep on iterating
+  }
+
+  static int compare_by_loader(InstanceKlass** a, InstanceKlass** b) {
+    ClassLoaderData* loader_a = a[0]->class_loader_data();
+    ClassLoaderData* loader_b = b[0]->class_loader_data();
+
+    if (loader_a != loader_b) {
+      return intx(loader_a) - intx(loader_b);
+    } else {
+      return intx(a[0]) - intx(b[0]);
+    }
+  }
+
+  void mark_duplicated_classes() {
+    // Two loaders may load two identical or similar hierarchies of classes. If we
+    // check for duplication in random order, we may end up excluding important base classes
+    // in both hierarchies, causing most of the classes to be excluded.
+    // We sort the classes by their loaders. This way we're likely to archive
+    // all classes in the one of the two hierarchies.
+    _list.sort(compare_by_loader);
+    for (int i = 0; i < _list.length(); i++) {
+      InstanceKlass* k = _list.at(i);
+      bool i_am_first = SystemDictionaryShared::add_unregistered_class(_thread, k);
+      if (!i_am_first) {
+        SystemDictionaryShared::warn_excluded(k, "Duplicated unregistered class");
+        SystemDictionaryShared::set_excluded_locked(k);
+      }
+    }
+  }
+};
+
 class ExcludeDumpTimeSharedClasses : StackObj {
 public:
-  bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
     SystemDictionaryShared::check_for_exclusion(k, &info);
     return true; // keep on iterating
   }
 };
 
 void SystemDictionaryShared::check_excluded_classes() {
+  assert(no_class_loading_should_happen(), "sanity");
+  assert_lock_strong(DumpTimeTable_lock);
+
+  if (DynamicDumpSharedSpaces) {
+    // Do this first -- if a base class is excluded due to duplication,
+    // all of its subclasses will also be excluded by ExcludeDumpTimeSharedClasses
+    ResourceMark rm;
+    UnregisteredClassesDuplicationChecker dup_checker;
+    _dumptime_table->iterate(&dup_checker);
+    dup_checker.mark_duplicated_classes();
+  }
+
   ExcludeDumpTimeSharedClasses excl;
   _dumptime_table->iterate(&excl);
   _dumptime_table->update_counts();
@@ -1494,13 +674,22 @@ bool SystemDictionaryShared::is_excluded_class(InstanceKlass* k) {
   assert(_no_class_loading_should_happen, "sanity");
   assert_lock_strong(DumpTimeTable_lock);
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* p = find_or_allocate_info_for_locked(k);
+  DumpTimeClassInfo* p = find_or_allocate_info_for_locked(k);
   return (p == NULL) ? true : p->is_excluded();
 }
 
+void SystemDictionaryShared::set_excluded_locked(InstanceKlass* k) {
+  assert_lock_strong(DumpTimeTable_lock);
+  Arguments::assert_is_dumping_archive();
+  DumpTimeClassInfo* info = find_or_allocate_info_for_locked(k);
+  if (info != NULL) {
+    info->set_excluded();
+  }
+}
+
 void SystemDictionaryShared::set_excluded(InstanceKlass* k) {
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* info = find_or_allocate_info_for(k);
+  DumpTimeClassInfo* info = find_or_allocate_info_for(k);
   if (info != NULL) {
     info->set_excluded();
   }
@@ -1508,7 +697,7 @@ void SystemDictionaryShared::set_excluded(InstanceKlass* k) {
 
 void SystemDictionaryShared::set_class_has_failed_verification(InstanceKlass* ik) {
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* p = find_or_allocate_info_for(ik);
+  DumpTimeClassInfo* p = find_or_allocate_info_for(ik);
   if (p != NULL) {
     p->set_failed_verification();
   }
@@ -1521,7 +710,7 @@ bool SystemDictionaryShared::has_class_failed_verification(InstanceKlass* ik) {
     assert(ik->is_shared(), "must be a shared class in the static archive");
     return false;
   }
-  DumpTimeSharedClassInfo* p = _dumptime_table->get(ik);
+  DumpTimeClassInfo* p = _dumptime_table->get(ik);
   return (p == NULL) ? false : p->failed_verification();
 }
 
@@ -1530,7 +719,7 @@ class IterateDumpTimeSharedClassTable : StackObj {
 public:
   IterateDumpTimeSharedClassTable(MetaspaceClosure* it) : _it(it) {}
 
-  bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
     assert_lock_strong(DumpTimeTable_lock);
     if (k->is_loader_alive() && !info.is_excluded()) {
       info.metaspace_pointers_do(_it);
@@ -1567,7 +756,7 @@ void SystemDictionaryShared::dumptime_classes_do(class MetaspaceClosure* it) {
 bool SystemDictionaryShared::add_verification_constraint(InstanceKlass* k, Symbol* name,
          Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object) {
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* info = find_or_allocate_info_for(k);
+  DumpTimeClassInfo* info = find_or_allocate_info_for(k);
   if (info != NULL) {
     info->add_verification_constraint(k, name, from_name, from_field_is_protected,
                                       from_is_array, from_is_object);
@@ -1593,36 +782,23 @@ bool SystemDictionaryShared::add_verification_constraint(InstanceKlass* k, Symbo
   }
 }
 
-void DumpTimeSharedClassInfo::add_verification_constraint(InstanceKlass* k, Symbol* name,
-         Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object) {
-  if (_verifier_constraints == NULL) {
-    _verifier_constraints = new(ResourceObj::C_HEAP, mtClass) GrowableArray<DTVerifierConstraint>(4, mtClass);
-  }
-  if (_verifier_constraint_flags == NULL) {
-    _verifier_constraint_flags = new(ResourceObj::C_HEAP, mtClass) GrowableArray<char>(4, mtClass);
-  }
-  GrowableArray<DTVerifierConstraint>* vc_array = _verifier_constraints;
-  for (int i = 0; i < vc_array->length(); i++) {
-    DTVerifierConstraint* p = vc_array->adr_at(i);
-    if (name == p->_name && from_name == p->_from_name) {
-      return;
-    }
+void SystemDictionaryShared::add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey& key,
+                                                           InstanceKlass* proxy_klass) {
+  assert_lock_strong(DumpTimeTable_lock);
+  if (_dumptime_lambda_proxy_class_dictionary == NULL) {
+    _dumptime_lambda_proxy_class_dictionary =
+      new (ResourceObj::C_HEAP, mtClass) DumpTimeLambdaProxyClassDictionary;
   }
-  DTVerifierConstraint cons(name, from_name);
-  vc_array->append(cons);
-
-  GrowableArray<char>* vcflags_array = _verifier_constraint_flags;
-  char c = 0;
-  c |= from_field_is_protected ? SystemDictionaryShared::FROM_FIELD_IS_PROTECTED : 0;
-  c |= from_is_array           ? SystemDictionaryShared::FROM_IS_ARRAY           : 0;
-  c |= from_is_object          ? SystemDictionaryShared::FROM_IS_OBJECT          : 0;
-  vcflags_array->append(c);
-
-  if (log_is_enabled(Trace, cds, verification)) {
-    ResourceMark rm;
-    log_trace(cds, verification)("add_verification_constraint: %s: %s must be subclass of %s [0x%x] array len %d flags len %d",
-                                 k->external_name(), from_name->as_klass_external_name(),
-                                 name->as_klass_external_name(), c, vc_array->length(), vcflags_array->length());
+  DumpTimeLambdaProxyClassInfo* lambda_info = _dumptime_lambda_proxy_class_dictionary->get(key);
+  if (lambda_info == NULL) {
+    DumpTimeLambdaProxyClassInfo info;
+    info.add_proxy_klass(proxy_klass);
+    _dumptime_lambda_proxy_class_dictionary->put(key, info);
+    //lambda_info = _dumptime_lambda_proxy_class_dictionary->get(key);
+    //assert(lambda_info->_proxy_klass == proxy_klass, "must be"); // debug only -- remove
+    ++_dumptime_lambda_proxy_class_dictionary->_count;
+  } else {
+    lambda_info->add_proxy_klass(proxy_klass);
   }
 }
 
@@ -1646,11 +822,11 @@ void SystemDictionaryShared::add_lambda_proxy_class(InstanceKlass* caller_ik,
   InstanceKlass* nest_host = caller_ik->nest_host(CHECK);
   assert(nest_host != NULL, "unexpected NULL nest_host");
 
-  DumpTimeSharedClassInfo* info = _dumptime_table->get(lambda_ik);
+  DumpTimeClassInfo* info = _dumptime_table->get(lambda_ik);
   if (info != NULL && !lambda_ik->is_non_strong_hidden() && is_builtin(lambda_ik) && is_builtin(caller_ik)
       // Don't include the lambda proxy if its nest host is not in the "linked" state.
       && nest_host->is_linked()) {
-    // Set _is_archived_lambda_proxy in DumpTimeSharedClassInfo so that the lambda_ik
+    // Set _is_archived_lambda_proxy in DumpTimeClassInfo so that the lambda_ik
     // won't be excluded during dumping of shared archive. See ExcludeDumpTimeSharedClasses.
     info->_is_archived_lambda_proxy = true;
     info->set_nest_host(nest_host);
@@ -1711,7 +887,7 @@ InstanceKlass* SystemDictionaryShared::get_shared_lambda_proxy_class(InstanceKla
 
 InstanceKlass* SystemDictionaryShared::get_shared_nest_host(InstanceKlass* lambda_ik) {
   assert(!DumpSharedSpaces && UseSharedSpaces, "called at run time with CDS enabled only");
-  RunTimeSharedClassInfo* record = RunTimeSharedClassInfo::get_for(lambda_ik);
+  RunTimeClassInfo* record = RunTimeClassInfo::get_for(lambda_ik);
   return record->nest_host();
 }
 
@@ -1721,7 +897,7 @@ InstanceKlass* SystemDictionaryShared::prepare_shared_lambda_proxy_class(Instanc
   Handle protection_domain;
   PackageEntry* pkg_entry = caller_ik->package();
   if (caller_ik->class_loader() != NULL) {
-    protection_domain = SystemDictionaryShared::init_security_info(class_loader, caller_ik, pkg_entry, CHECK_NULL);
+    protection_domain = CDSProtectionDomain::init_security_info(class_loader, caller_ik, pkg_entry, CHECK_NULL);
   }
 
   InstanceKlass* shared_nest_host = get_shared_nest_host(lambda_ik);
@@ -1761,73 +937,15 @@ InstanceKlass* SystemDictionaryShared::prepare_shared_lambda_proxy_class(Instanc
   return loaded_lambda;
 }
 
-static char get_loader_type_by(oop  loader) {
-  assert(SystemDictionary::is_builtin_class_loader(loader), "Must be built-in loader");
-  if (SystemDictionary::is_boot_class_loader(loader)) {
-    return (char)ClassLoader::BOOT_LOADER;
-  } else if (SystemDictionary::is_platform_class_loader(loader)) {
-    return (char)ClassLoader::PLATFORM_LOADER;
-  } else {
-    assert(SystemDictionary::is_system_class_loader(loader), "Class loader mismatch");
-    return (char)ClassLoader::APP_LOADER;
-  }
-}
-
-static oop get_class_loader_by(char type) {
-  if (type == (char)ClassLoader::BOOT_LOADER) {
-    return (oop)NULL;
-  } else if (type == (char)ClassLoader::PLATFORM_LOADER) {
-    return SystemDictionary::java_platform_loader();
-  } else {
-    assert (type == (char)ClassLoader::APP_LOADER, "Sanity");
-    return SystemDictionary::java_system_loader();
-  }
-}
-
-void DumpTimeSharedClassInfo::record_linking_constraint(Symbol* name, Handle loader1, Handle loader2) {
-  assert(loader1 != loader2, "sanity");
-  LogTarget(Info, class, loader, constraints) log;
-  if (_loader_constraints == NULL) {
-    _loader_constraints = new (ResourceObj::C_HEAP, mtClass) GrowableArray<DTLoaderConstraint>(4, mtClass);
-  }
-  char lt1 = get_loader_type_by(loader1());
-  char lt2 = get_loader_type_by(loader2());
-  DTLoaderConstraint lc(name, lt1, lt2);
-  for (int i = 0; i < _loader_constraints->length(); i++) {
-    DTLoaderConstraint dt = _loader_constraints->at(i);
-    if (lc.equals(dt)) {
-      if (log.is_enabled()) {
-        ResourceMark rm;
-        // Use loader[0]/loader[1] to be consistent with the logs in loaderConstraints.cpp
-        log.print("[CDS record loader constraint for class: %s constraint_name: %s loader[0]: %s loader[1]: %s already added]",
-                  _klass->external_name(), name->as_C_string(),
-                  ClassLoaderData::class_loader_data(loader1())->loader_name_and_id(),
-                  ClassLoaderData::class_loader_data(loader2())->loader_name_and_id());
-      }
-      return;
-    }
-  }
-  _loader_constraints->append(lc);
-  if (log.is_enabled()) {
-    ResourceMark rm;
-    // Use loader[0]/loader[1] to be consistent with the logs in loaderConstraints.cpp
-    log.print("[CDS record loader constraint for class: %s constraint_name: %s loader[0]: %s loader[1]: %s total %d]",
-              _klass->external_name(), name->as_C_string(),
-              ClassLoaderData::class_loader_data(loader1())->loader_name_and_id(),
-              ClassLoaderData::class_loader_data(loader2())->loader_name_and_id(),
-              _loader_constraints->length());
-  }
-}
-
 void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass,
                                                             TRAPS) {
   assert(!DumpSharedSpaces && UseSharedSpaces, "called at run time with CDS enabled only");
-  RunTimeSharedClassInfo* record = RunTimeSharedClassInfo::get_for(klass);
+  RunTimeClassInfo* record = RunTimeClassInfo::get_for(klass);
 
   int length = record->_num_verifier_constraints;
   if (length > 0) {
     for (int i = 0; i < length; i++) {
-      RunTimeSharedClassInfo::RTVerifierConstraint* vc = record->verifier_constraint_at(i);
+      RunTimeClassInfo::RTVerifierConstraint* vc = record->verifier_constraint_at(i);
       Symbol* name      = vc->name();
       Symbol* from_name = vc->from_name();
       char c            = record->verifier_constraint_flag(i);
@@ -1860,6 +978,17 @@ void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass
   }
 }
 
+static oop get_class_loader_by(char type) {
+  if (type == (char)ClassLoader::BOOT_LOADER) {
+    return (oop)NULL;
+  } else if (type == (char)ClassLoader::PLATFORM_LOADER) {
+    return SystemDictionary::java_platform_loader();
+  } else {
+    assert (type == (char)ClassLoader::APP_LOADER, "Sanity");
+    return SystemDictionary::java_system_loader();
+  }
+}
+
 // Record class loader constraints that are checked inside
 // InstanceKlass::link_class(), so that these can be checked quickly
 // at runtime without laying out the vtable/itables.
@@ -1920,7 +1049,7 @@ void SystemDictionaryShared::record_linking_constraint(Symbol* name, InstanceKla
   assert(!Thread::current()->is_VM_thread(), "must be");
 
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* info = find_or_allocate_info_for(klass);
+  DumpTimeClassInfo* info = find_or_allocate_info_for(klass);
   if (info != NULL) {
     info->record_linking_constraint(name, loader1, loader2);
   }
@@ -1936,12 +1065,12 @@ bool SystemDictionaryShared::check_linking_constraints(Thread* current, Instance
     return true;
   }
   if (klass->is_shared_platform_class() || klass->is_shared_app_class()) {
-    RunTimeSharedClassInfo* info = RunTimeSharedClassInfo::get_for(klass);
+    RunTimeClassInfo* info = RunTimeClassInfo::get_for(klass);
     assert(info != NULL, "Sanity");
     if (info->_num_loader_constraints > 0) {
       HandleMark hm(current);
       for (int i = 0; i < info->_num_loader_constraints; i++) {
-        RunTimeSharedClassInfo::RTLoaderConstraint* lc = info->loader_constraint_at(i);
+        RunTimeClassInfo::RTLoaderConstraint* lc = info->loader_constraint_at(i);
         Symbol* name = lc->constraint_name();
         Handle loader1(current, get_class_loader_by(lc->_loader_type1));
         Handle loader2(current, get_class_loader_by(lc->_loader_type2));
@@ -2032,9 +1161,9 @@ class EstimateSizeForArchive : StackObj {
     _num_unregistered_klasses = 0;
   }
 
-  bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
     if (!info.is_excluded()) {
-      size_t byte_size = RunTimeSharedClassInfo::byte_size(info._klass, info.num_verifier_constraints(), info.num_loader_constraints());
+      size_t byte_size = RunTimeClassInfo::byte_size(info._klass, info.num_verifier_constraints(), info.num_loader_constraints());
       _shared_class_info_size += align_up(byte_size, SharedSpaceObjectAlignment);
     }
     return true; // keep on iterating
@@ -2141,11 +1270,11 @@ class CopySharedClassInfoToArchive : StackObj {
                                bool is_builtin)
     : _writer(writer), _is_builtin(is_builtin), _builder(ArchiveBuilder::current()) {}
 
-  bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
     if (!info.is_excluded() && info.is_builtin() == _is_builtin) {
-      size_t byte_size = RunTimeSharedClassInfo::byte_size(info._klass, info.num_verifier_constraints(), info.num_loader_constraints());
-      RunTimeSharedClassInfo* record;
-      record = (RunTimeSharedClassInfo*)ArchiveBuilder::ro_region_alloc(byte_size);
+      size_t byte_size = RunTimeClassInfo::byte_size(info._klass, info.num_verifier_constraints(), info.num_loader_constraints());
+      RunTimeClassInfo* record;
+      record = (RunTimeClassInfo*)ArchiveBuilder::ro_region_alloc(byte_size);
       record->init(info);
 
       unsigned int hash;
@@ -2162,8 +1291,8 @@ class CopySharedClassInfoToArchive : StackObj {
         log_trace(cds,hashtables)("%s dictionary: %s", (_is_builtin ? "builtin" : "unregistered"), info._klass->external_name());
       }
 
-      // Save this for quick runtime lookup of InstanceKlass* -> RunTimeSharedClassInfo*
-      RunTimeSharedClassInfo::set_for(info._klass, record);
+      // Save this for quick runtime lookup of InstanceKlass* -> RunTimeClassInfo*
+      RunTimeClassInfo::set_for(info._klass, record);
     }
     return true; // keep on iterating
   }
@@ -2235,7 +1364,7 @@ void SystemDictionaryShared::serialize_vm_classes(SerializeClosure* soc) {
   }
 }
 
-const RunTimeSharedClassInfo*
+const RunTimeClassInfo*
 SystemDictionaryShared::find_record(RunTimeSharedDictionary* static_dict, RunTimeSharedDictionary* dynamic_dict, Symbol* name) {
   if (!UseSharedSpaces || !name->is_shared()) {
     // The names of all shared classes must also be a shared Symbol.
@@ -2243,7 +1372,7 @@ SystemDictionaryShared::find_record(RunTimeSharedDictionary* static_dict, RunTim
   }
 
   unsigned int hash = SystemDictionaryShared::hash_for_shared_dictionary_quick(name);
-  const RunTimeSharedClassInfo* record = NULL;
+  const RunTimeClassInfo* record = NULL;
   if (DynamicArchive::is_mapped()) {
     // Those regenerated holder classes are in dynamic archive
     if (name == vmSymbols::java_lang_invoke_Invokers_Holder() ||
@@ -2271,7 +1400,7 @@ SystemDictionaryShared::find_record(RunTimeSharedDictionary* static_dict, RunTim
 }
 
 InstanceKlass* SystemDictionaryShared::find_builtin_class(Symbol* name) {
-  const RunTimeSharedClassInfo* record = find_record(&_builtin_dictionary, &_dynamic_builtin_dictionary, name);
+  const RunTimeClassInfo* record = find_record(&_builtin_dictionary, &_dynamic_builtin_dictionary, name);
   if (record != NULL) {
     assert(!record->_klass->is_hidden(), "hidden class cannot be looked up by name");
     assert(check_alignment(record->_klass), "Address not aligned");
@@ -2283,7 +1412,7 @@ InstanceKlass* SystemDictionaryShared::find_builtin_class(Symbol* name) {
 
 void SystemDictionaryShared::update_shared_entry(InstanceKlass* k, int id) {
   assert(DumpSharedSpaces, "supported only when dumping");
-  DumpTimeSharedClassInfo* info = find_or_allocate_info_for(k);
+  DumpTimeClassInfo* info = find_or_allocate_info_for(k);
   info->_id = id;
 }
 
@@ -2311,7 +1440,7 @@ class SharedDictionaryPrinter : StackObj {
 public:
   SharedDictionaryPrinter(outputStream* st) : _st(st), _index(0) {}
 
-  void do_value(const RunTimeSharedClassInfo* record) {
+  void do_value(const RunTimeClassInfo* record) {
     ResourceMark rm;
     _st->print_cr("%4d: %s %s", _index++, record->_klass->external_name(),
         class_loader_name_for_shared(record->_klass));
@@ -2392,7 +1521,7 @@ void SystemDictionaryShared::print_table_statistics(outputStream* st) {
   }
 }
 
-bool SystemDictionaryShared::empty_dumptime_table() {
+bool SystemDictionaryShared::is_dumptime_table_empty() {
   if (_dumptime_table == NULL) {
     return true;
   }
@@ -2403,6 +1532,77 @@ bool SystemDictionaryShared::empty_dumptime_table() {
   return false;
 }
 
+class CloneDumpTimeClassTable: public StackObj {
+  DumpTimeSharedClassTable* _table;
+  DumpTimeSharedClassTable* _cloned_table;
+ public:
+  CloneDumpTimeClassTable(DumpTimeSharedClassTable* table, DumpTimeSharedClassTable* clone) :
+                      _table(table), _cloned_table(clone) {
+    assert(_table != NULL, "_dumptime_table is NULL");
+    assert(_cloned_table != NULL, "_cloned_table is NULL");
+  }
+  bool do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
+    if (!info.is_excluded()) {
+      bool created;
+      _cloned_table->put_if_absent(k, info.clone(), &created);
+    }
+    return true; // keep on iterating
+  }
+};
+
+class CloneDumpTimeLambdaProxyClassTable: StackObj {
+  DumpTimeLambdaProxyClassDictionary* _table;
+  DumpTimeLambdaProxyClassDictionary* _cloned_table;
+ public:
+  CloneDumpTimeLambdaProxyClassTable(DumpTimeLambdaProxyClassDictionary* table,
+                                     DumpTimeLambdaProxyClassDictionary* clone) :
+                      _table(table), _cloned_table(clone) {
+    assert(_table != NULL, "_dumptime_table is NULL");
+    assert(_cloned_table != NULL, "_cloned_table is NULL");
+  }
+
+  bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+    assert_lock_strong(DumpTimeTable_lock);
+    bool created;
+    // make copies then store in _clone_table
+    LambdaProxyClassKey keyCopy = key;
+    _cloned_table->put_if_absent(keyCopy, info.clone(), &created);
+    ++ _cloned_table->_count;
+    return true; // keep on iterating
+  }
+};
+
+void SystemDictionaryShared::clone_dumptime_tables() {
+  Arguments::assert_is_dumping_archive();
+  assert_lock_strong(DumpTimeTable_lock);
+  if (_dumptime_table != NULL) {
+    assert(_cloned_dumptime_table == NULL, "_cloned_dumptime_table must be cleaned");
+    _cloned_dumptime_table = new (ResourceObj::C_HEAP, mtClass) DumpTimeSharedClassTable;
+    CloneDumpTimeClassTable copy_classes(_dumptime_table, _cloned_dumptime_table);
+    _dumptime_table->iterate(&copy_classes);
+    _cloned_dumptime_table->update_counts();
+  }
+  if (_dumptime_lambda_proxy_class_dictionary != NULL) {
+    assert(_cloned_dumptime_lambda_proxy_class_dictionary == NULL,
+           "_cloned_dumptime_lambda_proxy_class_dictionary must be cleaned");
+    _cloned_dumptime_lambda_proxy_class_dictionary =
+                                          new (ResourceObj::C_HEAP, mtClass) DumpTimeLambdaProxyClassDictionary;
+    CloneDumpTimeLambdaProxyClassTable copy_proxy_classes(_dumptime_lambda_proxy_class_dictionary,
+                                                          _cloned_dumptime_lambda_proxy_class_dictionary);
+    _dumptime_lambda_proxy_class_dictionary->iterate(&copy_proxy_classes);
+  }
+}
+
+void SystemDictionaryShared::restore_dumptime_tables() {
+  assert_lock_strong(DumpTimeTable_lock);
+  delete _dumptime_table;
+  _dumptime_table = _cloned_dumptime_table;
+  _cloned_dumptime_table = NULL;
+  delete _dumptime_lambda_proxy_class_dictionary;
+  _dumptime_lambda_proxy_class_dictionary = _cloned_dumptime_lambda_proxy_class_dictionary;
+  _cloned_dumptime_lambda_proxy_class_dictionary = NULL;
+}
+
 #if INCLUDE_CDS_JAVA_HEAP
 
 class ArchivedMirrorPatcher {
@@ -2424,7 +1624,7 @@ class ArchivedMirrorPatcher {
     }
   }
 
-  void do_value(const RunTimeSharedClassInfo* info) {
+  void do_value(const RunTimeClassInfo* info) {
     InstanceKlass* ik = info->_klass;
     update(ik);
     update_array_klasses(ik->array_klasses());
@@ -2454,7 +1654,7 @@ void SystemDictionaryShared::update_archived_mirror_native_pointers_for(LambdaPr
 }
 
 void SystemDictionaryShared::update_archived_mirror_native_pointers() {
-  if (!HeapShared::open_archive_heap_region_mapped()) {
+  if (!HeapShared::open_regions_mapped()) {
     return;
   }
   if (MetaspaceShared::relocation_delta() == 0) {
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp
index 80f3c40c16e..ac041f92561 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.hpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp
@@ -26,6 +26,9 @@
 #define SHARE_CLASSFILE_SYSTEMDICTIONARYSHARED_HPP
 
 #include "cds/filemap.hpp"
+#include "cds/dumpTimeClassInfo.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
+#include "cds/runTimeClassInfo.hpp"
 #include "classfile/classLoaderData.hpp"
 #include "classfile/packageEntry.hpp"
 #include "classfile/systemDictionary.hpp"
@@ -106,11 +109,13 @@
 class BootstrapInfo;
 class ClassFileStream;
 class Dictionary;
-class DumpTimeSharedClassInfo;
+class DumpTimeClassInfo;
 class DumpTimeSharedClassTable;
 class LambdaProxyClassDictionary;
-class RunTimeSharedClassInfo;
+class RunTimeClassInfo;
 class RunTimeSharedDictionary;
+class DumpTimeLambdaProxyClassDictionary;
+class LambdaProxyClassKey;
 
 class SharedClassLoadingMark {
  private:
@@ -139,99 +144,39 @@ class SystemDictionaryShared: public SystemDictionary {
   };
 
 private:
-  // These _shared_xxxs arrays are used to initialize the java.lang.Package and
-  // java.security.ProtectionDomain objects associated with each shared class.
-  //
-  // See SystemDictionaryShared::init_security_info for more info.
-  static OopHandle _shared_protection_domains;
-  static OopHandle _shared_jar_urls;
-  static OopHandle _shared_jar_manifests;
+
+  static DumpTimeSharedClassTable* _dumptime_table;
+  static DumpTimeSharedClassTable* _cloned_dumptime_table;
+  static DumpTimeLambdaProxyClassDictionary* _dumptime_lambda_proxy_class_dictionary;
+  static DumpTimeLambdaProxyClassDictionary* _cloned_dumptime_lambda_proxy_class_dictionary;
+  // SystemDictionaries in the base layer static archive
+  static RunTimeSharedDictionary _builtin_dictionary;
+  static RunTimeSharedDictionary _unregistered_dictionary;
+  static LambdaProxyClassDictionary _lambda_proxy_class_dictionary;
+  // SystemDictionaries in the top layer dynamic archive
+  static RunTimeSharedDictionary _dynamic_builtin_dictionary;
+  static RunTimeSharedDictionary _dynamic_unregistered_dictionary;
+  static LambdaProxyClassDictionary _dynamic_lambda_proxy_class_dictionary;
 
   static InstanceKlass* load_shared_class_for_builtin_loader(
                                                Symbol* class_name,
                                                Handle class_loader,
                                                TRAPS);
-  static Handle get_package_name(Symbol*  class_name, TRAPS);
-
-  static PackageEntry* get_package_entry_from_class(InstanceKlass* ik, Handle class_loader);
-
-
-  // Package handling:
-  //
-  // 1. For named modules in the runtime image
-  //    BOOT classes: Reuses the existing JVM_GetSystemPackage(s) interfaces
-  //                  to get packages in named modules for shared classes.
-  //                  Package for non-shared classes in named module is also
-  //                  handled using JVM_GetSystemPackage(s).
-  //
-  //    APP  classes: VM calls ClassLoaders.AppClassLoader::definePackage(String, Module)
-  //                  to define package for shared app classes from named
-  //                  modules.
-  //
-  //    PLATFORM  classes: VM calls ClassLoaders.PlatformClassLoader::definePackage(String, Module)
-  //                  to define package for shared platform classes from named
-  //                  modules.
-  //
-  // 2. For unnamed modules
-  //    BOOT classes: Reuses the existing JVM_GetSystemPackage(s) interfaces to
-  //                  get packages for shared boot classes in unnamed modules.
-  //
-  //    APP  classes: VM calls ClassLoaders.AppClassLoader::defineOrCheckPackage()
-  //                  with with the manifest and url from archived data.
-  //
-  //    PLATFORM  classes: No package is defined.
-  //
-  // The following two define_shared_package() functions are used to define
-  // package for shared APP and PLATFORM classes.
-  static void define_shared_package(Symbol*  class_name,
-                                    Handle class_loader,
-                                    Handle manifest,
-                                    Handle url,
-                                    TRAPS);
-
-  static Handle get_shared_jar_manifest(int shared_path_index, TRAPS);
-  static Handle get_shared_jar_url(int shared_path_index, TRAPS);
-  static Handle get_protection_domain_from_classloader(Handle class_loader,
-                                                       Handle url, TRAPS);
-  static Handle get_shared_protection_domain(Handle class_loader,
-                                             int shared_path_index,
-                                             Handle url,
-                                             TRAPS);
-  static Handle get_shared_protection_domain(Handle class_loader,
-                                             ModuleEntry* mod, TRAPS);
-
-  static void atomic_set_array_index(OopHandle array, int index, oop o);
-
-  static oop shared_protection_domain(int index);
-  static void atomic_set_shared_protection_domain(int index, oop pd) {
-    atomic_set_array_index(_shared_protection_domains, index, pd);
-  }
-  static void allocate_shared_protection_domain_array(int size, TRAPS);
-  static oop shared_jar_url(int index);
-  static void atomic_set_shared_jar_url(int index, oop url) {
-    atomic_set_array_index(_shared_jar_urls, index, url);
-  }
-  static void allocate_shared_jar_url_array(int size, TRAPS);
-  static oop shared_jar_manifest(int index);
-  static void atomic_set_shared_jar_manifest(int index, oop man) {
-    atomic_set_array_index(_shared_jar_manifests, index, man);
-  }
-  static void allocate_shared_jar_manifest_array(int size, TRAPS);
   static InstanceKlass* acquire_class_for_current_thread(
                                  InstanceKlass *ik,
                                  Handle class_loader,
                                  Handle protection_domain,
                                  const ClassFileStream* cfs,
                                  TRAPS);
-  static DumpTimeSharedClassInfo* find_or_allocate_info_for(InstanceKlass* k);
-  static DumpTimeSharedClassInfo* find_or_allocate_info_for_locked(InstanceKlass* k);
+  static DumpTimeClassInfo* find_or_allocate_info_for(InstanceKlass* k);
+  static DumpTimeClassInfo* find_or_allocate_info_for_locked(InstanceKlass* k);
   static void write_dictionary(RunTimeSharedDictionary* dictionary,
                                bool is_builtin);
   static void write_lambda_proxy_class_dictionary(LambdaProxyClassDictionary* dictionary);
   static bool is_jfr_event_class(InstanceKlass *k);
   static bool is_registered_lambda_proxy_class(InstanceKlass* ik);
-  static bool warn_excluded(InstanceKlass* k, const char* reason);
   static bool check_for_exclusion_impl(InstanceKlass* k);
+  static void remove_dumptime_info(InstanceKlass* k) NOT_CDS_RETURN;
   static bool has_been_redefined(InstanceKlass* k);
 
   static bool _dump_in_progress;
@@ -245,10 +190,9 @@ class SystemDictionaryShared: public SystemDictionary {
 public:
   static bool is_hidden_lambda_proxy(InstanceKlass* ik);
   static bool is_early_klass(InstanceKlass* k);   // Was k loaded while JvmtiExport::is_early_phase()==true
-  static Handle init_security_info(Handle class_loader, InstanceKlass* ik, PackageEntry* pkg_entry, TRAPS);
   static InstanceKlass* find_builtin_class(Symbol* class_name);
 
-  static const RunTimeSharedClassInfo* find_record(RunTimeSharedDictionary* static_dict,
+  static const RunTimeClassInfo* find_record(RunTimeSharedDictionary* static_dict,
                                                    RunTimeSharedDictionary* dynamic_dict,
                                                    Symbol* name);
 
@@ -265,12 +209,12 @@ class SystemDictionaryShared: public SystemDictionary {
   // Check if sharing is supported for the class loader.
   static bool is_sharing_possible(ClassLoaderData* loader_data);
 
-  static bool add_unregistered_class(Thread* current, InstanceKlass* k);
+  static bool add_unregistered_class_for_static_archive(Thread* current, InstanceKlass* k);
   static InstanceKlass* lookup_super_for_unregistered_class(Symbol* class_name,
                                                             Symbol* super_name,  bool is_superclass);
 
   static void init_dumptime_info(InstanceKlass* k) NOT_CDS_RETURN;
-  static void remove_dumptime_info(InstanceKlass* k) NOT_CDS_RETURN;
+  static void handle_class_unloading(InstanceKlass* k) NOT_CDS_RETURN;
 
   static Dictionary* boot_loader_dictionary() {
     return ClassLoaderData::the_null_class_loader_data()->dictionary();
@@ -307,6 +251,8 @@ class SystemDictionaryShared: public SystemDictionary {
                                      Symbol* method_type,
                                      Method* member_method,
                                      Symbol* instantiated_method_type, TRAPS) NOT_CDS_RETURN;
+  static void add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey& key,
+                                                             InstanceKlass* proxy_klass) NOT_CDS_RETURN;
   static InstanceKlass* get_shared_lambda_proxy_class(InstanceKlass* caller_ik,
                                                       Symbol* invoked_name,
                                                       Symbol* invoked_type,
@@ -322,11 +268,24 @@ class SystemDictionaryShared: public SystemDictionary {
   static bool is_builtin(InstanceKlass* k) {
     return (k->shared_classpath_index() != UNREGISTERED_INDEX);
   }
+  static bool add_unregistered_class(Thread* current, InstanceKlass* k);
+
+  // For repeatable dumping, we
+  //   1. clone DumpTimeSharedClassTable, same for DumpTimeLambdaProxyClassDictionary
+  //      clone SharedClassPathTable
+  //   2. do dumping
+  //   3. restore DumpTimeSharedClassTable, DumpTimeLambdaProxyClassDictionary and SharedClassPathTable
+  //      from cloned versions.
+  static void clone_dumptime_tables();
+  static void restore_dumptime_tables();
+
   static void check_excluded_classes();
-  static bool check_for_exclusion(InstanceKlass* k, DumpTimeSharedClassInfo* info);
+  static bool check_for_exclusion(InstanceKlass* k, DumpTimeClassInfo* info);
   static void validate_before_archiving(InstanceKlass* k);
   static bool is_excluded_class(InstanceKlass* k);
   static void set_excluded(InstanceKlass* k);
+  static void set_excluded_locked(InstanceKlass* k);
+  static bool warn_excluded(InstanceKlass* k, const char* reason);
   static void dumptime_classes_do(class MetaspaceClosure* it);
   static size_t estimate_size_for_archive();
   static void write_to_archive(bool is_static_archive = true);
@@ -338,11 +297,9 @@ class SystemDictionaryShared: public SystemDictionary {
   static void print_on(outputStream* st) NOT_CDS_RETURN;
   static void print_shared_archive(outputStream* st, bool is_static = true) NOT_CDS_RETURN;
   static void print_table_statistics(outputStream* st) NOT_CDS_RETURN;
-  static bool empty_dumptime_table() NOT_CDS_RETURN_(true);
+  static bool is_dumptime_table_empty() NOT_CDS_RETURN_(true);
   static void start_dumping() NOT_CDS_RETURN;
-  static Handle create_jar_manifest(const char* man, size_t size, TRAPS) NOT_CDS_RETURN_(Handle());
   static bool is_supported_invokedynamic(BootstrapInfo* bsi) NOT_CDS_RETURN_(false);
-
   DEBUG_ONLY(static bool no_class_loading_should_happen() {return _no_class_loading_should_happen;})
 
 #ifdef ASSERT
diff --git a/src/hotspot/share/classfile/vmClasses.cpp b/src/hotspot/share/classfile/vmClasses.cpp
index 2bd1ca65cc0..1d46033176c 100644
--- a/src/hotspot/share/classfile/vmClasses.cpp
+++ b/src/hotspot/share/classfile/vmClasses.cpp
@@ -133,13 +133,13 @@ void vmClasses::resolve_all(TRAPS) {
     // ConstantPool::restore_unshareable_info (restores the archived
     // resolved_references array object).
     //
-    // HeapShared::fixup_mapped_heap_regions() fills the empty
+    // HeapShared::fixup_mapped_regions() fills the empty
     // spaces in the archived heap regions and may use
     // vmClasses::Object_klass(), so we can do this only after
     // Object_klass is resolved. See the above resolve_through()
     // call. No mirror objects are accessed/restored in the above call.
     // Mirrors are restored after java.lang.Class is loaded.
-    HeapShared::fixup_mapped_heap_regions();
+    HeapShared::fixup_mapped_regions();
 
     // Initialize the constant pool for the Object_class
     assert(Object_klass()->is_shared(), "must be");
diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp
index 65796664a46..d8806f0ace2 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.cpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.cpp
@@ -235,6 +235,8 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
   }
 
   switch (id) {
+  case vmIntrinsics::_asPrimaryType:
+  case vmIntrinsics::_asValueType:
   case vmIntrinsics::_isInstance:
   case vmIntrinsics::_isAssignableFrom:
   case vmIntrinsics::_getModifiers:
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index 3f5f3819e86..6b7382f81be 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -237,6 +237,10 @@ class methodHandle;
    do_signature(currentThread_signature,                         "()Ljava/lang/Thread;")                                \
                                                                                                                         \
   /* reflective intrinsics, for java/lang/Class, etc. */                                                                \
+  do_intrinsic(_asPrimaryType,            java_lang_Class,        asPrimaryType_name, void_class_signature,      F_R)   \
+   do_name(     asPrimaryType_name,                              "asPrimaryType")                                       \
+  do_intrinsic(_asValueType,              java_lang_Class,        asValueType_name, void_class_signature,        F_R)   \
+   do_name(     asValueType_name,                                "asValueType")                                         \
   do_intrinsic(_isAssignableFrom,         java_lang_Class,        isAssignableFrom_name, class_boolean_signature, F_RN) \
    do_name(     isAssignableFrom_name,                           "isAssignableFrom")                                    \
   do_intrinsic(_isInstance,               java_lang_Class,        isInstance_name, object_boolean_signature,     F_RN)  \
@@ -453,7 +457,7 @@ class methodHandle;
   do_class(java_util_Base64_Decoder, "java/util/Base64$Decoder")                                                        \
   do_intrinsic(_base64_decodeBlock, java_util_Base64_Decoder, decodeBlock_name, decodeBlock_signature, F_R)             \
    do_name(decodeBlock_name, "decodeBlock")                                                                             \
-   do_signature(decodeBlock_signature, "([BII[BIZ)I")                                                                   \
+   do_signature(decodeBlock_signature, "([BII[BIZZ)I")                                                                   \
                                                                                                                         \
   /* support for com.sun.crypto.provider.GHASH */                                                                       \
   do_class(com_sun_crypto_provider_ghash, "com/sun/crypto/provider/GHASH")                                              \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index e99f2bb9a29..b2f75cc54ce 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -41,6 +41,8 @@
 #include "prims/jvmtiExport.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/javaFrameAnchor.hpp"
+#include "runtime/jniHandles.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/sharedRuntime.hpp"
@@ -742,26 +744,34 @@ void DeoptimizationBlob::print_value_on(outputStream* st) const {
 // Implementation of OptimizedEntryBlob
 
 OptimizedEntryBlob::OptimizedEntryBlob(const char* name, int size, CodeBuffer* cb, intptr_t exception_handler_offset,
-                     jobject receiver, ByteSize jfa_sp_offset) :
+                                       jobject receiver, ByteSize frame_data_offset) :
   BufferBlob(name, sizeof(OptimizedEntryBlob), size, cb),
   _exception_handler_offset(exception_handler_offset),
   _receiver(receiver),
-  _jfa_sp_offset(jfa_sp_offset) {
+  _frame_data_offset(frame_data_offset) {
   CodeCache::commit(this);
 }
 
 OptimizedEntryBlob* OptimizedEntryBlob::create(const char* name, CodeBuffer* cb, intptr_t exception_handler_offset,
-                             jobject receiver, ByteSize jfa_sp_offset) {
+                                               jobject receiver, ByteSize frame_data_offset) {
   ThreadInVMfromUnknown __tiv;  // get to VM state in case we block on CodeCache_lock
 
   OptimizedEntryBlob* blob = nullptr;
   unsigned int size = CodeBlob::allocation_size(cb, sizeof(OptimizedEntryBlob));
   {
     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
-    blob = new (size) OptimizedEntryBlob(name, size, cb, exception_handler_offset, receiver, jfa_sp_offset);
+    blob = new (size) OptimizedEntryBlob(name, size, cb, exception_handler_offset, receiver, frame_data_offset);
   }
   // Track memory usage statistic after releasing CodeCache_lock
   MemoryService::track_code_cache_memory_usage();
 
   return blob;
 }
+
+void OptimizedEntryBlob::oops_do(OopClosure* f, const frame& frame) {
+  frame_data_for_frame(frame)->old_handles->oops_do(f);
+}
+
+JavaFrameAnchor* OptimizedEntryBlob::jfa_for_frame(const frame& frame) const {
+  return &frame_data_for_frame(frame)->jfa;
+}
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index fd68b0aa4cf..cf81d70c86d 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -27,6 +27,7 @@
 
 #include "asm/codeBuffer.hpp"
 #include "compiler/compilerDefinitions.hpp"
+#include "runtime/javaFrameAnchor.hpp"
 #include "runtime/frame.hpp"
 #include "runtime/handles.hpp"
 #include "utilities/align.hpp"
@@ -34,6 +35,7 @@
 
 class ImmutableOopMap;
 class ImmutableOopMapSet;
+class JNIHandleBlock;
 class OopMapSet;
 
 // CodeBlob Types
@@ -77,7 +79,7 @@ struct CodeBlobType {
 
 class CodeBlobLayout;
 class OptimizedEntryBlob; // for as_optimized_entry_blob()
-class JavaFrameAnchor; // for EntryBlob::jfa_for_frame
+class JavaFrameAnchor; // for OptimizedEntryBlob::jfa_for_frame
 
 class CodeBlob {
   friend class VMStructs;
@@ -757,29 +759,41 @@ class SafepointBlob: public SingletonBlob {
 
 //----------------------------------------------------------------------------------------------------
 
-// For optimized upcall stubs
+class ProgrammableUpcallHandler;
+
 class OptimizedEntryBlob: public BufferBlob {
+  friend class ProgrammableUpcallHandler;
  private:
   intptr_t _exception_handler_offset;
   jobject _receiver;
-  ByteSize _jfa_sp_offset;
+  ByteSize _frame_data_offset;
 
   OptimizedEntryBlob(const char* name, int size, CodeBuffer* cb, intptr_t exception_handler_offset,
-            jobject receiver, ByteSize jfa_sp_offset);
+                     jobject receiver, ByteSize frame_data_offset);
+
+  struct FrameData {
+    JavaFrameAnchor jfa;
+    JavaThread* thread;
+    JNIHandleBlock* old_handles;
+    JNIHandleBlock* new_handles;
+    bool should_detach;
+  };
 
+  // defined in frame_ARCH.cpp
+  FrameData* frame_data_for_frame(const frame& frame) const;
  public:
   // Creation
   static OptimizedEntryBlob* create(const char* name, CodeBuffer* cb,
-                           intptr_t exception_handler_offset, jobject receiver,
-                           ByteSize jfa_sp_offset);
+                                    intptr_t exception_handler_offset, jobject receiver,
+                                    ByteSize frame_data_offset);
 
   address exception_handler() { return code_begin() + _exception_handler_offset; }
   jobject receiver() { return _receiver; }
-  ByteSize jfa_sp_offset() const { return _jfa_sp_offset; }
 
-  // defined in frame_ARCH.cpp
   JavaFrameAnchor* jfa_for_frame(const frame& frame) const;
 
+  void oops_do(OopClosure* f, const frame& frame);
+
   // Typing
   virtual bool is_optimized_entry_blob() const override { return true; }
 };
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index a2e1e9dc479..71c015cad5b 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -1077,12 +1077,6 @@ void CodeCache::old_nmethods_do(MetadataClosure* f) {
   log_debug(redefine, class, nmethod)("Walked %d nmethods for mark_on_stack", length);
 }
 
-// Just marks the methods in this class as needing deoptimization
-void CodeCache::mark_for_evol_deoptimization(InstanceKlass* dependee) {
-  assert(SafepointSynchronize::is_at_safepoint(), "Can only do this at a safepoint!");
-}
-
-
 // Walk compiled methods and mark dependent methods for deoptimization.
 int CodeCache::mark_dependents_for_evol_deoptimization() {
   assert(SafepointSynchronize::is_at_safepoint(), "Can only do this at a safepoint!");
diff --git a/src/hotspot/share/code/codeCache.hpp b/src/hotspot/share/code/codeCache.hpp
index d889d006d57..53705aadcbe 100644
--- a/src/hotspot/share/code/codeCache.hpp
+++ b/src/hotspot/share/code/codeCache.hpp
@@ -271,7 +271,6 @@ class CodeCache : AllStatic {
 
   // RedefineClasses support
   // Flushing and deoptimization in case of evolution
-  static void mark_for_evol_deoptimization(InstanceKlass* dependee);
   static int  mark_dependents_for_evol_deoptimization();
   static void mark_all_nmethods_for_evol_deoptimization();
   static void flush_evol_dependents();
diff --git a/src/hotspot/share/code/dependencies.cpp b/src/hotspot/share/code/dependencies.cpp
index f956a7f369c..0ed0c27330d 100644
--- a/src/hotspot/share/code/dependencies.cpp
+++ b/src/hotspot/share/code/dependencies.cpp
@@ -51,7 +51,7 @@
 static bool must_be_in_vm() {
   Thread* thread = Thread::current();
   if (thread->is_Java_thread()) {
-    return thread->as_Java_thread()->thread_state() == _thread_in_vm;
+    return JavaThread::cast(thread)->thread_state() == _thread_in_vm;
   } else {
     return true;  // Could be VMThread or GC thread
   }
@@ -1237,9 +1237,9 @@ class ConcreteMethodFinder : public AbstractClassHierarchyWalker {
   virtual Klass* find_witness_in(KlassDepChange& changes);
   virtual Klass* find_witness_anywhere(InstanceKlass* context_type);
 
+ public:
   bool witnessed_reabstraction_in_supers(Klass* k);
 
- public:
   ConcreteMethodFinder(Method* m, Klass* participant = NULL) : AbstractClassHierarchyWalker(participant) {
     assert(m != NULL && m->is_method(), "sanity");
     _name      = m->name();
@@ -1752,20 +1752,90 @@ Klass* Dependencies::find_unique_concrete_subtype(InstanceKlass* ctxk) {
   }
 }
 
+// Try to determine whether root method in some context is concrete or not based on the information about the unique method
+// in that context. It exploits the fact that concrete root method is always inherited into the context when there's a unique method.
+// Hence, unique method holder is always a supertype of the context class when root method is concrete.
+// Examples for concrete_root_method
+//      C (C.m uniqm)
+//      |
+//      CX (ctxk) uniqm is inherited into context.
+//
+//      CX (ctxk) (CX.m uniqm) here uniqm is defined in ctxk.
+// Examples for !concrete_root_method
+//      CX (ctxk)
+//      |
+//      C (C.m uniqm) uniqm is in subtype of ctxk.
+bool Dependencies::is_concrete_root_method(Method* uniqm, InstanceKlass* ctxk) {
+  if (uniqm == NULL) {
+    return false; // match Dependencies::is_concrete_method() behavior
+  }
+  // Theoretically, the "direction" of subtype check matters here.
+  // On one hand, in case of interface context with a single implementor, uniqm can be in a superclass of the implementor which
+  // is not related to context class.
+  // On another hand, uniqm could come from an interface unrelated to the context class, but right now it is not possible:
+  // it is required that uniqm->method_holder() is the participant (uniqm->method_holder() <: ctxk), hence a default method
+  // can't be used as unique.
+  if (ctxk->is_interface()) {
+    InstanceKlass* implementor = ctxk->implementor();
+    assert(implementor != ctxk, "single implementor only"); // should have been invalidated earlier
+    ctxk = implementor;
+  }
+  InstanceKlass* holder = uniqm->method_holder();
+  assert(!holder->is_interface(), "no default methods allowed");
+  assert(ctxk->is_subclass_of(holder) || holder->is_subclass_of(ctxk), "not related");
+  return ctxk->is_subclass_of(holder);
+}
+
 // If a class (or interface) has a unique concrete method uniqm, return NULL.
 // Otherwise, return a class that contains an interfering method.
 Klass* Dependencies::check_unique_concrete_method(InstanceKlass* ctxk,
                                                   Method* uniqm,
                                                   NewKlassDepChange* changes) {
-  // Here is a missing optimization:  If uniqm->is_final(),
-  // we don't really need to search beneath it for overrides.
-  // This is probably not important, since we don't use dependencies
-  // to track final methods.  (They can't be "definalized".)
   ConcreteMethodFinder wf(uniqm, uniqm->method_holder());
   Klass* k = wf.find_witness(ctxk, changes);
-  return k;
+  if (k != NULL) {
+    return k;
+  }
+  if (!Dependencies::is_concrete_root_method(uniqm, ctxk) || changes != NULL) {
+    Klass* conck = find_witness_AME(ctxk, uniqm, changes);
+    if (conck != NULL) {
+      // Found a concrete subtype 'conck' which does not override abstract root method.
+      return conck;
+    }
+  }
+  return NULL;
+}
+
+// Search for AME.
+// There are two version of checks.
+//   1) Spot checking version(Classload time). Newly added class is checked for AME.
+//      Checks whether abstract/overpass method is inherited into/declared in newly added concrete class.
+//   2) Compile time analysis for abstract/overpass(abstract klass) root_m. The non uniqm subtrees are checked for concrete classes.
+Klass* Dependencies::find_witness_AME(InstanceKlass* ctxk, Method* m, KlassDepChange* changes) {
+  if (m != NULL) {
+    if (changes != NULL) {
+      // Spot checking version.
+      ConcreteMethodFinder wf(m);
+      Klass* new_type = changes->as_new_klass_change()->new_type();
+      if (wf.witnessed_reabstraction_in_supers(new_type)) {
+        return new_type;
+      }
+    } else {
+      // Note: It is required that uniqm->method_holder() is the participant (see ClassHierarchyWalker::found_method()).
+      ConcreteSubtypeFinder wf(m->method_holder());
+      Klass* conck = wf.find_witness(ctxk);
+      if (conck != NULL) {
+        Method* cm = InstanceKlass::cast(conck)->find_instance_method(m->name(), m->signature(), Klass::PrivateLookupMode::skip);
+        if (!Dependencies::is_concrete_method(cm, conck)) {
+          return conck;
+        }
+      }
+    }
+  }
+  return NULL;
 }
 
+
 // Find the set of all non-abstract methods under ctxk that match m.
 // (The method m must be defined or inherited in ctxk.)
 // Include m itself in the set, unless it is abstract.
@@ -1787,6 +1857,9 @@ Method* Dependencies::find_unique_concrete_method(InstanceKlass* ctxk, Method* m
   if (participant != NULL) {
     (*participant) = wf.participant(0);
   }
+  if (!Dependencies::is_concrete_method(fm, NULL)) {
+    fm = NULL; // ignore abstract methods
+  }
   if (Dependencies::is_concrete_method(m, ctxk)) {
     if (fm == NULL) {
       // It turns out that m was always the only implementation.
@@ -1796,7 +1869,11 @@ Method* Dependencies::find_unique_concrete_method(InstanceKlass* ctxk, Method* m
       // (This can happen if m is inherited into ctxk and fm overrides it.)