diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp index e7df38ff221a9..15bc7ddc67d60 100644 --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -4357,7 +4357,7 @@ bool MatchRule::is_vector() const { "RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector", "LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked", "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert", - "VectorRearrange","VectorLoadShuffle", "VectorLoadConst", + "VectorRearrange", "VectorLoadShuffle", "VectorLoadConst", "VectorCastB2X", "VectorCastS2X", "VectorCastI2X", "VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F", "VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X", diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 4b772c171d5a6..2ce2a7577d1a0 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -1007,6 +1007,15 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ do_name(vector_shuffle_to_vector_name, "shuffleToVector") \ \ + do_intrinsic(_VectorWrapShuffleIndexes, jdk_internal_vm_vector_VectorSupport, vector_wrap_shuffle_indexes_name, \ + vector_wrap_shuffle_indexes_sig, F_S) \ + do_signature(vector_wrap_shuffle_indexes_sig, "(Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \ + "ILjdk/internal/vm/vector/VectorSupport$WrapShuffleIndexesOperation;)" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \ + do_name(vector_wrap_shuffle_indexes_name, "wrapShuffleIndexes") \ + \ do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \ do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \ "Ljava/lang/Class;" \ @@ -1128,6 +1137,18 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ do_name(vector_rearrange_name, "rearrangeOp") \ \ + do_intrinsic(_VectorSelectFrom, jdk_internal_vm_vector_VectorSupport, vector_select_from_name, vector_select_from_sig, F_S) \ + do_signature(vector_select_from_sig, "(Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "Ljava/lang/Class;" \ + "I" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorSelectFromOp;)" \ + "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ + do_name(vector_select_from_name, "selectFromOp") \ + \ do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S) \ do_signature(vector_extract_sig, "(Ljava/lang/Class;" \ "Ljava/lang/Class;" \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 2f087858efd48..61b76c721cd19 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -810,6 +810,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_VectorFromBitsCoerced: case vmIntrinsics::_VectorShuffleIota: case vmIntrinsics::_VectorShuffleToVector: + case vmIntrinsics::_VectorWrapShuffleIndexes: case vmIntrinsics::_VectorLoadOp: case vmIntrinsics::_VectorLoadMaskedOp: case vmIntrinsics::_VectorStoreOp: @@ -820,6 +821,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) { case vmIntrinsics::_VectorTest: case vmIntrinsics::_VectorBlend: case vmIntrinsics::_VectorRearrange: + case vmIntrinsics::_VectorSelectFrom: case vmIntrinsics::_VectorCompare: case vmIntrinsics::_VectorBroadcastInt: case vmIntrinsics::_VectorConvert: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index c95a450272989..b96c531f51fbd 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -716,6 +716,8 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_mask_operation(); case vmIntrinsics::_VectorShuffleToVector: return inline_vector_shuffle_to_vector(); + case vmIntrinsics::_VectorWrapShuffleIndexes: + return inline_vector_wrap_shuffle_indexes(); case vmIntrinsics::_VectorLoadOp: return inline_vector_mem_operation(/*is_store=*/false); case vmIntrinsics::_VectorLoadMaskedOp: @@ -736,6 +738,8 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_blend(); case vmIntrinsics::_VectorRearrange: return inline_vector_rearrange(); + case vmIntrinsics::_VectorSelectFrom: + return inline_vector_select_from(); case vmIntrinsics::_VectorCompare: return inline_vector_compare(); case vmIntrinsics::_VectorBroadcastInt: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index dd74734802f65..10375fc23f650 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -353,6 +353,7 @@ class LibraryCallKit : public GraphKit { bool inline_vector_nary_operation(int n); bool inline_vector_frombits_coerced(); bool inline_vector_shuffle_to_vector(); + bool inline_vector_wrap_shuffle_indexes(); bool inline_vector_shuffle_iota(); Node* partially_wrap_indexes(Node* index_vec, int num_elem, BasicType type_bt); bool inline_vector_mask_operation(); @@ -363,6 +364,7 @@ class LibraryCallKit : public GraphKit { bool inline_vector_test(); bool inline_vector_blend(); bool inline_vector_rearrange(); + bool inline_vector_select_from(); bool inline_vector_compare(); bool inline_vector_broadcast_int(); bool inline_vector_convert(); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index cfcd903e79d95..3753619822938 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -757,6 +757,64 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() { return true; } +// public static +// > +// SH wrapShuffleIndexes(Class eClass, Class shClass, SH sh, int length, +// ShuffleWrapIndexesOperation defaultImpl) +bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() { + const TypeInstPtr* elem_klass = gvn().type(argument(0))->isa_instptr(); + const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr(); + Node* shuffle = argument(2); + const TypeInt* vlen = gvn().type(argument(3))->isa_int(); + + if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr || + !vlen->is_con() || shuffle_klass->const_oop() == nullptr) { + // not enough info for intrinsification + return false; + } + + if (!is_klass_initialized(shuffle_klass)) { + log_if_needed(" ** klass argument not initialized"); + return false; + } + + int num_elem = vlen->get_con(); + if ((num_elem < 4) || !is_power_of_2(num_elem)) { + log_if_needed(" ** vlen < 4 or not power of two=%d", num_elem); + return false; + } + + // Shuffles use byte array based backing storage + BasicType shuffle_bt = T_BYTE; + if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) || + !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) { + log_if_needed(" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s", + num_elem, type2name(shuffle_bt)); + return false; + } + + ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); + + // Unbox shuffle with true flag to indicate its load shuffle to vector + // shuffle is a byte array + Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true); + + const TypeVect* vt = TypeVect::make(shuffle_bt, num_elem); + const Type* shuffle_type_bt = Type::get_const_basic_type(shuffle_bt); + Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1)); + Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_type_bt)); + // Wrap the indices greater than lane count. + Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt)); + + // Wrap it up in VectorBox to keep object type information. + res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem); + set_result(res); + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt)))); + return true; +} + // public static // , @@ -2044,6 +2102,150 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, return addr; } +// public static +// , +// M extends VectorMask, +// E> +// V selectFromOp(Class vClass, Class mClass, Class eClass, +// int length, V v1, V v2, M m, +// VectorSelectFromOp defaultImpl) +bool LibraryCallKit::inline_vector_select_from() { + const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr(); + const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->isa_int(); + + if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + vector_klass->const_oop() == nullptr || + elem_klass->const_oop() == nullptr || + !vlen->is_con()) { + log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s", + NodeClassNames[argument(0)->Opcode()], + NodeClassNames[argument(2)->Opcode()], + NodeClassNames[argument(3)->Opcode()]); + return false; // not enough info for intrinsification + } + if (!is_klass_initialized(vector_klass)) { + log_if_needed(" ** klass argument not initialized"); + return false; + } + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + if (!elem_type->is_primitive_type()) { + log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type()); + return false; // should be primitive type + } + BasicType elem_bt = elem_type->basic_type(); + int num_elem = vlen->get_con(); + if (!is_power_of_2(num_elem)) { + log_if_needed(" ** vlen not power of two=%d", num_elem); + return false; + } + + int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt + if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)|| + !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) || + !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) || + !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) { + log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no", + num_elem, type2name(elem_bt)); + return false; // not supported + } + + bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR; + bool use_predicate = is_masked_op; + if (is_masked_op && + (mask_klass == nullptr || + mask_klass->const_oop() == nullptr || + !is_klass_initialized(mask_klass))) { + log_if_needed(" ** mask_klass argument not initialized"); + return false; // not supported + } + VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed); + if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) { + use_predicate = false; + if(!is_masked_op || + (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) || + !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) || + !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) { + log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d", + num_elem, type2name(elem_bt), is_masked_op); + return false; // not supported + } + } + ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); + + // v1 is the index vector + Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); + // v2 is the vector being rearranged + Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); + + if (v1 == nullptr) { + log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]); + return false; // operand unboxing failed + } + + if (v2 == nullptr) { + log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]); + return false; // operand unboxing failed + } + + Node* mask = nullptr; + if (is_masked_op) { + ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); + mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem); + if (mask == nullptr) { + log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]); + return false; + } + } + + // cast index vector from elem_bt vector to byte vector + const Type * byte_bt = Type::get_const_basic_type(T_BYTE); + const TypeVect * byte_vt = TypeVect::make(byte_bt, num_elem); + Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem)); + + // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length + // this is a simple AND operation as we come here only for power of two vector length + Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1)); + Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, byte_bt)); + byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt)); + + // load the shuffle to use in rearrange + const TypeVect * shuffle_vt = TypeVect::make(elem_bt, num_elem); + Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt)); + + // and finally rearrange + Node* rearrange = new VectorRearrangeNode(v2, load_shuffle); + if (is_masked_op) { + if (use_predicate) { + // masked rearrange is supported so use that directly + rearrange->add_req(mask); + rearrange->add_flag(Node::Flag_is_predicated_vector); + } else { + // masked rearrange is not supported so emulate usig blend + const TypeVect* vt = v1->bottom_type()->is_vect(); + rearrange = gvn().transform(rearrange); + + // create a zero vector with each lane element set as zero + Node* zero = gvn().makecon(Type::get_zero_type(elem_bt)); + Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt))); + + // For each lane for which mask is set, blend in the rearranged lane into zero vector + rearrange = new VectorBlendNode(zerovec, rearrange, mask); + } + } + rearrange = gvn().transform(rearrange); + + // box the result + Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem); + set_result(box); + + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) { assert(UseVectorStubs, "sanity"); assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id"); diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index ccfa006b102c2..5c4040d912804 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -263,6 +263,20 @@ V shuffleToVector(Class> vClass, Class eClass, Class> { + SH apply(SH sh); + } + + @IntrinsicCandidate + public static + > + SH wrapShuffleIndexes(Class eClass, Class shClass, SH sh, int length, + WrapShuffleIndexesOperation defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(sh); + } + /* ============================================================================ */ public interface IndexOperation, S extends VectorSpecies> { @@ -605,6 +619,23 @@ V rearrangeOp(Class vClass, Class shClass, Class mClass, Cla return defaultImpl.apply(v, sh, m); } + public interface VectorSelectFromOp, + M extends VectorMask> { + V apply(V v1, V v2, M m); + } + + @IntrinsicCandidate + public static + , + M extends VectorMask, + E> + V selectFromOp(Class vClass, Class mClass, Class eClass, + int length, V v1, V v2, M m, + VectorSelectFromOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(v1, v2, m); + } + /* ============================================================================ */ public interface VectorBlendOp, diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index d648fd86484e0..fc39cb6adaccb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -133,7 +133,7 @@ public final VectorShuffle checkIndexes() { } @ForceInline - public final VectorShuffle wrapIndexes() { + public final VectorShuffle wrapIndexesTemplate() { Vector shufvec = this.toVector(); VectorMask vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero()); if (vecmask.anyTrue()) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 02a389d08b2e9..0bc25958a7617 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -503,7 +503,7 @@ public Byte128Vector selectFrom(Vector v, VectorMask m) { return (Byte128Vector) super.selectFromTemplate((Byte128Vector) v, - (Byte128Mask) m); // specialize + Byte128Mask.class, (Byte128Mask) m); // specialize } @@ -860,6 +860,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Byte128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Byte128Shuffle.class, this, VLENGTH, + (s) -> ((Byte128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Byte128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index 4e035b13b5e04..639646aa77ad5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -503,7 +503,7 @@ public Byte256Vector selectFrom(Vector v, VectorMask m) { return (Byte256Vector) super.selectFromTemplate((Byte256Vector) v, - (Byte256Mask) m); // specialize + Byte256Mask.class, (Byte256Mask) m); // specialize } @@ -892,6 +892,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Byte256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Byte256Shuffle.class, this, VLENGTH, + (s) -> ((Byte256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Byte256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index 15dd06cdccc00..2d8151f6800f3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -503,7 +503,7 @@ public Byte512Vector selectFrom(Vector v, VectorMask m) { return (Byte512Vector) super.selectFromTemplate((Byte512Vector) v, - (Byte512Mask) m); // specialize + Byte512Mask.class, (Byte512Mask) m); // specialize } @@ -956,6 +956,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Byte512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Byte512Shuffle.class, this, VLENGTH, + (s) -> ((Byte512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Byte512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index 6ac9d7a8918b1..bc8ed7d704d96 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -503,7 +503,7 @@ public Byte64Vector selectFrom(Vector v, VectorMask m) { return (Byte64Vector) super.selectFromTemplate((Byte64Vector) v, - (Byte64Mask) m); // specialize + Byte64Mask.class, (Byte64Mask) m); // specialize } @@ -844,6 +844,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Byte64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Byte64Shuffle.class, this, VLENGTH, + (s) -> ((Byte64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Byte64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index ed6e15ca293b2..597afd8d165b3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -503,7 +503,7 @@ public ByteMaxVector selectFrom(Vector v, VectorMask m) { return (ByteMaxVector) super.selectFromTemplate((ByteMaxVector) v, - (ByteMaxMask) m); // specialize + ByteMaxMask.class, (ByteMaxMask) m); // specialize } @@ -830,6 +830,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public ByteMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, ByteMaxShuffle.class, this, VLENGTH, + (s) -> ((ByteMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public ByteMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index 8fae8d71b042a..a23bbc7f70957 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -2393,17 +2393,18 @@ ByteVector sliceTemplate(int origin) { */ @Override public abstract - ByteVector rearrange(VectorShuffle m); + ByteVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > ByteVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, byte.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2428,17 +2429,14 @@ ByteVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, byte.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2551,7 +2549,10 @@ byte.class, length(), this, m, /*package-private*/ @ForceInline final ByteVector selectFromTemplate(ByteVector v) { - return v.rearrange(this.toShuffle()); + return (ByteVector)VectorSupport.selectFromOp(getClass(), null, byte.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2563,9 +2564,15 @@ final ByteVector selectFromTemplate(ByteVector v) { /*package-private*/ @ForceInline - final ByteVector selectFromTemplate(ByteVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + ByteVector selectFromTemplate(ByteVector v, + Class masktype, M m) { + m.check(masktype, this); + return (ByteVector)VectorSupport.selectFromOp(getClass(), masktype, byte.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index f2a077604f7fc..00840026fff9f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -490,7 +490,7 @@ public Double128Vector selectFrom(Vector v, VectorMask m) { return (Double128Vector) super.selectFromTemplate((Double128Vector) v, - (Double128Mask) m); // specialize + Double128Mask.class, (Double128Mask) m); // specialize } @@ -821,6 +821,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Double128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Double128Shuffle.class, this, VLENGTH, + (s) -> ((Double128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Double128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index da9dd421b17c2..4b42deba73841 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -490,7 +490,7 @@ public Double256Vector selectFrom(Vector v, VectorMask m) { return (Double256Vector) super.selectFromTemplate((Double256Vector) v, - (Double256Mask) m); // specialize + Double256Mask.class, (Double256Mask) m); // specialize } @@ -825,6 +825,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Double256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Double256Shuffle.class, this, VLENGTH, + (s) -> ((Double256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Double256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index d23f09e774b4a..c188f990c335f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -490,7 +490,7 @@ public Double512Vector selectFrom(Vector v, VectorMask m) { return (Double512Vector) super.selectFromTemplate((Double512Vector) v, - (Double512Mask) m); // specialize + Double512Mask.class, (Double512Mask) m); // specialize } @@ -833,6 +833,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Double512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Double512Shuffle.class, this, VLENGTH, + (s) -> ((Double512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Double512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 19bd640f97844..032fa1ac277d9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -490,7 +490,7 @@ public Double64Vector selectFrom(Vector v, VectorMask m) { return (Double64Vector) super.selectFromTemplate((Double64Vector) v, - (Double64Mask) m); // specialize + Double64Mask.class, (Double64Mask) m); // specialize } @@ -819,6 +819,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Double64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Double64Shuffle.class, this, VLENGTH, + (s) -> ((Double64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Double64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 73f6f2ece5d12..7251ec82aa63c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -490,7 +490,7 @@ public DoubleMaxVector selectFrom(Vector v, VectorMask m) { return (DoubleMaxVector) super.selectFromTemplate((DoubleMaxVector) v, - (DoubleMaxMask) m); // specialize + DoubleMaxMask.class, (DoubleMaxMask) m); // specialize } @@ -818,6 +818,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public DoubleMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, DoubleMaxShuffle.class, this, VLENGTH, + (s) -> ((DoubleMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public DoubleMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 59e6719573209..6cc12048d4632 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -2235,17 +2235,18 @@ DoubleVector sliceTemplate(int origin) { */ @Override public abstract - DoubleVector rearrange(VectorShuffle m); + DoubleVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > DoubleVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, double.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2270,17 +2271,14 @@ DoubleVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, double.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2393,7 +2391,10 @@ DoubleVector expandTemplate(Class masktype, M m) { /*package-private*/ @ForceInline final DoubleVector selectFromTemplate(DoubleVector v) { - return v.rearrange(this.toShuffle()); + return (DoubleVector)VectorSupport.selectFromOp(getClass(), null, double.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2405,9 +2406,15 @@ final DoubleVector selectFromTemplate(DoubleVector v) { /*package-private*/ @ForceInline - final DoubleVector selectFromTemplate(DoubleVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + DoubleVector selectFromTemplate(DoubleVector v, + Class masktype, M m) { + m.check(masktype, this); + return (DoubleVector)VectorSupport.selectFromOp(getClass(), masktype, double.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index ae47beca5de53..2e016725f812b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -490,7 +490,7 @@ public Float128Vector selectFrom(Vector v, VectorMask m) { return (Float128Vector) super.selectFromTemplate((Float128Vector) v, - (Float128Mask) m); // specialize + Float128Mask.class, (Float128Mask) m); // specialize } @@ -825,6 +825,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Float128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Float128Shuffle.class, this, VLENGTH, + (s) -> ((Float128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Float128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index d5c0506a54257..00e6083588339 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -490,7 +490,7 @@ public Float256Vector selectFrom(Vector v, VectorMask m) { return (Float256Vector) super.selectFromTemplate((Float256Vector) v, - (Float256Mask) m); // specialize + Float256Mask.class, (Float256Mask) m); // specialize } @@ -833,6 +833,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Float256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Float256Shuffle.class, this, VLENGTH, + (s) -> ((Float256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Float256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 536f7db69465e..1f2a792c52c2e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -490,7 +490,7 @@ public Float512Vector selectFrom(Vector v, VectorMask m) { return (Float512Vector) super.selectFromTemplate((Float512Vector) v, - (Float512Mask) m); // specialize + Float512Mask.class, (Float512Mask) m); // specialize } @@ -849,6 +849,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Float512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Float512Shuffle.class, this, VLENGTH, + (s) -> ((Float512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Float512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index 849062c6cb84b..6c913ce84a9ba 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -490,7 +490,7 @@ public Float64Vector selectFrom(Vector v, VectorMask m) { return (Float64Vector) super.selectFromTemplate((Float64Vector) v, - (Float64Mask) m); // specialize + Float64Mask.class, (Float64Mask) m); // specialize } @@ -821,6 +821,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Float64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Float64Shuffle.class, this, VLENGTH, + (s) -> ((Float64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Float64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index b14797f6788d4..b9a0a93f91253 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -490,7 +490,7 @@ public FloatMaxVector selectFrom(Vector v, VectorMask m) { return (FloatMaxVector) super.selectFromTemplate((FloatMaxVector) v, - (FloatMaxMask) m); // specialize + FloatMaxMask.class, (FloatMaxMask) m); // specialize } @@ -818,6 +818,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public FloatMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, FloatMaxShuffle.class, this, VLENGTH, + (s) -> ((FloatMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public FloatMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 45427817e3dac..b962dc55ce351 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -2247,17 +2247,18 @@ FloatVector sliceTemplate(int origin) { */ @Override public abstract - FloatVector rearrange(VectorShuffle m); + FloatVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > FloatVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, float.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2282,17 +2283,14 @@ FloatVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, float.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2405,7 +2403,10 @@ float.class, length(), this, m, /*package-private*/ @ForceInline final FloatVector selectFromTemplate(FloatVector v) { - return v.rearrange(this.toShuffle()); + return (FloatVector)VectorSupport.selectFromOp(getClass(), null, float.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2417,9 +2418,15 @@ final FloatVector selectFromTemplate(FloatVector v) { /*package-private*/ @ForceInline - final FloatVector selectFromTemplate(FloatVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + FloatVector selectFromTemplate(FloatVector v, + Class masktype, M m) { + m.check(masktype, this); + return (FloatVector)VectorSupport.selectFromOp(getClass(), masktype, float.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index cd652941fb366..f7135e19cb6e2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -503,7 +503,7 @@ public Int128Vector selectFrom(Vector v, VectorMask m) { return (Int128Vector) super.selectFromTemplate((Int128Vector) v, - (Int128Mask) m); // specialize + Int128Mask.class, (Int128Mask) m); // specialize } @@ -836,6 +836,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Int128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Int128Shuffle.class, this, VLENGTH, + (s) -> ((Int128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Int128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index b76a1035561f6..474ff974b3169 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -503,7 +503,7 @@ public Int256Vector selectFrom(Vector v, VectorMask m) { return (Int256Vector) super.selectFromTemplate((Int256Vector) v, - (Int256Mask) m); // specialize + Int256Mask.class, (Int256Mask) m); // specialize } @@ -844,6 +844,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Int256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Int256Shuffle.class, this, VLENGTH, + (s) -> ((Int256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Int256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 3a42c6611445b..9fec8c0c99f13 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -503,7 +503,7 @@ public Int512Vector selectFrom(Vector v, VectorMask m) { return (Int512Vector) super.selectFromTemplate((Int512Vector) v, - (Int512Mask) m); // specialize + Int512Mask.class, (Int512Mask) m); // specialize } @@ -860,6 +860,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Int512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Int512Shuffle.class, this, VLENGTH, + (s) -> ((Int512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Int512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 4181e6b4ea3e7..3b3c0723ee1a3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -503,7 +503,7 @@ public Int64Vector selectFrom(Vector v, VectorMask m) { return (Int64Vector) super.selectFromTemplate((Int64Vector) v, - (Int64Mask) m); // specialize + Int64Mask.class, (Int64Mask) m); // specialize } @@ -832,6 +832,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Int64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Int64Shuffle.class, this, VLENGTH, + (s) -> ((Int64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Int64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index 785022fcd65f0..5738cb7a4bc94 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -503,7 +503,7 @@ public IntMaxVector selectFrom(Vector v, VectorMask m) { return (IntMaxVector) super.selectFromTemplate((IntMaxVector) v, - (IntMaxMask) m); // specialize + IntMaxMask.class, (IntMaxMask) m); // specialize } @@ -841,6 +841,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public IntMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, IntMaxShuffle.class, this, VLENGTH, + (s) -> ((IntMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public IntMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 3317e25e73e73..16b5ceecba35e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -2378,17 +2378,18 @@ IntVector sliceTemplate(int origin) { */ @Override public abstract - IntVector rearrange(VectorShuffle m); + IntVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > IntVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, int.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2413,17 +2414,14 @@ IntVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, int.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2536,7 +2534,10 @@ int.class, length(), this, m, /*package-private*/ @ForceInline final IntVector selectFromTemplate(IntVector v) { - return v.rearrange(this.toShuffle()); + return (IntVector)VectorSupport.selectFromOp(getClass(), null, int.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2548,9 +2549,15 @@ final IntVector selectFromTemplate(IntVector v) { /*package-private*/ @ForceInline - final IntVector selectFromTemplate(IntVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + IntVector selectFromTemplate(IntVector v, + Class masktype, M m) { + m.check(masktype, this); + return (IntVector)VectorSupport.selectFromOp(getClass(), masktype, int.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 302c71bc13b10..567789627c6cb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -493,7 +493,7 @@ public Long128Vector selectFrom(Vector v, VectorMask m) { return (Long128Vector) super.selectFromTemplate((Long128Vector) v, - (Long128Mask) m); // specialize + Long128Mask.class, (Long128Mask) m); // specialize } @@ -822,6 +822,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Long128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Long128Shuffle.class, this, VLENGTH, + (s) -> ((Long128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Long128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 04c51c377e1de..5ef0f121464f6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -493,7 +493,7 @@ public Long256Vector selectFrom(Vector v, VectorMask m) { return (Long256Vector) super.selectFromTemplate((Long256Vector) v, - (Long256Mask) m); // specialize + Long256Mask.class, (Long256Mask) m); // specialize } @@ -826,6 +826,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Long256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Long256Shuffle.class, this, VLENGTH, + (s) -> ((Long256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Long256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index fbcd57400d579..acdb471609f22 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -493,7 +493,7 @@ public Long512Vector selectFrom(Vector v, VectorMask m) { return (Long512Vector) super.selectFromTemplate((Long512Vector) v, - (Long512Mask) m); // specialize + Long512Mask.class, (Long512Mask) m); // specialize } @@ -834,6 +834,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Long512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Long512Shuffle.class, this, VLENGTH, + (s) -> ((Long512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Long512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index 3b2e77f1a54ac..627f7437367cb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -493,7 +493,7 @@ public Long64Vector selectFrom(Vector v, VectorMask m) { return (Long64Vector) super.selectFromTemplate((Long64Vector) v, - (Long64Mask) m); // specialize + Long64Mask.class, (Long64Mask) m); // specialize } @@ -820,6 +820,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Long64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Long64Shuffle.class, this, VLENGTH, + (s) -> ((Long64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Long64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index cc3641be3d62f..aec3bb89fcd02 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -493,7 +493,7 @@ public LongMaxVector selectFrom(Vector v, VectorMask m) { return (LongMaxVector) super.selectFromTemplate((LongMaxVector) v, - (LongMaxMask) m); // specialize + LongMaxMask.class, (LongMaxMask) m); // specialize } @@ -820,6 +820,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public LongMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, LongMaxShuffle.class, this, VLENGTH, + (s) -> ((LongMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public LongMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 9dd3f2eb13626..15ac2bc7b7f6c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -2244,17 +2244,18 @@ LongVector sliceTemplate(int origin) { */ @Override public abstract - LongVector rearrange(VectorShuffle m); + LongVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > LongVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, long.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2279,17 +2280,14 @@ LongVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, long.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2402,7 +2400,10 @@ long.class, length(), this, m, /*package-private*/ @ForceInline final LongVector selectFromTemplate(LongVector v) { - return v.rearrange(this.toShuffle()); + return (LongVector)VectorSupport.selectFromOp(getClass(), null, long.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2414,9 +2415,15 @@ final LongVector selectFromTemplate(LongVector v) { /*package-private*/ @ForceInline - final LongVector selectFromTemplate(LongVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + LongVector selectFromTemplate(LongVector v, + Class masktype, M m) { + m.check(masktype, this); + return (LongVector)VectorSupport.selectFromOp(getClass(), masktype, long.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 7703df9a59a3d..fe34886512a13 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -503,7 +503,7 @@ public Short128Vector selectFrom(Vector v, VectorMask m) { return (Short128Vector) super.selectFromTemplate((Short128Vector) v, - (Short128Mask) m); // specialize + Short128Mask.class, (Short128Mask) m); // specialize } @@ -844,6 +844,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Short128Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Short128Shuffle.class, this, VLENGTH, + (s) -> ((Short128Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Short128Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index cf84593019523..243e24ad26bef 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -503,7 +503,7 @@ public Short256Vector selectFrom(Vector v, VectorMask m) { return (Short256Vector) super.selectFromTemplate((Short256Vector) v, - (Short256Mask) m); // specialize + Short256Mask.class, (Short256Mask) m); // specialize } @@ -860,6 +860,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Short256Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Short256Shuffle.class, this, VLENGTH, + (s) -> ((Short256Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Short256Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index 67a5073df0525..4114783608960 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -503,7 +503,7 @@ public Short512Vector selectFrom(Vector v, VectorMask m) { return (Short512Vector) super.selectFromTemplate((Short512Vector) v, - (Short512Mask) m); // specialize + Short512Mask.class, (Short512Mask) m); // specialize } @@ -892,6 +892,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Short512Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Short512Shuffle.class, this, VLENGTH, + (s) -> ((Short512Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Short512Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index 263ee10d907ba..d80d4c4e2ec52 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -503,7 +503,7 @@ public Short64Vector selectFrom(Vector v, VectorMask m) { return (Short64Vector) super.selectFromTemplate((Short64Vector) v, - (Short64Mask) m); // specialize + Short64Mask.class, (Short64Mask) m); // specialize } @@ -836,6 +836,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public Short64Shuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, Short64Shuffle.class, this, VLENGTH, + (s) -> ((Short64Shuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public Short64Shuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index 07a2caebef0ae..799483a667590 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -503,7 +503,7 @@ public ShortMaxVector selectFrom(Vector v, VectorMask m) { return (ShortMaxVector) super.selectFromTemplate((ShortMaxVector) v, - (ShortMaxMask) m); // specialize + ShortMaxMask.class, (ShortMaxMask) m); // specialize } @@ -830,6 +830,13 @@ public VectorShuffle cast(VectorSpecies s) { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public ShortMaxShuffle wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, ShortMaxShuffle.class, this, VLENGTH, + (s) -> ((ShortMaxShuffle)(((AbstractShuffle)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public ShortMaxShuffle rearrange(VectorShuffle shuffle) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index ba21e8a9e9563..fb0512fd5b9ea 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -2394,17 +2394,18 @@ ShortVector sliceTemplate(int origin) { */ @Override public abstract - ShortVector rearrange(VectorShuffle m); + ShortVector rearrange(VectorShuffle shuffle); /*package-private*/ @ForceInline final > ShortVector rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, short.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2429,17 +2430,14 @@ ShortVector rearrangeTemplate(Class shuffletype, M m) { m.check(masktype, this); - VectorMask valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, short.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2552,7 +2550,10 @@ short.class, length(), this, m, /*package-private*/ @ForceInline final ShortVector selectFromTemplate(ShortVector v) { - return v.rearrange(this.toShuffle()); + return (ShortVector)VectorSupport.selectFromOp(getClass(), null, short.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2564,9 +2565,15 @@ final ShortVector selectFromTemplate(ShortVector v) { /*package-private*/ @ForceInline - final ShortVector selectFromTemplate(ShortVector v, - AbstractMask m) { - return v.rearrange(this.toShuffle(), m); + final + > + ShortVector selectFromTemplate(ShortVector v, + Class masktype, M m) { + m.check(masktype, this); + return (ShortVector)VectorSupport.selectFromOp(getClass(), masktype, short.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Vector.java index d34ac79e7c32c..fda073f686389 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Vector.java @@ -2614,17 +2614,14 @@ public abstract VectorMask compare(VectorOperators.Comparison op, * elements of this vector. * * For each lane {@code N} of the shuffle, and for each lane - * source index {@code I=s.laneSource(N)} in the shuffle, + * source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle, * the output lane {@code N} obtains the value from * the input vector at lane {@code I}. * * @param s the shuffle controlling lane index selection * @return the rearrangement of the lane elements of this vector - * @throws IndexOutOfBoundsException if there are any exceptional - * source indexes in the shuffle * @see #rearrange(VectorShuffle,VectorMask) * @see #rearrange(VectorShuffle,Vector) - * @see VectorShuffle#laneIsValid() */ public abstract Vector rearrange(VectorShuffle s); @@ -2636,27 +2633,22 @@ public abstract VectorMask compare(VectorOperators.Comparison op, * elements of this vector. * * For each lane {@code N} of the shuffle, and for each lane - * source index {@code I=s.laneSource(N)} in the shuffle, + * source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle, * the output lane {@code N} obtains the value from * the input vector at lane {@code I} if the mask is set. * Otherwise the output lane {@code N} is set to zero. * *

This method returns the value of this pseudocode: *

{@code
-     * Vector r = this.rearrange(s.wrapIndexes());
-     * VectorMask valid = s.laneIsValid();
-     * if (m.andNot(valid).anyTrue()) throw ...;
+     * Vector r = this.rearrange(s);
      * return broadcast(0).blend(r, m);
      * }
* * @param s the shuffle controlling lane index selection * @param m the mask controlling application of the shuffle * @return the rearrangement of the lane elements of this vector - * @throws IndexOutOfBoundsException if there are any exceptional - * source indexes in the shuffle where the mask is set * @see #rearrange(VectorShuffle) * @see #rearrange(VectorShuffle,Vector) - * @see VectorShuffle#laneIsValid() */ public abstract Vector rearrange(VectorShuffle s, VectorMask m); @@ -2747,7 +2739,7 @@ public abstract VectorMask compare(VectorOperators.Comparison op, * this vector. * * For each lane {@code N} of this vector, and for each lane - * value {@code I=this.lane(N)} in this vector, + * value {@code I=wrapIndex(this.lane(N))} in this vector, * the output lane {@code N} obtains the value from * the argument vector at lane {@code I}. * @@ -2760,8 +2752,6 @@ public abstract VectorMask compare(VectorOperators.Comparison op, * * @param v the vector supplying the result values * @return the rearrangement of the lane elements of {@code v} - * @throws IndexOutOfBoundsException if any invalid - * source indexes are found in {@code this} * @see #rearrange(VectorShuffle) */ public abstract Vector selectFrom(Vector v); @@ -2787,9 +2777,6 @@ public abstract VectorMask compare(VectorOperators.Comparison op, * @param v the vector supplying the result values * @param m the mask controlling selection from {@code v} * @return the rearrangement of the lane elements of {@code v} - * @throws IndexOutOfBoundsException if any invalid - * source indexes are found in {@code this}, - * in a lane which is set in the mask * @see #selectFrom(Vector) * @see #rearrange(VectorShuffle,VectorMask) */ diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index d7562bae4755a..fcc128ea8c7b0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2770,17 +2770,18 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { */ @Override public abstract - $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m); + $abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle); /*package-private*/ @ForceInline final > $abstractvectortype$ rearrangeTemplate(Class shuffletype, S shuffle) { - shuffle.checkIndexes(); + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, null, $type$.class, length(), - this, shuffle, null, + this, ws, null, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); return v1.lane(ei); @@ -2805,17 +2806,14 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { M m) { m.check(masktype, this); - VectorMask<$Boxtype$> valid = shuffle.laneIsValid(); - if (m.andNot(valid).anyTrue()) { - shuffle.checkIndexes(); - throw new AssertionError(); - } + @SuppressWarnings("unchecked") + S ws = (S) shuffle.wrapIndexes(); return VectorSupport.rearrangeOp( getClass(), shuffletype, masktype, $type$.class, length(), - this, shuffle, m, + this, ws, m, (v1, s_, m_) -> v1.uOp((i, a) -> { int ei = s_.laneSource(i); - return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei); + return !m_.laneIsSet(i) ? 0 : v1.lane(ei); })); } @@ -2928,7 +2926,10 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { /*package-private*/ @ForceInline final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) { - return v.rearrange(this.toShuffle()); + return ($Type$Vector)VectorSupport.selectFromOp(getClass(), null, $type$.class, + length(), this, v, null, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle())); } /** @@ -2940,9 +2941,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { /*package-private*/ @ForceInline - final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v, - AbstractMask<$Boxtype$> m) { - return v.rearrange(this.toShuffle(), m); + final + > + $abstractvectortype$ selectFromTemplate($abstractvectortype$ v, + Class masktype, M m) { + m.check(masktype, this); + return ($Type$Vector)VectorSupport.selectFromOp(getClass(), masktype, $type$.class, + length(), this, v, m, + (v1, v2, _m) -> + v2.rearrange(v1.toShuffle(), _m)); } /// Ternary operations diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 111d4bbefd44b..483962b4e0670 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -509,7 +509,7 @@ final class $vectortype$ extends $abstractvectortype$ { VectorMask<$Boxtype$> m) { return ($vectortype$) super.selectFromTemplate(($vectortype$) v, - ($masktype$) m); // specialize + $masktype$.class, ($masktype$) m); // specialize } @@ -1118,6 +1118,13 @@ final class $vectortype$ extends $abstractvectortype$ { return s.shuffleFromArray(shuffleArray, 0).check(s); } + @Override + @ForceInline + public $shuffletype$ wrapIndexes() { + return VectorSupport.wrapShuffleIndexes(ETYPE, $shuffletype$.class, this, VLENGTH, + (s) -> (($shuffletype$)(((AbstractShuffle<$Boxtype$>)(s)).wrapIndexesTemplate()))); + } + @ForceInline @Override public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {