diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
index 2e978b61380862..86abcf019375d4 100644
--- a/.github/PULL_REQUEST_TEMPLATE
+++ b/.github/PULL_REQUEST_TEMPLATE
@@ -4,7 +4,7 @@ the Gerrit change.
**Please ensure you adhere to every item in this list.**
-More info can be found at https://github.com/golang/go/wiki/CommitMessage
+More info can be found at https://go.dev/wiki/CommitMessage
+ The PR title is formatted as follows: `net/http: frob the quux before blarfing`
+ The package name goes before the colon
@@ -12,14 +12,14 @@ More info can be found at https://github.com/golang/go/wiki/CommitMessage
"This change modifies Go to ___________"
+ Lowercase verb after the colon
+ No trailing period
- + Keep the title as short as possible. ideally under 76 characters or shorter
+ + Keep the title as short as possible, ideally 72 characters or shorter
+ No Markdown
-+ The first PR comment (this one) is wrapped at 76 characters, unless it's
++ The first PR comment (this one) is wrapped at around 72 characters, unless it's
really needed (ASCII art, table, or long link)
+ If there is a corresponding issue, add either `Fixes #1234` or `Updates #1234`
(the latter if this is not a complete fix) to this comment
+ If referring to a repo other than `golang/go` you can use the
- `owner/repo#issue_number` syntax: `Fixes golang/tools#1234`
+ `owner/repo#issue_number` syntax: `Fixes golang/vscode-go#1234`
+ We do not use Signed-off-by lines in Go. Please don't add them.
Our Gerrit server & GitHub bots enforce CLA compliance instead.
+ Delete these instructions once you have read and applied them
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000000000..0db2e81c5b78d9
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,41 @@
+name: CI
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - master
+ - 'cockroach-*'
+ pull_request:
+ branches:
+ - master
+ - 'cockroach-*'
+
+jobs:
+ build-and-test:
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Set up bootstrap Go
+ uses: actions/setup-go@v6
+ with:
+ go-version: '1.24'
+ cache: false
+
+ - name: Save GOROOT_BOOTSTRAP
+ run: echo "GOROOT_BOOTSTRAP=$(go env GOROOT)" >> "$GITHUB_ENV"
+
+ - name: Build toolchain
+ run: cd src && ./make.bash
+
+ # Run the dist test suite, skipping:
+ # :race / :race-external — need race detector build (dist test -race)
+ # testfortran — needs gfortran
+ # relnote:check — needs doc/next/ fragments (not on release branches)
+ - name: Run tests
+ run: |
+ bin/go tool dist test -k \
+ -run '!:(race|race-external)$|testfortran|relnote:check'
+ env:
+ GOROOT: ${{ github.workspace }}
diff --git a/.gitignore b/.gitignore
index c6512e64a4ef39..344b31f7ac1c1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ _testmain.go
/misc/cgo/testso/main
/pkg/
/src/*.*/
+/src/_artifacts/
/src/cmd/cgo/zdefaultcc.go
/src/cmd/dist/dist
/src/cmd/go/internal/cfg/zdefaultcc.go
diff --git a/VERSION b/VERSION
new file mode 100644
index 00000000000000..8a3457effbde79
--- /dev/null
+++ b/VERSION
@@ -0,0 +1,2 @@
+go1.27.2
+time 2026-10-02T20:28:03Z
diff --git a/api/except.txt b/api/except.txt
index facae98033fe38..7dddd0e3805179 100644
--- a/api/except.txt
+++ b/api/except.txt
@@ -1,6 +1,13 @@
pkg crypto/tls, type ConnectionState struct, TLSUnique //deprecated
pkg debug/elf, const R_PPC64_SECTOFF_LO_DS = 61
pkg encoding/json, method (*RawMessage) MarshalJSON() ([]uint8, error)
+pkg encoding/json, method (*RawMessage) UnmarshalJSON([]uint8) error
+pkg encoding/json, method (RawMessage) MarshalJSON() ([]uint8, error)
+pkg encoding/json, type Marshaler interface { MarshalJSON }
+pkg encoding/json, type Marshaler interface, MarshalJSON() ([]uint8, error)
+pkg encoding/json, type RawMessage []uint8
+pkg encoding/json, type Unmarshaler interface { UnmarshalJSON }
+pkg encoding/json, type Unmarshaler interface, UnmarshalJSON([]uint8) error
pkg math, const MaxFloat64 = 1.79769e+308 // 179769313486231570814527423731704356798100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
pkg math, const SmallestNonzeroFloat32 = 1.4013e-45 // 17516230804060213386546619791123951641/12500000000000000000000000000000000000000000000000000000000000000000000000000000000
pkg math, const SmallestNonzeroFloat64 = 4.94066e-324 // 4940656458412465441765687928682213723651/1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
@@ -503,6 +510,7 @@ pkg unicode, const Version = "10.0.0"
pkg unicode, const Version = "11.0.0"
pkg unicode, const Version = "12.0.0"
pkg unicode, const Version = "13.0.0"
+pkg unicode, const Version = "15.0.0"
pkg unicode, const Version = "6.2.0"
pkg unicode, const Version = "6.3.0"
pkg unicode, const Version = "7.0.0"
diff --git a/api/go1.16.txt b/api/go1.16.txt
index e12a050939a374..1cae348af9a592 100644
--- a/api/go1.16.txt
+++ b/api/go1.16.txt
@@ -538,7 +538,6 @@ pkg encoding/json, type UnmarshalFieldError //deprecated
pkg go/build, const AllowBinary //deprecated
pkg go/doc, type Package struct, Bugs //deprecated
pkg go/importer, func For //deprecated
-pkg go/importer, func ForCompiler //deprecated
pkg go/types, func NewInterface //deprecated
pkg go/types, method (*Interface) Embedded //deprecated
pkg image, var ZP //deprecated
diff --git a/api/go1.25.txt b/api/go1.25.txt
index d50d19545f163a..273b3a14de0b67 100644
--- a/api/go1.25.txt
+++ b/api/go1.25.txt
@@ -109,3 +109,9 @@ pkg testing/synctest, func Wait() #67434
pkg unicode, var CategoryAliases map[string]string #70780
pkg unicode, var Cn *RangeTable #70780
pkg unicode, var LC *RangeTable #70780
+pkg context, func PropagateCancel(Context, func()) bool #0
+pkg runtime, func NumRunnableGoroutines() (int, int) #0
+pkg runtime, func CurrentP() int #0
+pkg runtime, func Yield() int64 #0
+pkg runtime, func GCAssistEnabled() bool #0
+pkg runtime, func SetGCAssistEnabled(bool) bool #0
diff --git a/api/go1.26.txt b/api/go1.26.txt
new file mode 100644
index 00000000000000..0706895bbdfef4
--- /dev/null
+++ b/api/go1.26.txt
@@ -0,0 +1,180 @@
+pkg bytes, method (*Buffer) Peek(int) ([]uint8, error) #73794
+pkg crypto, type Decapsulator interface { Decapsulate, Encapsulator } #75300
+pkg crypto, type Decapsulator interface, Decapsulate([]uint8) ([]uint8, error) #75300
+pkg crypto, type Decapsulator interface, Encapsulator() Encapsulator #75300
+pkg crypto, type Encapsulator interface { Bytes, Encapsulate } #75300
+pkg crypto, type Encapsulator interface, Bytes() []uint8 #75300
+pkg crypto, type Encapsulator interface, Encapsulate() ([]uint8, []uint8) #75300
+pkg crypto/ecdh, type KeyExchanger interface { Curve, ECDH, PublicKey } #75300
+pkg crypto/ecdh, type KeyExchanger interface, Curve() Curve #75300
+pkg crypto/ecdh, type KeyExchanger interface, ECDH(*PublicKey) ([]uint8, error) #75300
+pkg crypto/ecdh, type KeyExchanger interface, PublicKey() *PublicKey #75300
+pkg crypto/ecdsa, type PrivateKey struct, D //deprecated #63963
+pkg crypto/ecdsa, type PublicKey struct, X //deprecated #63963
+pkg crypto/ecdsa, type PublicKey struct, Y //deprecated #63963
+pkg crypto/fips140, func Enforced() bool #74630
+pkg crypto/fips140, func Version() string #75301
+pkg crypto/fips140, func WithoutEnforcement(func()) #74630
+pkg crypto/hpke, func AES128GCM() AEAD #75300
+pkg crypto/hpke, func AES256GCM() AEAD #75300
+pkg crypto/hpke, func ChaCha20Poly1305() AEAD #75300
+pkg crypto/hpke, func DHKEM(ecdh.Curve) KEM #75300
+pkg crypto/hpke, func ExportOnly() AEAD #75300
+pkg crypto/hpke, func HKDFSHA256() KDF #75300
+pkg crypto/hpke, func HKDFSHA384() KDF #75300
+pkg crypto/hpke, func HKDFSHA512() KDF #75300
+pkg crypto/hpke, func MLKEM1024() KEM #75300
+pkg crypto/hpke, func MLKEM1024P384() KEM #75300
+pkg crypto/hpke, func MLKEM768() KEM #75300
+pkg crypto/hpke, func MLKEM768P256() KEM #75300
+pkg crypto/hpke, func MLKEM768X25519() KEM #75300
+pkg crypto/hpke, func NewAEAD(uint16) (AEAD, error) #75300
+pkg crypto/hpke, func NewDHKEMPrivateKey(ecdh.KeyExchanger) (PrivateKey, error) #75300
+pkg crypto/hpke, func NewDHKEMPublicKey(*ecdh.PublicKey) (PublicKey, error) #75300
+pkg crypto/hpke, func NewHybridPrivateKey(crypto.Decapsulator, ecdh.KeyExchanger) (PrivateKey, error) #75300
+pkg crypto/hpke, func NewHybridPublicKey(crypto.Encapsulator, *ecdh.PublicKey) (PublicKey, error) #75300
+pkg crypto/hpke, func NewKDF(uint16) (KDF, error) #75300
+pkg crypto/hpke, func NewKEM(uint16) (KEM, error) #75300
+pkg crypto/hpke, func NewMLKEMPrivateKey(crypto.Decapsulator) (PrivateKey, error) #75300
+pkg crypto/hpke, func NewMLKEMPublicKey(crypto.Encapsulator) (PublicKey, error) #75300
+pkg crypto/hpke, func NewRecipient([]uint8, PrivateKey, KDF, AEAD, []uint8) (*Recipient, error) #75300
+pkg crypto/hpke, func NewSender(PublicKey, KDF, AEAD, []uint8) ([]uint8, *Sender, error) #75300
+pkg crypto/hpke, func Open(PrivateKey, KDF, AEAD, []uint8, []uint8) ([]uint8, error) #75300
+pkg crypto/hpke, func SHAKE128() KDF #75300
+pkg crypto/hpke, func SHAKE256() KDF #75300
+pkg crypto/hpke, func Seal(PublicKey, KDF, AEAD, []uint8, []uint8) ([]uint8, error) #75300
+pkg crypto/hpke, method (*Recipient) Export(string, int) ([]uint8, error) #75300
+pkg crypto/hpke, method (*Recipient) Open([]uint8, []uint8) ([]uint8, error) #75300
+pkg crypto/hpke, method (*Sender) Export(string, int) ([]uint8, error) #75300
+pkg crypto/hpke, method (*Sender) Seal([]uint8, []uint8) ([]uint8, error) #75300
+pkg crypto/hpke, type AEAD interface, ID() uint16 #75300
+pkg crypto/hpke, type AEAD interface, unexported methods #75300
+pkg crypto/hpke, type KDF interface, ID() uint16 #75300
+pkg crypto/hpke, type KDF interface, unexported methods #75300
+pkg crypto/hpke, type KEM interface, DeriveKeyPair([]uint8) (PrivateKey, error) #75300
+pkg crypto/hpke, type KEM interface, GenerateKey() (PrivateKey, error) #75300
+pkg crypto/hpke, type KEM interface, ID() uint16 #75300
+pkg crypto/hpke, type KEM interface, NewPrivateKey([]uint8) (PrivateKey, error) #75300
+pkg crypto/hpke, type KEM interface, NewPublicKey([]uint8) (PublicKey, error) #75300
+pkg crypto/hpke, type KEM interface, unexported methods #75300
+pkg crypto/hpke, type PrivateKey interface, Bytes() ([]uint8, error) #75300
+pkg crypto/hpke, type PrivateKey interface, KEM() KEM #75300
+pkg crypto/hpke, type PrivateKey interface, PublicKey() PublicKey #75300
+pkg crypto/hpke, type PrivateKey interface, unexported methods #75300
+pkg crypto/hpke, type PublicKey interface, Bytes() []uint8 #75300
+pkg crypto/hpke, type PublicKey interface, KEM() KEM #75300
+pkg crypto/hpke, type PublicKey interface, unexported methods #75300
+pkg crypto/hpke, type Recipient struct #75300
+pkg crypto/hpke, type Sender struct #75300
+pkg crypto/mlkem, method (*DecapsulationKey1024) Encapsulator() crypto.Encapsulator #75300
+pkg crypto/mlkem, method (*DecapsulationKey768) Encapsulator() crypto.Encapsulator #75300
+pkg crypto/mlkem/mlkemtest, func Encapsulate1024(*mlkem.EncapsulationKey1024, []uint8) ([]uint8, []uint8, error) #73627
+pkg crypto/mlkem/mlkemtest, func Encapsulate768(*mlkem.EncapsulationKey768, []uint8) ([]uint8, []uint8, error) #73627
+pkg crypto/rsa, func DecryptPKCS1v15 //deprecated #75302
+pkg crypto/rsa, func DecryptPKCS1v15SessionKey //deprecated #75302
+pkg crypto/rsa, func EncryptOAEPWithOptions(io.Reader, *PublicKey, []uint8, *OAEPOptions) ([]uint8, error) #65716
+pkg crypto/rsa, func EncryptPKCS1v15 //deprecated #75302
+pkg crypto/rsa, type PKCS1v15DecryptOptions //deprecated #75302
+pkg crypto/tls, const QUICErrorEvent = 10 #75108
+pkg crypto/tls, const QUICErrorEvent QUICEventKind #75108
+pkg crypto/tls, const SecP256r1MLKEM768 = 4587 #71206
+pkg crypto/tls, const SecP256r1MLKEM768 CurveID #71206
+pkg crypto/tls, const SecP384r1MLKEM1024 = 4589 #71206
+pkg crypto/tls, const SecP384r1MLKEM1024 CurveID #71206
+pkg crypto/tls, type ClientHelloInfo struct, HelloRetryRequest bool #74425
+pkg crypto/tls, type ConnectionState struct, HelloRetryRequest bool #74425
+pkg crypto/tls, type QUICEvent struct, Err error #75108
+pkg crypto/x509, func OIDFromASN1OID(asn1.ObjectIdentifier) (OID, error) #75325
+pkg crypto/x509, method (ExtKeyUsage) OID() OID #75325
+pkg crypto/x509, method (ExtKeyUsage) String() string #56866
+pkg crypto/x509, method (KeyUsage) String() string #56866
+pkg debug/elf, const R_LARCH_CALL36 = 110 #75562
+pkg debug/elf, const R_LARCH_CALL36 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC32 = 13 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC32 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64 = 14 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_HI12 = 118 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_HI12 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_LO20 = 117 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_LO20 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_PC_HI12 = 114 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_PC_HI12 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_PC_LO20 = 113 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC64_PC_LO20 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_CALL = 120 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_CALL R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_HI20 = 115 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_HI20 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_LD = 119 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_LD R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_LO12 = 116 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_LO12 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PCREL20_S2 = 126 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PCREL20_S2 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PC_HI20 = 111 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PC_HI20 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PC_LO12 = 112 #75562
+pkg debug/elf, const R_LARCH_TLS_DESC_PC_LO12 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_GD_PCREL20_S2 = 125 #75562
+pkg debug/elf, const R_LARCH_TLS_GD_PCREL20_S2 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_LD_PCREL20_S2 = 124 #75562
+pkg debug/elf, const R_LARCH_TLS_LD_PCREL20_S2 R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_LE_ADD_R = 122 #75562
+pkg debug/elf, const R_LARCH_TLS_LE_ADD_R R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_LE_HI20_R = 121 #75562
+pkg debug/elf, const R_LARCH_TLS_LE_HI20_R R_LARCH #75562
+pkg debug/elf, const R_LARCH_TLS_LE_LO12_R = 123 #75562
+pkg debug/elf, const R_LARCH_TLS_LE_LO12_R R_LARCH #75562
+pkg errors, func AsType[$0 error](error) ($0, bool) #51945
+pkg go/ast, func ParseDirective(token.Pos, string) (Directive, bool) #68021
+pkg go/ast, method (*Directive) End() token.Pos #68021
+pkg go/ast, method (*Directive) ParseArgs() ([]DirectiveArg, error) #68021
+pkg go/ast, method (*Directive) Pos() token.Pos #68021
+pkg go/ast, type BasicLit struct, ValueEnd token.Pos #76031
+pkg go/ast, type Directive struct #68021
+pkg go/ast, type Directive struct, Args string #68021
+pkg go/ast, type Directive struct, ArgsPos token.Pos #68021
+pkg go/ast, type Directive struct, Name string #68021
+pkg go/ast, type Directive struct, Slash token.Pos #68021
+pkg go/ast, type Directive struct, Tool string #68021
+pkg go/ast, type DirectiveArg struct #68021
+pkg go/ast, type DirectiveArg struct, Arg string #68021
+pkg go/ast, type DirectiveArg struct, Pos token.Pos #68021
+pkg go/token, method (*File) End() Pos #75849
+pkg log/slog, func NewMultiHandler(...Handler) *MultiHandler #65954
+pkg log/slog, method (*MultiHandler) Enabled(context.Context, Level) bool #65954
+pkg log/slog, method (*MultiHandler) Handle(context.Context, Record) error #65954
+pkg log/slog, method (*MultiHandler) WithAttrs([]Attr) Handler #65954
+pkg log/slog, method (*MultiHandler) WithGroup(string) Handler #65954
+pkg log/slog, type MultiHandler struct #65954
+pkg net, method (*Dialer) DialIP(context.Context, string, netip.Addr, netip.Addr) (*IPConn, error) #49097
+pkg net, method (*Dialer) DialTCP(context.Context, string, netip.AddrPort, netip.AddrPort) (*TCPConn, error) #49097
+pkg net, method (*Dialer) DialUDP(context.Context, string, netip.AddrPort, netip.AddrPort) (*UDPConn, error) #49097
+pkg net, method (*Dialer) DialUnix(context.Context, string, *UnixAddr, *UnixAddr) (*UnixConn, error) #49097
+pkg net/http, method (*ClientConn) Available() int #75772
+pkg net/http, method (*ClientConn) Close() error #75772
+pkg net/http, method (*ClientConn) Err() error #75772
+pkg net/http, method (*ClientConn) InFlight() int #75772
+pkg net/http, method (*ClientConn) Release() #75772
+pkg net/http, method (*ClientConn) Reserve() error #75772
+pkg net/http, method (*ClientConn) RoundTrip(*Request) (*Response, error) #75772
+pkg net/http, method (*ClientConn) SetStateHook(func(*ClientConn)) #75772
+pkg net/http, method (*Transport) NewClientConn(context.Context, string, string) (*ClientConn, error) #75772
+pkg net/http, type ClientConn struct #75772
+pkg net/http, type HTTP2Config struct, StrictMaxConcurrentRequests bool #67813
+pkg net/http/httputil, type ReverseProxy struct, Director //deprecated #73161
+pkg net/netip, method (Prefix) Compare(Prefix) int #61642
+pkg os, method (*Process) WithHandle(func(uintptr)) error #70352
+pkg os, var ErrNoHandle error #70352
+pkg reflect, method (Value) Fields() iter.Seq2[StructField, Value] #66631
+pkg reflect, method (Value) Methods() iter.Seq2[Method, Value] #66631
+pkg reflect, type Type interface, Fields() iter.Seq[StructField] #66631
+pkg reflect, type Type interface, Ins() iter.Seq[Type] #66631
+pkg reflect, type Type interface, Methods() iter.Seq[Method] #66631
+pkg reflect, type Type interface, Outs() iter.Seq[Type] #66631
+pkg testing, method (*B) ArtifactDir() string #71287
+pkg testing, method (*F) ArtifactDir() string #71287
+pkg testing, method (*T) ArtifactDir() string #71287
+pkg testing, type TB interface, ArtifactDir() string #71287
+pkg testing/cryptotest, func SetGlobalRandom(*testing.T, uint64) #70942
diff --git a/api/go1.27.txt b/api/go1.27.txt
new file mode 100644
index 00000000000000..04b797d9f9a7c1
--- /dev/null
+++ b/api/go1.27.txt
@@ -0,0 +1,317 @@
+pkg bytes, func CutLast([]uint8, []uint8) ([]uint8, []uint8, bool) #71151
+pkg crypto, const MLDSAMu = 20 #77626
+pkg crypto, const MLDSAMu Hash #77626
+pkg crypto/mldsa, const MLDSA44PublicKeySize = 1312 #77626
+pkg crypto/mldsa, const MLDSA44PublicKeySize ideal-int #77626
+pkg crypto/mldsa, const MLDSA44SignatureSize = 2420 #77626
+pkg crypto/mldsa, const MLDSA44SignatureSize ideal-int #77626
+pkg crypto/mldsa, const MLDSA65PublicKeySize = 1952 #77626
+pkg crypto/mldsa, const MLDSA65PublicKeySize ideal-int #77626
+pkg crypto/mldsa, const MLDSA65SignatureSize = 3309 #77626
+pkg crypto/mldsa, const MLDSA65SignatureSize ideal-int #77626
+pkg crypto/mldsa, const MLDSA87PublicKeySize = 2592 #77626
+pkg crypto/mldsa, const MLDSA87PublicKeySize ideal-int #77626
+pkg crypto/mldsa, const MLDSA87SignatureSize = 4627 #77626
+pkg crypto/mldsa, const MLDSA87SignatureSize ideal-int #77626
+pkg crypto/mldsa, const PrivateKeySize = 32 #77626
+pkg crypto/mldsa, const PrivateKeySize ideal-int #77626
+pkg crypto/mldsa, func GenerateKey(Parameters) (*PrivateKey, error) #77626
+pkg crypto/mldsa, func MLDSA44() Parameters #77626
+pkg crypto/mldsa, func MLDSA65() Parameters #77626
+pkg crypto/mldsa, func MLDSA87() Parameters #77626
+pkg crypto/mldsa, func NewPrivateKey(Parameters, []uint8) (*PrivateKey, error) #77626
+pkg crypto/mldsa, func NewPublicKey(Parameters, []uint8) (*PublicKey, error) #77626
+pkg crypto/mldsa, func Verify(*PublicKey, []uint8, []uint8, *Options) error #77626
+pkg crypto/mldsa, method (*Options) HashFunc() crypto.Hash #77626
+pkg crypto/mldsa, method (*PrivateKey) Bytes() []uint8 #77626
+pkg crypto/mldsa, method (*PrivateKey) Equal(crypto.PrivateKey) bool #77626
+pkg crypto/mldsa, method (*PrivateKey) Public() crypto.PublicKey #77626
+pkg crypto/mldsa, method (*PrivateKey) PublicKey() *PublicKey #77626
+pkg crypto/mldsa, method (*PrivateKey) Sign(io.Reader, []uint8, crypto.SignerOpts) ([]uint8, error) #77626
+pkg crypto/mldsa, method (*PrivateKey) SignDeterministic([]uint8, crypto.SignerOpts) ([]uint8, error) #77626
+pkg crypto/mldsa, method (*PublicKey) Bytes() []uint8 #77626
+pkg crypto/mldsa, method (*PublicKey) Equal(crypto.PublicKey) bool #77626
+pkg crypto/mldsa, method (*PublicKey) Parameters() Parameters #77626
+pkg crypto/mldsa, method (Parameters) PublicKeySize() int #77626
+pkg crypto/mldsa, method (Parameters) SignatureSize() int #77626
+pkg crypto/mldsa, method (Parameters) String() string #77626
+pkg crypto/mldsa, type Options struct #77626
+pkg crypto/mldsa, type Options struct, Context string #77626
+pkg crypto/mldsa, type Parameters struct #77626
+pkg crypto/mldsa, type PrivateKey struct #77626
+pkg crypto/mldsa, type PublicKey struct #77626
+pkg crypto/tls, const MLDSA44 = 2308 #78888
+pkg crypto/tls, const MLDSA44 SignatureScheme #78888
+pkg crypto/tls, const MLDSA65 = 2309 #78888
+pkg crypto/tls, const MLDSA65 SignatureScheme #78888
+pkg crypto/tls, const MLDSA87 = 2310 #78888
+pkg crypto/tls, const MLDSA87 SignatureScheme #78888
+pkg crypto/tls, const MLKEM1024 = 514 #78543
+pkg crypto/tls, const MLKEM1024 CurveID #78543
+pkg crypto/tls, type Config struct, Rand //deprecated #79367
+pkg crypto/tls, type ConnectionState struct, LocalCertificate [][]uint8 #24673
+pkg crypto/tls, type QUICConfig struct, ClientHelloInfoConn net.Conn #77363
+pkg crypto/x509, const MLDSA = 5 #78888
+pkg crypto/x509, const MLDSA PublicKeyAlgorithm #78888
+pkg crypto/x509, const MLDSA44 = 17 #78888
+pkg crypto/x509, const MLDSA44 SignatureAlgorithm #78888
+pkg crypto/x509, const MLDSA65 = 18 #78888
+pkg crypto/x509, const MLDSA65 SignatureAlgorithm #78888
+pkg crypto/x509, const MLDSA87 = 19 #78888
+pkg crypto/x509, const MLDSA87 SignatureAlgorithm #78888
+pkg crypto/x509, type Certificate struct, RawSignatureAlgorithm []uint8 #76133
+pkg crypto/x509, type CertificateRequest struct, RawSignatureAlgorithm []uint8 #76133
+pkg crypto/x509, type RevocationList struct, RawSignatureAlgorithm []uint8 #76133
+pkg database/sql, func ConvertAssign(driver.ScanContext, interface{}, driver.Value) error #67546
+pkg database/sql/driver, type RowsColumnScanner interface { Close, Columns, Next, NextRow, ScanColumn } #67546
+pkg database/sql/driver, type RowsColumnScanner interface, Close() error #67546
+pkg database/sql/driver, type RowsColumnScanner interface, Columns() []string #67546
+pkg database/sql/driver, type RowsColumnScanner interface, Next([]Value) error #67546
+pkg database/sql/driver, type RowsColumnScanner interface, NextRow() error #67546
+pkg database/sql/driver, type RowsColumnScanner interface, ScanColumn(ScanContext, int, interface{}) error #67546
+pkg database/sql/driver, type ScanContext struct #67546
+pkg encoding/json, func CallMethodsWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func DefaultOptionsV1() jsonopts.Options #71497
+pkg encoding/json, func FormatByteArrayAsArray(bool) jsonopts.Options #71497
+pkg encoding/json, func FormatBytesWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func FormatDurationAsNano(bool) jsonopts.Options #71497
+pkg encoding/json, func MatchCaseSensitiveDelimiter(bool) jsonopts.Options #71497
+pkg encoding/json, func MergeWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func OmitEmptyWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func ParseBytesWithLooseRFC4648(bool) jsonopts.Options #71497
+pkg encoding/json, func ParseTimeWithLooseRFC3339(bool) jsonopts.Options #71497
+pkg encoding/json, func ReportErrorsWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func StringifyWithLegacySemantics(bool) jsonopts.Options #71497
+pkg encoding/json, func UnmarshalArrayFromAnyLength(bool) jsonopts.Options #71497
+pkg encoding/json, method (*Number) UnmarshalJSONFrom(*jsontext.Decoder) error #71497
+pkg encoding/json, method (*UnmarshalTypeError) Unwrap() error #71497
+pkg encoding/json, method (Number) MarshalJSONTo(*jsontext.Encoder) error #71497
+pkg encoding/json, type Marshaler = json.Marshaler #71497
+pkg encoding/json, type Options = jsonopts.Options #71497
+pkg encoding/json, type RawMessage = jsontext.Value #71497
+pkg encoding/json, type UnmarshalTypeError struct, Err error #71497
+pkg encoding/json, type Unmarshaler = json.Unmarshaler #71497
+pkg encoding/json/jsontext, const KindBeginArray = 91 #71497
+pkg encoding/json/jsontext, const KindBeginArray Kind #71497
+pkg encoding/json/jsontext, const KindBeginObject = 123 #71497
+pkg encoding/json/jsontext, const KindBeginObject Kind #71497
+pkg encoding/json/jsontext, const KindEndArray = 93 #71497
+pkg encoding/json/jsontext, const KindEndArray Kind #71497
+pkg encoding/json/jsontext, const KindEndObject = 125 #71497
+pkg encoding/json/jsontext, const KindEndObject Kind #71497
+pkg encoding/json/jsontext, const KindFalse = 102 #71497
+pkg encoding/json/jsontext, const KindFalse Kind #71497
+pkg encoding/json/jsontext, const KindInvalid = 0 #71497
+pkg encoding/json/jsontext, const KindInvalid Kind #71497
+pkg encoding/json/jsontext, const KindNull = 110 #71497
+pkg encoding/json/jsontext, const KindNull Kind #71497
+pkg encoding/json/jsontext, const KindNumber = 48 #71497
+pkg encoding/json/jsontext, const KindNumber Kind #71497
+pkg encoding/json/jsontext, const KindString = 34 #71497
+pkg encoding/json/jsontext, const KindString Kind #71497
+pkg encoding/json/jsontext, const KindTrue = 116 #71497
+pkg encoding/json/jsontext, const KindTrue Kind #71497
+pkg encoding/json/jsontext, func AllowDuplicateNames(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func AllowInvalidUTF8(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func AppendFloat([]uint8, float64, int) []uint8 #71497
+pkg encoding/json/jsontext, func AppendFormat[$0 interface{ ~[]uint8 | ~string }]([]uint8, $0, ...jsonopts.Options) ([]uint8, error) #71497
+pkg encoding/json/jsontext, func AppendQuote[$0 interface{ ~[]uint8 | ~string }]([]uint8, $0) ([]uint8, error) #71497
+pkg encoding/json/jsontext, func AppendUnquote[$0 interface{ ~[]uint8 | ~string }]([]uint8, $0) ([]uint8, error) #71497
+pkg encoding/json/jsontext, func Bool(bool) Token #71497
+pkg encoding/json/jsontext, func CanonicalizeRawFloats(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func CanonicalizeRawInts(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func EscapeForHTML(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func EscapeForJS(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func Float(float64) Token #71497
+pkg encoding/json/jsontext, func Float32(float32) Token #71497
+pkg encoding/json/jsontext, func Int(int64) Token #71497
+pkg encoding/json/jsontext, func Multiline(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func NewDecoder(io.Reader, ...jsonopts.Options) *Decoder #71497
+pkg encoding/json/jsontext, func NewEncoder(io.Writer, ...jsonopts.Options) *Encoder #71497
+pkg encoding/json/jsontext, func PreserveRawStrings(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func ReorderRawObjects(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func SpaceAfterColon(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func SpaceAfterComma(bool) jsonopts.Options #71497
+pkg encoding/json/jsontext, func String(string) Token #71497
+pkg encoding/json/jsontext, func Uint(uint64) Token #71497
+pkg encoding/json/jsontext, func WithIndent(string) jsonopts.Options #71497
+pkg encoding/json/jsontext, func WithIndentPrefix(string) jsonopts.Options #71497
+pkg encoding/json/jsontext, method (*Decoder) InputOffset() int64 #71497
+pkg encoding/json/jsontext, method (*Decoder) Options() jsonopts.Options #71497
+pkg encoding/json/jsontext, method (*Decoder) PeekKind() Kind #71497
+pkg encoding/json/jsontext, method (*Decoder) ReadToken() (Token, error) #71497
+pkg encoding/json/jsontext, method (*Decoder) ReadValue() (Value, error) #71497
+pkg encoding/json/jsontext, method (*Decoder) Reset(io.Reader, ...jsonopts.Options) #71497
+pkg encoding/json/jsontext, method (*Decoder) SkipValue() error #71497
+pkg encoding/json/jsontext, method (*Decoder) StackDepth() int #71497
+pkg encoding/json/jsontext, method (*Decoder) StackIndex(int) (Kind, int64) #71497
+pkg encoding/json/jsontext, method (*Decoder) StackPointer() Pointer #71497
+pkg encoding/json/jsontext, method (*Decoder) UnreadBuffer() []uint8 #71497
+pkg encoding/json/jsontext, method (*Encoder) AvailableBuffer() []uint8 #71497
+pkg encoding/json/jsontext, method (*Encoder) Options() jsonopts.Options #71497
+pkg encoding/json/jsontext, method (*Encoder) OutputOffset() int64 #71497
+pkg encoding/json/jsontext, method (*Encoder) Reset(io.Writer, ...jsonopts.Options) #71497
+pkg encoding/json/jsontext, method (*Encoder) StackDepth() int #71497
+pkg encoding/json/jsontext, method (*Encoder) StackIndex(int) (Kind, int64) #71497
+pkg encoding/json/jsontext, method (*Encoder) StackPointer() Pointer #71497
+pkg encoding/json/jsontext, method (*Encoder) WriteToken(Token) error #71497
+pkg encoding/json/jsontext, method (*Encoder) WriteValue(Value) error #71497
+pkg encoding/json/jsontext, method (*SyntacticError) Error() string #71497
+pkg encoding/json/jsontext, method (*SyntacticError) Unwrap() error #71497
+pkg encoding/json/jsontext, method (*Value) Canonicalize(...jsonopts.Options) error #71497
+pkg encoding/json/jsontext, method (*Value) Compact(...jsonopts.Options) error #71497
+pkg encoding/json/jsontext, method (*Value) Format(...jsonopts.Options) error #71497
+pkg encoding/json/jsontext, method (*Value) Indent(...jsonopts.Options) error #71497
+pkg encoding/json/jsontext, method (*Value) UnmarshalJSON([]uint8) error #71497
+pkg encoding/json/jsontext, method (Kind) String() string #71497
+pkg encoding/json/jsontext, method (Pointer) AppendToken(string) Pointer #71497
+pkg encoding/json/jsontext, method (Pointer) Contains(Pointer) bool #71497
+pkg encoding/json/jsontext, method (Pointer) IsValid() bool #71497
+pkg encoding/json/jsontext, method (Pointer) LastToken() string #71497
+pkg encoding/json/jsontext, method (Pointer) Parent() Pointer #71497
+pkg encoding/json/jsontext, method (Pointer) Tokens() iter.Seq[string] #71497
+pkg encoding/json/jsontext, method (Token) Bool() bool #71497
+pkg encoding/json/jsontext, method (Token) Clone() Token #71497
+pkg encoding/json/jsontext, method (Token) Float() (float64, error) #71497
+pkg encoding/json/jsontext, method (Token) Float32() (float32, error) #71497
+pkg encoding/json/jsontext, method (Token) Int() (int64, error) #71497
+pkg encoding/json/jsontext, method (Token) Kind() Kind #71497
+pkg encoding/json/jsontext, method (Token) String() string #71497
+pkg encoding/json/jsontext, method (Token) Uint() (uint64, error) #71497
+pkg encoding/json/jsontext, method (Value) Clone() Value #71497
+pkg encoding/json/jsontext, method (Value) IsValid(...jsonopts.Options) bool #71497
+pkg encoding/json/jsontext, method (Value) Kind() Kind #71497
+pkg encoding/json/jsontext, method (Value) MarshalJSON() ([]uint8, error) #71497
+pkg encoding/json/jsontext, method (Value) String() string #71497
+pkg encoding/json/jsontext, type Decoder struct #71497
+pkg encoding/json/jsontext, type Encoder struct #71497
+pkg encoding/json/jsontext, type Kind uint8 #71497
+pkg encoding/json/jsontext, type Options = jsonopts.Options #71497
+pkg encoding/json/jsontext, type Pointer string #71497
+pkg encoding/json/jsontext, type SyntacticError struct #71497
+pkg encoding/json/jsontext, type SyntacticError struct, ByteOffset int64 #71497
+pkg encoding/json/jsontext, type SyntacticError struct, Err error #71497
+pkg encoding/json/jsontext, type SyntacticError struct, JSONPointer Pointer #71497
+pkg encoding/json/jsontext, type Token struct #71497
+pkg encoding/json/jsontext, type Value []uint8 #71497
+pkg encoding/json/jsontext, var BeginArray Token #71497
+pkg encoding/json/jsontext, var BeginObject Token #71497
+pkg encoding/json/jsontext, var EndArray Token #71497
+pkg encoding/json/jsontext, var EndObject Token #71497
+pkg encoding/json/jsontext, var ErrDuplicateName error #71497
+pkg encoding/json/jsontext, var ErrNonStringName error #71497
+pkg encoding/json/jsontext, var False Token #71497
+pkg encoding/json/jsontext, var Internal exporter #71497
+pkg encoding/json/jsontext, var Null Token #71497
+pkg encoding/json/jsontext, var True Token #71497
+pkg encoding/json/v2, func DefaultOptionsV2() jsonopts.Options #71497
+pkg encoding/json/v2, func Deterministic(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func FormatNilMapAsNull(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func FormatNilSliceAsNull(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func GetOption[$0 interface{}](jsonopts.Options, func($0) jsonopts.Options) ($0, bool) #71497
+pkg encoding/json/v2, func JoinMarshalers(...*typedArshalers[jsontext.Encoder]) *typedArshalers[jsontext.Encoder] #71497
+pkg encoding/json/v2, func JoinOptions(...jsonopts.Options) jsonopts.Options #71497
+pkg encoding/json/v2, func JoinUnmarshalers(...*typedArshalers[jsontext.Decoder]) *typedArshalers[jsontext.Decoder] #71497
+pkg encoding/json/v2, func Marshal(interface{}, ...jsonopts.Options) ([]uint8, error) #71497
+pkg encoding/json/v2, func MarshalEncode(*jsontext.Encoder, interface{}, ...jsonopts.Options) error #71497
+pkg encoding/json/v2, func MarshalFunc[$0 interface{}](func($0) ([]uint8, error)) *typedArshalers[jsontext.Encoder] #71497
+pkg encoding/json/v2, func MarshalToFunc[$0 interface{}](func(*jsontext.Encoder, $0) error) *typedArshalers[jsontext.Encoder] #71497
+pkg encoding/json/v2, func MarshalWrite(io.Writer, interface{}, ...jsonopts.Options) error #71497
+pkg encoding/json/v2, func MatchCaseInsensitiveNames(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func OmitZeroStructFields(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func RejectUnknownMembers(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func StringifyNumbers(bool) jsonopts.Options #71497
+pkg encoding/json/v2, func Unmarshal([]uint8, interface{}, ...jsonopts.Options) error #71497
+pkg encoding/json/v2, func UnmarshalDecode(*jsontext.Decoder, interface{}, ...jsonopts.Options) error #71497
+pkg encoding/json/v2, func UnmarshalFromFunc[$0 interface{}](func(*jsontext.Decoder, $0) error) *typedArshalers[jsontext.Decoder] #71497
+pkg encoding/json/v2, func UnmarshalFunc[$0 interface{}](func([]uint8, $0) error) *typedArshalers[jsontext.Decoder] #71497
+pkg encoding/json/v2, func UnmarshalRead(io.Reader, interface{}, ...jsonopts.Options) error #71497
+pkg encoding/json/v2, func WithMarshalers(*typedArshalers[jsontext.Encoder]) jsonopts.Options #71497
+pkg encoding/json/v2, func WithUnmarshalers(*typedArshalers[jsontext.Decoder]) jsonopts.Options #71497
+pkg encoding/json/v2, method (*SemanticError) Error() string #71497
+pkg encoding/json/v2, method (*SemanticError) Unwrap() error #71497
+pkg encoding/json/v2, type Marshaler interface { MarshalJSON } #71497
+pkg encoding/json/v2, type Marshaler interface, MarshalJSON() ([]uint8, error) #71497
+pkg encoding/json/v2, type MarshalerTo interface { MarshalJSONTo } #71497
+pkg encoding/json/v2, type MarshalerTo interface, MarshalJSONTo(*jsontext.Encoder) error #71497
+pkg encoding/json/v2, type Marshalers = typedArshalers[jsontext.Encoder] #71497
+pkg encoding/json/v2, type Options = jsonopts.Options #71497
+pkg encoding/json/v2, type SemanticError struct #71497
+pkg encoding/json/v2, type SemanticError struct, ByteOffset int64 #71497
+pkg encoding/json/v2, type SemanticError struct, Err error #71497
+pkg encoding/json/v2, type SemanticError struct, GoType reflect.Type #71497
+pkg encoding/json/v2, type SemanticError struct, JSONKind jsontext.Kind #71497
+pkg encoding/json/v2, type SemanticError struct, JSONPointer jsontext.Pointer #71497
+pkg encoding/json/v2, type SemanticError struct, JSONValue jsontext.Value #71497
+pkg encoding/json/v2, type Unmarshaler interface { UnmarshalJSON } #71497
+pkg encoding/json/v2, type Unmarshaler interface, UnmarshalJSON([]uint8) error #71497
+pkg encoding/json/v2, type UnmarshalerFrom interface { UnmarshalJSONFrom } #71497
+pkg encoding/json/v2, type UnmarshalerFrom interface, UnmarshalJSONFrom(*jsontext.Decoder) error #71497
+pkg encoding/json/v2, type Unmarshalers = typedArshalers[jsontext.Decoder] #71497
+pkg encoding/json/v2, var ErrUnknownName error #71497
+pkg go/constant, func StringLen(Value) int64 #79042
+pkg go/scanner, method (*Scanner) End() token.Pos #74958
+pkg go/token, method (*File) String() string #76285
+pkg go/types, method (*TypeList) String() string #79287
+pkg go/types, method (*TypeParamList) String() string #79287
+pkg go/types, method (Hasher) Equal(Type, Type) bool #69420
+pkg go/types, method (Hasher) Hash(*maphash.Hash, Type) #69420
+pkg go/types, method (HasherIgnoreTags) Equal(Type, Type) bool #69420
+pkg go/types, method (HasherIgnoreTags) Hash(*maphash.Hash, Type) #69420
+pkg go/types, method (Instance) String() string #79287
+pkg go/types, type Hasher struct #69420
+pkg go/types, type HasherIgnoreTags struct #69420
+pkg hash/maphash, method (ComparableHasher[$0]) Equal($0, $0) bool #70471
+pkg hash/maphash, method (ComparableHasher[$0]) Hash(*Hash, $0) #70471
+pkg hash/maphash, type ComparableHasher[$0 comparable] struct #70471
+pkg hash/maphash, type Hasher[$0 interface{}] interface { Equal, Hash } #70471
+pkg hash/maphash, type Hasher[$0 interface{}] interface, Equal($0, $0) bool #70471
+pkg hash/maphash, type Hasher[$0 interface{}] interface, Hash(*Hash, $0) #70471
+pkg math/big, const Ceil = 5 #76821
+pkg math/big, const Ceil RoundingMode #76821
+pkg math/big, const Floor = 4 #76821
+pkg math/big, const Floor RoundingMode #76821
+pkg math/big, const Round = 0 #76821
+pkg math/big, const Round RoundingMode #76821
+pkg math/big, const Trunc = 2 #76821
+pkg math/big, const Trunc RoundingMode #76821
+pkg math/big, method (*Int) Divide(*Int, *Int, *Int, RoundingMode) (*Int, *Int) #76821
+pkg math/rand/v2, method (*Rand) N[$0 intType]($0) $0 #77853
+pkg net/http, type Server struct, DisableClientPriority bool #75500
+pkg net/http, const DefaultMaxHeaderValueCount = 500 #79936
+pkg net/http, const DefaultMaxHeaderValueCount ideal-int #79936
+pkg net/http, type Server struct, MaxHeaderValueCount int #79936
+pkg net/http/httptest, func NewTestServer(testing.TB, http.Handler) *Server #76608
+pkg net/url, method (*URL) Clone() *URL #73450
+pkg net/url, method (Values) Clone() Values #73450
+pkg strings, func CutLast(string, string) (string, string, bool) #71151
+pkg testing/synctest, func Sleep(time.Duration) #77169
+pkg unicode, const Version = "17.0.0" #77266
+pkg unicode, var Beria_Erfe *RangeTable #77266
+pkg unicode, var Garay *RangeTable #77266
+pkg unicode, var Gurung_Khema *RangeTable #77266
+pkg unicode, var IDS_Unary_Operator *RangeTable #77266
+pkg unicode, var ID_Compat_Math_Continue *RangeTable #77266
+pkg unicode, var ID_Compat_Math_Start *RangeTable #77266
+pkg unicode, var Kirat_Rai *RangeTable #77266
+pkg unicode, var Modifier_Combining_Mark *RangeTable #77266
+pkg unicode, var Ol_Onal *RangeTable #77266
+pkg unicode, var Sidetic *RangeTable #77266
+pkg unicode, var Sunuwar *RangeTable #77266
+pkg unicode, var Tai_Yo *RangeTable #77266
+pkg unicode, var Todhri *RangeTable #77266
+pkg unicode, var Tolong_Siki *RangeTable #77266
+pkg unicode, var Tulu_Tigalari *RangeTable #77266
+pkg uuid, func Max() UUID #62026
+pkg uuid, func MustParse(string) UUID #62026
+pkg uuid, func New() UUID #62026
+pkg uuid, func NewV4() UUID #62026
+pkg uuid, func NewV7() UUID #62026
+pkg uuid, func Nil() UUID #62026
+pkg uuid, func Parse(string) (UUID, error) #62026
+pkg uuid, method (*UUID) UnmarshalText([]uint8) error #62026
+pkg uuid, method (UUID) AppendText([]uint8) ([]uint8, error) #62026
+pkg uuid, method (UUID) Compare(UUID) int #62026
+pkg uuid, method (UUID) MarshalText() ([]uint8, error) #62026
+pkg uuid, method (UUID) String() string #62026
+pkg uuid, type UUID [16]uint8 #62026
diff --git a/api/go1.txt b/api/go1.txt
index 5e3dea5a372525..54e4a2833e048a 100644
--- a/api/go1.txt
+++ b/api/go1.txt
@@ -30869,3 +30869,6 @@ pkg unicode/utf8, func RuneLen(int32) int
pkg unicode/utf8, func RuneStart(uint8) bool
pkg unicode/utf8, func Valid([]uint8) bool
pkg unicode/utf8, func ValidString(string) bool
+pkg context, func PropagateCancel(Context, func()) bool
+pkg runtime, func Grunningnanos() int64
+pkg runtime, func NumRunnableGoroutines() (int, int)
diff --git a/api/next/49097.txt b/api/next/49097.txt
deleted file mode 100644
index f7240954c66228..00000000000000
--- a/api/next/49097.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-pkg net, method (*Dialer) DialIP(context.Context, string, netip.Addr, netip.Addr) (*IPConn, error) #49097
-pkg net, method (*Dialer) DialTCP(context.Context, string, netip.AddrPort, netip.AddrPort) (*TCPConn, error) #49097
-pkg net, method (*Dialer) DialUDP(context.Context, string, netip.AddrPort, netip.AddrPort) (*UDPConn, error) #49097
-pkg net, method (*Dialer) DialUnix(context.Context, string, *UnixAddr, *UnixAddr) (*UnixConn, error) #49097
diff --git a/api/next/51945.txt b/api/next/51945.txt
deleted file mode 100644
index 7db1f093e5a2a5..00000000000000
--- a/api/next/51945.txt
+++ /dev/null
@@ -1 +0,0 @@
-pkg errors, func AsType[$0 error](error) ($0, bool) #51945
diff --git a/api/next/61642.txt b/api/next/61642.txt
deleted file mode 100644
index dd67874ab9fba4..00000000000000
--- a/api/next/61642.txt
+++ /dev/null
@@ -1 +0,0 @@
-pkg net/netip, method (Prefix) Compare(Prefix) int #61642
diff --git a/api/next/63963.txt b/api/next/63963.txt
deleted file mode 100644
index c1aaae5044c0a4..00000000000000
--- a/api/next/63963.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-pkg crypto/ecdsa, type PrivateKey struct, D //deprecated #63963
-pkg crypto/ecdsa, type PublicKey struct, X //deprecated #63963
-pkg crypto/ecdsa, type PublicKey struct, Y //deprecated #63963
diff --git a/api/next/65954.txt b/api/next/65954.txt
deleted file mode 100644
index 88d7c2668fe02d..00000000000000
--- a/api/next/65954.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-pkg log/slog, func NewMultiHandler(...Handler) *MultiHandler #65954
-pkg log/slog, method (*MultiHandler) Enabled(context.Context, Level) bool #65954
-pkg log/slog, method (*MultiHandler) Handle(context.Context, Record) error #65954
-pkg log/slog, method (*MultiHandler) WithAttrs([]Attr) Handler #65954
-pkg log/slog, method (*MultiHandler) WithGroup(string) Handler #65954
-pkg log/slog, type MultiHandler struct #65954
diff --git a/api/next/67546.txt b/api/next/67546.txt
deleted file mode 100644
index 0b5b4b981c19a6..00000000000000
--- a/api/next/67546.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-pkg database/sql/driver, type RowsColumnScanner interface { Close, Columns, Next, ScanColumn } #67546
-pkg database/sql/driver, type RowsColumnScanner interface, Close() error #67546
-pkg database/sql/driver, type RowsColumnScanner interface, Columns() []string #67546
-pkg database/sql/driver, type RowsColumnScanner interface, Next([]Value) error #67546
-pkg database/sql/driver, type RowsColumnScanner interface, ScanColumn(interface{}, int) error #67546
diff --git a/api/next/67813.txt b/api/next/67813.txt
deleted file mode 100644
index b420a141e1fbc7..00000000000000
--- a/api/next/67813.txt
+++ /dev/null
@@ -1 +0,0 @@
-pkg net/http, type HTTP2Config struct, StrictMaxConcurrentRequests bool #67813
diff --git a/api/next/68021.txt b/api/next/68021.txt
deleted file mode 100644
index 46156e06654878..00000000000000
--- a/api/next/68021.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-pkg go/ast, func ParseDirective(token.Pos, string) (Directive, bool) #68021
-pkg go/ast, method (*Directive) End() token.Pos #68021
-pkg go/ast, method (*Directive) ParseArgs() ([]DirectiveArg, error) #68021
-pkg go/ast, method (*Directive) Pos() token.Pos #68021
-pkg go/ast, type Directive struct #68021
-pkg go/ast, type Directive struct, Args string #68021
-pkg go/ast, type Directive struct, ArgsPos token.Pos #68021
-pkg go/ast, type Directive struct, Name string #68021
-pkg go/ast, type Directive struct, Slash token.Pos #68021
-pkg go/ast, type Directive struct, Tool string #68021
-pkg go/ast, type DirectiveArg struct #68021
-pkg go/ast, type DirectiveArg struct, Arg string #68021
-pkg go/ast, type DirectiveArg struct, Pos token.Pos #68021
diff --git a/api/next/70352.txt b/api/next/70352.txt
deleted file mode 100644
index cd264046c3641d..00000000000000
--- a/api/next/70352.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-pkg os, method (*Process) WithHandle(func(uintptr)) error #70352
-pkg os, var ErrNoHandle error #70352
diff --git a/api/next/71287.txt b/api/next/71287.txt
deleted file mode 100644
index c1e09a1f523082..00000000000000
--- a/api/next/71287.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-pkg testing, method (*B) ArtifactDir() string #71287
-pkg testing, method (*F) ArtifactDir() string #71287
-pkg testing, method (*T) ArtifactDir() string #71287
-pkg testing, type TB interface, ArtifactDir() string #71287
diff --git a/api/next/73161.txt b/api/next/73161.txt
deleted file mode 100644
index 86526b597a9463..00000000000000
--- a/api/next/73161.txt
+++ /dev/null
@@ -1 +0,0 @@
-pkg net/http/httputil, type ReverseProxy struct, Director //deprecated #73161
diff --git a/codereview.cfg b/codereview.cfg
index 77a74f108eae36..374849066b3121 100644
--- a/codereview.cfg
+++ b/codereview.cfg
@@ -1 +1,2 @@
-branch: master
+branch: release-branch.go1.27
+parent-branch: master
diff --git a/doc/asm.html b/doc/asm.html
index dd395ec8331467..5db13905e3e50e 100644
--- a/doc/asm.html
+++ b/doc/asm.html
@@ -1039,6 +1039,12 @@ MIPS, MIPS64
GOMIPS64_hardfloat or GOMIPS64_softfloat.
+RISCV64
+
+
+Reference: Go RISCV64 Assembly Instructions Reference Manual
+
+
Unsupported opcodes
diff --git a/doc/go_spec.html b/doc/go_spec.html
index d6c7962a961b88..1234e9aa374879 100644
--- a/doc/go_spec.html
+++ b/doc/go_spec.html
@@ -1,6 +1,6 @@
@@ -825,8 +825,11 @@
Types
-Predeclared types, defined types, and type parameters are called named types .
-An alias denotes a named type if the type given in the alias declaration is a named type.
+Predeclared types (excluding any),
+defined types , and
+type parameters are called named types .
+An alias denotes a named type if the type given in the alias declaration is a named type.
+All named types are distinct .
Boolean types
@@ -835,7 +838,7 @@ Boolean types
A boolean type represents the set of Boolean truth values
denoted by the predeclared constants true
and false. The predeclared boolean type is bool;
-it is a defined type .
+it is a named type .
Numeric types
@@ -884,8 +887,7 @@ Numeric types
-To avoid portability issues all numeric types are defined
-types and thus distinct except
+To avoid portability issues all numeric types are named types and thus distinct except
byte, which is an alias for uint8, and
rune, which is an alias for int32.
Explicit conversions
@@ -904,7 +906,7 @@
String types
Strings are immutable: once created,
it is impossible to change the contents of a string.
The predeclared string type is string;
-it is a defined type .
+it is a named type .
@@ -1118,9 +1120,7 @@
Struct types
-Promoted fields act like ordinary fields
-of a struct except that they cannot be used as field names in
-composite literals of the struct.
+Promoted fields act like ordinary fields of a struct.
@@ -1294,6 +1294,8 @@
Basic interfaces
exactly of the methods specified by the interface.
Interfaces whose type sets can be defined entirely by a list of methods are called
basic interfaces.
+Interface methods cannot declare type parameters ,
+but they may use type parameters from the interface declaration.
@@ -1349,7 +1351,8 @@ Basic interfaces
-For convenience, the predeclared type any is an alias for the empty interface.
+For convenience, the predeclared type any is an alias for the empty interface;
+it is not a named type .
[Go 1.18 ]
@@ -1984,6 +1987,13 @@ Assignability
x implements T.
+x is a (possibly partially instantiated) generic function, T
+is a function type, and any type arguments not provided explicitly for x
+can be inferred such that (after full instantiation)
+x and T have identical underlying types
+[Go 1.27 ].
+
+
x is the predeclared identifier nil and T
is a pointer, function, slice, map, channel, or interface type,
but not a type parameter.
@@ -2487,11 +2497,15 @@ Alias declarations
-In an alias declaration the given type cannot be a type parameter.
+In an alias declaration the given type cannot be a type parameter declared in the same declaration.
-type A[P any] = P // illegal: P is a type parameter
+type A[P any] = P // illegal: P is a type parameter declared in the declaration of A
+
+func f[P any]() {
+ type A = P // ok: T is a type parameter declared by the enclosing function
+}
Type definitions
@@ -2601,8 +2615,8 @@ Type definitions
type T[P any] P // illegal: P is a type parameter
-func f[T any]() {
- type L T // illegal: T is a type parameter declared by the enclosing function
+func f[P any]() {
+ type L P // illegal: P is a type parameter declared by the enclosing function
}
@@ -2620,11 +2634,11 @@ Type definitions
Type parameter declarations
-A type parameter list declares the type parameters of a generic function or type declaration.
+A type parameter list declares the type parameters of a generic function, method, or type declaration.
The type parameter list looks like an ordinary function parameter list
except that the type parameter names must all be present and the list is enclosed
in square brackets rather than parentheses
-[Go 1.18 ].
+[Go 1.18 , Go 1.27 ].
@@ -2638,7 +2652,7 @@ Type parameter declarations
Each name declares a type parameter, which is a new and different named type
that acts as a placeholder for an (as of yet) unknown type in the declaration.
The type parameter is replaced with a type argument upon
-instantiation of the generic function or type.
+instantiation of the generic function, method, or type.
@@ -2686,22 +2700,6 @@ Type parameter declarations
with a generic type.
-
-Within a type parameter list of a generic type T, a type constraint
-may not (directly, or indirectly through the type parameter list of another
-generic type) refer to T.
-
-
-
-type T1[P T1[P]] … // illegal: T1 refers to itself
-type T2[P interface{ T2[int] }] … // illegal: T2 refers to itself
-type T3[P interface{ m(T3[int])}] … // illegal: T3 refers to itself
-type T4[P T5[P]] … // illegal: T4 refers to T5 and
-type T5[P T4[P]] … // T5 refers to T4
-
-type T6[P int] struct{ f *T6[P] } // ok: reference to T6 is not in type parameter list
-
-
Type constraints
@@ -2956,7 +2954,7 @@
Function declarations
If the function declaration specifies type parameters ,
-the function name denotes a generic function .
+the function name denotes a generic function [Go 1.18 ].
A generic function must be instantiated before it can be
called or used as a value.
@@ -2989,7 +2987,7 @@ Method declarations
-MethodDecl = "func" Receiver MethodName Signature [ FunctionBody ] .
+MethodDecl = "func" Receiver MethodName [ TypeParameters ] Signature [ FunctionBody ] .
Receiver = Parameters .
@@ -3000,7 +2998,7 @@ Method declarations
pointer to a defined type T, possibly followed by a list of type parameter
names [P1, P2, …] enclosed in square brackets.
T is called the receiver base type . A receiver base type cannot be
-a pointer or interface type and it must be defined in the same package as the method.
+a pointer or interface type and it must be declared in the same package as the method.
The method is said to be bound to its receiver base type and the method name
is visible only within selectors for type T
or *T.
@@ -3082,6 +3080,27 @@ Method declarations
func (*IPair) Second() int { … } // illegal: alias must not denote instantiated type Pair[int, int]
+
+If the method declaration specifies type parameters
+(possibly in addition to type parameters declared by the receiver specification), the method name denotes
+a generic method [Go 1.27 ].
+Like a generic function, a generic method must be instantiated before it can
+be called or used as a value.
+
+
+
+type List[E any] []E
+
+// Apply returns the list obtained from applying f to each element of l.
+func (l List[E]) Apply[F any](f func(E) F) List[F] {
+ r := make(List[F], len(l))
+ for i, x := range l {
+ r[i] = f(x)
+ }
+ return r
+}
+
+
Expressions
@@ -3155,7 +3174,8 @@
Composite literals
Composite literals construct new values for structs, arrays, slices, and maps
each time they are evaluated.
-They consist of the type of the literal followed by a brace-bound list of elements.
+They consist of the type of the literal followed by a (possibly empty)
+brace-bound list of elements.
Each element may optionally be preceded by a corresponding key.
@@ -3180,38 +3200,61 @@ Composite literals
If the LiteralType is a type parameter, all types in its type set
must have the same underlying type which must be
a valid composite literal type.
+
+
+
The types of the elements and keys must be assignable
-to the respective field, element, and key types of type T;
+to the respective field, element, and key types of the LiteralType;
there is no additional conversion.
-The key is interpreted as a field name for struct literals,
+The key is interpreted as a field selector for struct literals,
an index for array and slice literals, and a key for map literals.
-For map literals, all elements must have a key. It is an error
-to specify multiple elements with the same field name or
-constant key value. For non-constant map keys, see the section on
-evaluation order .
+It is an error to specify multiple elements with the same field selector
+or constant key value.
+A literal may omit the element list; such a literal evaluates
+to the zero value for its type.
+
+
+
+A parsing ambiguity arises when a composite literal using the
+TypeName form of the LiteralType appears as an operand between the
+keyword and the opening brace of the block
+of an "if", "for", or "switch" statement, and the composite literal
+is not enclosed in parentheses, square brackets, or curly braces.
+In this rare case, the opening brace of the literal is erroneously parsed
+as the one introducing the block of statements. To resolve the ambiguity,
+the composite literal must appear within parentheses.
+
+
+
+if x == (T{a,b,c}[i]) { … }
+if (x == T{a,b,c}[i]) { … }
+
+
+Struct literals
+
+
+For struct literals without keys, the element list must contain an element
+for each struct field in the order in which the fields are declared.
-For struct literals the following rules apply:
+For struct literals with keys the following rules apply:
- A key must be a field name declared in the struct type.
+ Every element must have a key.
- An element list that does not contain any keys must
- list an element for each struct field in the
- order in which the fields are declared.
+ Each key must be a valid field selector
+ [Go 1.27 ] for a (possibly promoted) field
+ of the struct; the key selects that field.
- If any element has a key, every element must have a key.
+ The types of the embedded fields (if any) traversed
+ to reach a selected field must not be pointer types.
- An element list that contains keys does not need to
- have an element for each struct field. Omitted fields
- get the zero value for that field.
+ A key must not denote a promoted field inside an embedded struct if
+ that struct is also specified by another key.
- A literal may omit the element list; such a literal evaluates
- to the zero value for its type.
-
- It is an error to specify an element for a non-exported
- field of a struct belonging to a different package.
+ The element list does not need to have an element for each struct field.
+ Omitted fields get the zero value for that field.
@@ -3219,8 +3262,9 @@ Composite literals
Given the declarations
-type Point3D struct { x, y, z float64 }
-type Line struct { p, q Point3D }
+type Object struct { name, color string }
+type Point3D struct { Object; x, y, z float64 }
+type Line struct { Object; p, q Point3D }
@@ -3228,10 +3272,22 @@
Composite literals
-origin := Point3D{} // zero value for Point3D
-line := Line{origin, Point3D{y: -4, z: 12.3}} // zero value for line.q.x
+origin := Point3D{} // zero value for Point3D
+line1 := Line{Object{}, origin, Point3D{y: -4, z: 12.3}} // zero value for line1.q.x
+line2 := Line{name: "diagonal", q: Point3D{1, 1, 1}} // zero value for line2.Object.color, line2.p
+
+
+
+but field selectors may not denote overlapping fields:
+
+
+
+obj := Object{"edge", "black"}
+line3 := Line{Object: obj, name: "diagonal"} // invalid: name denotes a field inside Object
+Array and slice literals
+
For array and slice literals the following rules apply:
@@ -3307,6 +3363,16 @@ Composite literals
tmp[0 : n]
+Map literals
+
+
+For map literals, each element must have a key.
+For non-constant map keys, see the section on
+evaluation order .
+
+
+Elision of element types
+
Within a composite literal of array, slice, or map type T,
elements or map keys that are themselves composite literals may elide the respective
@@ -3327,22 +3393,6 @@
Composite literals
[2]PPoint{{1.5, -3.5}, {}} // same as [2]PPoint{PPoint(&Point{1.5, -3.5}), PPoint(&Point{})}
-
-A parsing ambiguity arises when a composite literal using the
-TypeName form of the LiteralType appears as an operand between the
-keyword and the opening brace of the block
-of an "if", "for", or "switch" statement, and the composite literal
-is not enclosed in parentheses, square brackets, or curly braces.
-In this rare case, the opening brace of the literal is erroneously parsed
-as the one introducing the block of statements. To resolve the ambiguity,
-the composite literal must appear within parentheses.
-
-
-
-if x == (T{a,b,c}[i]) { … }
-if (x == T{a,b,c}[i]) { … }
-
-
Examples of valid array, slice, and map literals:
@@ -3390,7 +3440,7 @@ Function literals
Function literals are closures : they may refer to variables
-defined in a surrounding function. Those variables are then shared between
+declared in a surrounding function. Those variables are then shared between
the surrounding function and the function literal, and they survive as long
as they are accessible.
@@ -4334,8 +4384,8 @@ Passing arguments to ...Instantiations
-A generic function or type is instantiated by substituting type arguments
-for the type parameters [Go 1.18 ].
+A generic function, method, or type is instantiated by substituting type arguments
+for the type parameters [Go 1.18 ][Go 1.27 ].
Instantiation proceeds in two steps:
@@ -4355,8 +4405,8 @@ Instantiations
-Instantiating a type results in a new non-generic named type ;
-instantiating a function produces a new non-generic function.
+Instantiating a generic type, function, or method results in a non-generic type, function,
+or method, respectively.
@@ -4369,7 +4419,7 @@ Instantiations
-When using a generic function, type arguments may be provided explicitly,
+When using a generic function or method, type arguments may be provided explicitly,
or they may be partially or completely inferred
from the context in which the function is used.
Provided that they can be inferred, type argument lists may be omitted entirely if the function is:
@@ -4393,7 +4443,7 @@
Instantiations
In all other cases, a (possibly partial) type argument list must be present.
If a type argument list is absent or partial, all missing type arguments
-must be inferrable from the context in which the function is used.
+must be inferable from the context in which the function is used.
@@ -4532,11 +4582,16 @@ Type inference
-Type inference supports calls of generic functions and assignments
-of generic functions to (explicitly function-typed) variables.
-This includes passing generic functions as arguments to other
-(possibly also generic) functions, and returning generic functions
-as results.
+Type inference supports calls of generic functions and any use of
+a generic function in a context where the function must be
+assignable to a (non-generic)
+function type.
+The latter includes assigning a generic function to a variable
+(including passing it as an argument to another function),
+converting a generic function to a function type, and others.
+
+
+
Type inference operates on a set of equations specific to each of
these cases.
The equations are as follows (type argument lists are omitted for clarity):
@@ -4545,12 +4600,12 @@
Type inference
- For a function call f(a0 , a1 , …) where
+ In a function call f(a0 , a1 , …) where
f or a function argument ai is
a generic function:
(ai , pi ) of corresponding
- function arguments and parameters where ai is not an
+ function arguments and parameters of fwhere ai is not an
untyped constant yields an equation
typeof(pi ) ≡A typeof(ai ).
Type inference
- For an assignment v = f of a generic function f to a
- (non-generic) variable v of function type:
- typeof(v) ≡A typeof(f).
-
-
-
-
- For a return statement return …, f, … where f is a
- generic function returned as a result to a (non-generic) result variable
- r of function type:
+ In a context where a generic function f must be
+ assignable to a (non-generic) function type T:
typeof(r) ≡A typeof(f).
+ typeof(f) ≡A T.
@@ -4873,7 +4919,7 @@ Operator precedence
x <= f() // x <= f()
^a >> b // (^a) >> b
f() || g() // f() || g()
-x == y+1 && <-chanInt > 0 // (x == (y+1)) && ((<-chanInt) > 0)
+x == y+1 && <-chanInt > 0 // (x == (y+1)) && ((<-chanInt) > 0)
@@ -6043,7 +6089,7 @@ Expression statements
-append cap complex imag len make new real
+append cap complex imag len make max min new real
unsafe.Add unsafe.Alignof unsafe.Offsetof unsafe.Sizeof unsafe.Slice unsafe.SliceData unsafe.String unsafe.StringData
@@ -6718,9 +6764,9 @@ For statements with range clause
map m map[K]V key k K m[k] V
channel c chan E, <-chan E element e E
integer value n integer type, or untyped int value i see below
-function, 0 values f func(func() bool)
-function, 1 value f func(func(V) bool) value v V
-function, 2 values f func(func(K, V) bool) key k K v V
+function, 0 values f func(yield func() bool)
+function, 1 value f func(yield func(V) bool) value v V yield cannot be variadic
+function, 2 values f func(yield func(K, V) bool) key k K v V yield cannot be variadic
@@ -6791,6 +6837,13 @@ For statements with range clause
+
+If the type of the range expression is a type parameter ,
+all types in its type set must have the same underlying type and the range expression must be valid
+for that type, or, if the type set contains channel types, it must only contain channel types with
+identical element types, and all channel types must permit receive operations.
+
+
The iteration variables may be declared by the "range" clause using a form of
short variable declaration
@@ -6883,7 +6936,7 @@
For statements with range clause
}
func (t *Tree[K, V]) walk(yield func(key K, val V) bool) bool {
- return t == nil || t.left.walk(yield) && yield(t.key, t.value) && t.right.walk(yield)
+ return t == nil || t.left.walk(yield) && yield(t.key, t.value) && t.right.walk(yield)
}
func (t *Tree[K, V]) Walk(yield func(key K, val V) bool) {
@@ -6895,14 +6948,16 @@ For statements with range clause
for k, v := range t.Walk {
// process k, v
}
-
-
-If the type of the range expression is a type parameter ,
-all types in its type set must have the same underlying type and the range expression must be valid
-for that type, or, if the type set contains channel types, it must only contain channel types with
-identical element types, and all channel types must permit receive operations.
-
+// xor returns the xor-ed bytes of S
+func xor[S ~[]byte](s S) byte {
+ var r byte
+ for _, b := range s {
+ r ^= b
+ }
+ return r
+}
+
Go statements
@@ -7383,8 +7438,8 @@ Appending to and copying slices
where E is the element type of S
and the respective parameter
passing rules apply.
-As a special case, append also accepts a first argument assignable
-to type []byte with a second argument of string type followed by
+As a special case, append also accepts a slice whose type is assignable to
+type []byte with a second argument of string type followed by
....
This form appends the bytes of the string.
@@ -7508,7 +7563,7 @@ Close
If the type of the argument to close is a
type parameter ,
-all types in its type set must be channels with the same element type.
+all types in its type set must be channels.
It is an error if any of those channels is a receive-only channel.
@@ -7806,40 +7861,32 @@ Min and max
Allocation
- The built-in function new creates a new, initialized
- variable and returns
- a pointer to it.
-
- It accepts a single argument, which may be either an expression or a type.
-
-
- If the argument expr is an expression of
- type T, or an untyped constant expression
- whose default type is T,
- then new(expr) allocates a variable of
- type T, initializes it to the value
- of expr, and returns its address, a value of
- type *T.
+The built-in function new creates a new, initialized
+variable and returns
+a pointer to it.
+It accepts a single argument, which may be either a type or an expression.
+
- If the argument is a type T, then new(T)
- allocates a variable initialized to
- the zero value of type T.
+If the argument is a type T, then new(T)
+allocates a variable of type T initialized to its
+zero value .
-
- For example, new(123) and new(int) each
- return a pointer to a new variable of type int.
- The value of the first variable is 123, and the value
- of the second is 0.
+
+If the argument is an expression x, then new(x)
+allocates a variable of the type of x initialized to the value of x.
+If that value is an untyped constant, it is first implicitly converted
+to its default type ;
+if it is an untyped boolean value, it is first implicitly converted to type bool.
+The predeclared identifier nil cannot be used as an argument to new.
-
-new(T)
-
-
-For instance
+For example, new(int) and new(123) each
+return a pointer to a new variable of type int.
+The value of the first variable is 0, and the value
+of the second is 123. Similarly
@@ -7848,13 +7895,12 @@ Allocation
-allocates storage for a variable of type S,
+allocates a variable of type S,
initializes it (a=0, b=0.0),
and returns a value of type *S containing the address
-of the location.
+of the variable.
-
Handling panics
Two built-in functions, panic and recover,
@@ -8330,7 +8376,7 @@
Package initialization
-Multiple such functions may be defined per package, even within a single
+Multiple such functions may be declared per package, even within a single
source file. In the package block, the init identifier can
be used only to declare init functions, yet the identifier
itself is not declared . Thus
@@ -8412,7 +8458,7 @@
Errors
It is the conventional interface for representing an error condition,
with the nil value representing no error.
-For instance, a function to read data from a file might be defined:
+For instance, a function to read data from a file might be declared:
@@ -8793,6 +8839,23 @@ Go 1.24
+Go 1.27
+
+
Type unification rules
diff --git a/doc/godebug.md b/doc/godebug.md
index c12ce5311d90d1..49b513c4cb559e 100644
--- a/doc/godebug.md
+++ b/doc/godebug.md
@@ -85,6 +85,9 @@ if the work module's `go.mod` or the workspace's `go.work`
says `go` `1.20`, then the program defaults to `panicnil=1`,
matching Go 1.20 instead of Go 1.21.
+As an exception, GODEBUGs introduced for security releases
+will have the new behavior apply to all versions.
+
Because this method of setting GODEBUG defaults was introduced only in Go 1.21,
programs listing versions of Go earlier than Go 1.20 are configured to match Go 1.20,
not the older version.
@@ -133,9 +136,7 @@ are also treated as invalid.
The defaults that will be compiled into a main package
are reported by the command:
-{{raw `
go list -f '{{.DefaultGODEBUG}}' my/main/package
-`}}
Only differences from the base Go toolchain defaults are reported.
@@ -153,16 +154,100 @@ for example,
see the [runtime documentation](/pkg/runtime#hdr-Environment_Variables)
and the [go command documentation](/cmd/go#hdr-Build_and_test_caching).
+### Go 1.28
+
+Go 1.28 changed
+[`net/http.FileServer`](/pkg/net/http#FileServer),
+[`net/http.FileServerFS`](/pkg/net/http#FileServerFS),
+[`net/http.ServeContent`](/pkg/net/http#ServeContent),
+[`net/http.ServeFile`](/pkg/net/http#ServeFile), and
+[`net/http.ServeFileFS`](/pkg/net/http#ServeFileFS) to
+limit the maximum number of ranges in a Range header.
+When the number of ranges in a header exceeds the new
+`httpservecontentmaxranges` setting, the header is ignored.
+The default value is `httpservecontentmaxranges=200`.
+Setting `httpservecontentmaxranges=0` disables the limit.
+To avoid denial of service attacks, this setting and default
+was backported to Go 1.27.2 and Go 1.26.9.
+
+### Go 1.27
+
+Go 1.27 removed the `gotypesalias` setting, as noted in the [Go 1.22](#go-122) section.
+
+Go 1.27 removed the `tlsunsafeekm` setting, as noted in the [Go 1.22](#go-122) section.
+
+Go 1.27 removed the `tlsrsakex` setting, as noted in the [Go 1.22](#go-122) section.
+
+Go 1.27 removed the `tls3des` setting, as noted in the [Go 1.23](#go-123) section.
+
+Go 1.27 removed the `tls10server` setting, as noted in the [Go 1.22](#go-122) section.
+
+Go 1.27 removed the `x509keypairleaf` setting, as noted in the [Go 1.23](#go-123) section.
+
+Go 1.27 removed the `asynctimerchan` setting, as noted in the [Go 1.23](#go-123) section.
+
+Go 1.27 added a new `htmlmetacontenturlescape` setting that controls whether
+html/template will escape URLs in the `url=` portion of the content attribute of
+HTML meta tags. The default `htmlmetacontentescape=1` will cause URLs to be
+escaped. Setting `htmlmetacontentescape=0` disables this behavior. To avoid
+content injection attacks, this setting and default was backported to Go 1.25.8
+and Go 1.26.1.
+
+Go 1.27 changes the default for `tracebacklabels` (added in [Go 1.26](#go-126))
+to `1`. This opt-out is expected to be kept indefinitely in case goroutine
+labels acquire sensitive information that shouldn't be made available in
+tracebacks.
+
+Go 1.27 added a new `x509sslcertoverrideplatform` setting that controls whether
+crypto/x509 will load roots from disk on Windows and Darwin when `SSL_CERT_FILE`
+or `SSL_CERT_DIR` are set. The default value `x509sslcertoverrideplatform=1` will
+cause roots to be loaded from disk when these environment variables are set.
+Setting `x509sslcertoverrideplatform=0` disables this behavior in favor of using
+the platform certificate store instead of honoring the environment variables. We
+plan to remove this setting in Go 1.31.
+
+Go 1.27 added a `fips140ems` setting that when set to `0` disables the
+enforcement of Extended Master Secret in FIPS 140-3 mode. There is no change in
+default behavior. This setting was backported to Go 1.26.6 and Go 1.25.13.
+We plan to remove this setting in Go 1.31.
+
### Go 1.26
Go 1.26 added a new `httpcookiemaxnum` setting that controls the maximum number
of cookies that net/http will accept when parsing HTTP headers. If the number of
-cookie in a header exceeds the number set in `httpcookiemaxnum`, cookie parsing
+cookies in a header exceeds the number set in `httpcookiemaxnum`, cookie parsing
will fail early. The default value is `httpcookiemaxnum=3000`. Setting
`httpcookiemaxnum=0` will allow the cookie parsing to accept an indefinite
number of cookies. To avoid denial of service attacks, this setting and default
was backported to Go 1.25.2 and Go 1.24.8.
+Go 1.26 added a new `urlmaxqueryparams` setting that controls the maximum number
+of query parameters that net/url will accept when parsing a URL-encoded query string.
+If the number of parameters exceeds the number set in `urlmaxqueryparams`,
+parsing will fail early. The default value is `urlmaxqueryparams=10000`.
+Setting `urlmaxqueryparams=0` disables the limit. To avoid denial of service
+attacks, this setting and default was backported to Go 1.25.6 and Go 1.24.12.
+
+Go 1.26 added a new `urlstrictcolons` setting that controls whether `net/url.Parse`
+allows malformed hostnames containing colons outside of a bracketed IPv6 address.
+The default `urlstrictcolons=1` rejects URLs such as `http://localhost:1:2` or `http://::1/`.
+Colons are permitted as part of a bracketed IPv6 address, such as `http://[::1]/`.
+
+Go 1.26 enabled two additional post-quantum key exchange mechanisms:
+SecP256r1MLKEM768 and SecP384r1MLKEM1024. The default can be reverted using the
+[`tlssecpmlkem` setting](/pkg/crypto/tls/#Config.CurvePreferences).
+
+Go 1.26 added a new `tracebacklabels` setting that controls the inclusion of
+goroutine labels set through the the `runtime/pprof` package. Setting `tracebacklabels=1`
+includes these key/value pairs in the goroutine status header of runtime
+tracebacks and debug=2 runtime/pprof stack dumps. This format may change in the future.
+(see go.dev/issue/76349)
+
+Go 1.26 added a new `cryptocustomrand` setting that controls whether most crypto/...
+APIs ignore the random `io.Reader` parameter. For Go 1.26, it defaults
+to `cryptocustomrand=0`, ignoring the random parameters. Using `cryptocustomrand=1`
+reverts to the pre-Go 1.26 behavior.
+
### Go 1.25
Go 1.25 added a new `decoratemappings` setting that controls whether the Go
@@ -285,8 +370,8 @@ Go 1.23 changed the channels created by package time to be unbuffered
(synchronous), which makes correct use of the [`Timer.Stop`](/pkg/time/#Timer.Stop)
and [`Timer.Reset`](/pkg/time/#Timer.Reset) method results much easier.
The [`asynctimerchan` setting](/pkg/time/#NewTimer) disables this change.
-There are no runtime metrics for this change,
-This setting may be removed in a future release, Go 1.27 at the earliest.
+There are no runtime metrics for this change.
+This setting will be removed in Go 1.27.
Go 1.23 changed the mode bits reported by [`os.Lstat`](/pkg/os#Lstat) and [`os.Stat`](/pkg/os#Stat)
for reparse points, which can be controlled with the `winsymlink` setting.
@@ -323,6 +408,7 @@ any effect.
Go 1.23 changed the default TLS cipher suites used by clients and servers when
not explicitly configured, removing 3DES cipher suites. The default can be reverted
using the [`tls3des` setting](/pkg/crypto/tls/#Config.CipherSuites).
+This setting will be removed in Go 1.27.
Go 1.23 changed the behavior of [`tls.X509KeyPair`](/pkg/crypto/tls#X509KeyPair)
and [`tls.LoadX509KeyPair`](/pkg/crypto/tls#LoadX509KeyPair) to populate the
@@ -330,6 +416,7 @@ Leaf field of the returned [`tls.Certificate`](/pkg/crypto/tls#Certificate).
This behavior is controlled by the `x509keypairleaf` setting. For Go 1.23, it
defaults to `x509keypairleaf=1`. Previous versions default to
`x509keypairleaf=0`.
+This setting will be removed in Go 1.27.
Go 1.23 changed
[`net/http.ServeContent`](/pkg/net/http#ServeContent),
@@ -363,21 +450,24 @@ Whether the type checker produces `Alias` types or not is controlled by the
[`gotypesalias` setting](/pkg/go/types#Alias).
For Go 1.22 it defaults to `gotypesalias=0`.
For Go 1.23, `gotypesalias=1` will become the default.
-This setting will be removed in a future release, Go 1.27 at the earliest.
+This setting will be removed in Go 1.27.
Go 1.22 changed the default minimum TLS version supported by both servers
and clients to TLS 1.2. The default can be reverted to TLS 1.0 using the
[`tls10server` setting](/pkg/crypto/tls/#Config).
+This setting will be removed in Go 1.27.
Go 1.22 changed the default TLS cipher suites used by clients and servers when
not explicitly configured, removing the cipher suites which used RSA based key
exchange. The default can be reverted using the [`tlsrsakex` setting](/pkg/crypto/tls/#Config).
+This setting will be removed in Go 1.27.
Go 1.22 disabled
[`ConnectionState.ExportKeyingMaterial`](/pkg/crypto/tls/#ConnectionState.ExportKeyingMaterial)
when the connection supports neither TLS 1.3 nor Extended Master Secret
(implemented in Go 1.21). It can be reenabled with the [`tlsunsafeekm`
setting](/pkg/crypto/tls/#ConnectionState.ExportKeyingMaterial).
+This setting will be removed in Go 1.27.
Go 1.22 changed how the runtime interacts with transparent huge pages on Linux.
In particular, a common default Linux kernel configuration can result in
diff --git a/doc/initial/6-stdlib/99-minor/0-heading.md b/doc/initial/6-stdlib/99-minor/0-heading.md
index e80ebc64c380fd..266d98f496a779 100644
--- a/doc/initial/6-stdlib/99-minor/0-heading.md
+++ b/doc/initial/6-stdlib/99-minor/0-heading.md
@@ -1,10 +1 @@
### Minor changes to the library {#minor_library_changes}
-
-#### go/types
-
-The `Var.Kind` method returns an enumeration of type `VarKind` that
-classifies the variable (package-level, local, receiver, parameter,
-result, or struct field). See issue #70250.
-
-Callers of `NewVar` or `NewParam` are encouraged to call `Var.SetKind`
-to ensure that this attribute is set correctly in all cases.
diff --git a/doc/next/1-intro.md b/doc/next/1-intro.md
deleted file mode 100644
index 3cd0d66b5ad4e3..00000000000000
--- a/doc/next/1-intro.md
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-## DRAFT RELEASE NOTES — Introduction to Go 1.26 {#introduction}
-
-**Go 1.26 is not yet released. These are work-in-progress release notes.
-Go 1.26 is expected to be released in February 2026.**
diff --git a/doc/next/2-language.md b/doc/next/2-language.md
deleted file mode 100644
index 71da62f59e59ab..00000000000000
--- a/doc/next/2-language.md
+++ /dev/null
@@ -1,32 +0,0 @@
-## Changes to the language {#language}
-
-
-
-The built-in `new` function, which creates a new variable, now allows
-its operand to be an expression, specifying the initial value of the
-variable.
-
-This feature is particularly useful when working with serialization
-packages such as `encoding/json` or protocol buffers that use a
-pointer to represent an optional value, as it enables an optional
-field to be populated in a simple expression, for example:
-
-```go
-import "encoding/json"
-
-type Person struct {
- Name string `json:"name"`
- Age *int `json:"age"` // age if known; nil otherwise
-}
-
-func personJSON(name string, born time.Time) ([]byte, error) {
- return json.Marshal(Person{
- Name: name,
- Age: new(yearsSince(born)),
- })
-}
-
-func yearsSince(t time.Time) int {
- return int(time.Since(t).Hours() / (365.25 * 24)) // approximately
-}
-```
diff --git a/doc/next/3-tools.md b/doc/next/3-tools.md
deleted file mode 100644
index c0a4601c0b9e74..00000000000000
--- a/doc/next/3-tools.md
+++ /dev/null
@@ -1,21 +0,0 @@
-## Tools {#tools}
-
-### Go command {#go-command}
-
-
-`cmd/doc`, and `go tool doc` have been deleted. `go doc` can be used as
-a replacement for `go tool doc`: it takes the same flags and arguments and
-has the same behavior.
-
-
-The `go fix` command, following the pattern of `go vet` in Go 1.10,
-now uses the Go analysis framework (`golang.org/x/tools/go/analysis`).
-This means the same analyzers that provide diagnostics in `go vet`
-can be used to suggest and apply fixes in `go fix`.
-The `go fix` command's historical fixers, all of which were obsolete,
-have been removed and replaced by a suite of new analyzers that
-offer fixes to use newer features of the language and library.
-
-
-### Cgo {#cgo}
-
diff --git a/doc/next/4-runtime.md b/doc/next/4-runtime.md
deleted file mode 100644
index 1f8e445e0b10de..00000000000000
--- a/doc/next/4-runtime.md
+++ /dev/null
@@ -1 +0,0 @@
-## Runtime {#runtime}
diff --git a/doc/next/5-toolchain.md b/doc/next/5-toolchain.md
deleted file mode 100644
index cc32f30a521a6d..00000000000000
--- a/doc/next/5-toolchain.md
+++ /dev/null
@@ -1,12 +0,0 @@
-## Compiler {#compiler}
-
-## Assembler {#assembler}
-
-## Linker {#linker}
-
-## Bootstrap {#bootstrap}
-
-
-As mentioned in the [Go 1.24 release notes](/doc/go1.24#bootstrap), Go 1.26 now requires
-Go 1.24.6 or later for bootstrap.
-We expect that Go 1.28 will require a minor release of Go 1.26 or later for bootstrap.
diff --git a/doc/next/6-stdlib/0-heading.md b/doc/next/6-stdlib/0-heading.md
deleted file mode 100644
index a992170d433326..00000000000000
--- a/doc/next/6-stdlib/0-heading.md
+++ /dev/null
@@ -1,2 +0,0 @@
-## Standard library {#library}
-
diff --git a/doc/next/6-stdlib/99-minor/0-heading.md b/doc/next/6-stdlib/99-minor/0-heading.md
deleted file mode 100644
index e80ebc64c380fd..00000000000000
--- a/doc/next/6-stdlib/99-minor/0-heading.md
+++ /dev/null
@@ -1,10 +0,0 @@
-### Minor changes to the library {#minor_library_changes}
-
-#### go/types
-
-The `Var.Kind` method returns an enumeration of type `VarKind` that
-classifies the variable (package-level, local, receiver, parameter,
-result, or struct field). See issue #70250.
-
-Callers of `NewVar` or `NewParam` are encouraged to call `Var.SetKind`
-to ensure that this attribute is set correctly in all cases.
diff --git a/doc/next/6-stdlib/99-minor/README b/doc/next/6-stdlib/99-minor/README
deleted file mode 100644
index fac778de050642..00000000000000
--- a/doc/next/6-stdlib/99-minor/README
+++ /dev/null
@@ -1 +0,0 @@
-API changes and other small changes to the standard library go here.
diff --git a/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md b/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md
deleted file mode 100644
index 81efc00bb5d657..00000000000000
--- a/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md
+++ /dev/null
@@ -1 +0,0 @@
-The `big.Int` fields of [PublicKey] and [PrivateKey] are now deprecated.
diff --git a/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md b/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md
deleted file mode 100644
index a3647a79f2efd8..00000000000000
--- a/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md
+++ /dev/null
@@ -1,5 +0,0 @@
-If [PrivateKey] fields are modified after calling [PrivateKey.Precompute],
-[PrivateKey.Validate] now fails.
-
-[PrivateKey.D] is now checked for consistency with precomputed values, even if
-it is not used.
diff --git a/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md b/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md
deleted file mode 100644
index 8cb9089583a89d..00000000000000
--- a/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md
+++ /dev/null
@@ -1 +0,0 @@
-A database driver may implement [RowsColumnScanner] to entirely override `Scan` behavior.
diff --git a/doc/next/6-stdlib/99-minor/errors/51945.md b/doc/next/6-stdlib/99-minor/errors/51945.md
deleted file mode 100644
index 44ac7222e6d990..00000000000000
--- a/doc/next/6-stdlib/99-minor/errors/51945.md
+++ /dev/null
@@ -1,2 +0,0 @@
-The new [AsType] function is a generic version of [As]. It is type-safe, faster,
-and, in most cases, easier to use.
diff --git a/doc/next/6-stdlib/99-minor/go/ast/68021.md b/doc/next/6-stdlib/99-minor/go/ast/68021.md
deleted file mode 100644
index 0ff1a0b11e8e0c..00000000000000
--- a/doc/next/6-stdlib/99-minor/go/ast/68021.md
+++ /dev/null
@@ -1,4 +0,0 @@
-The new [ParseDirective] function parses [directive
-comments](/doc/comment#Syntax), which are comments such as `//go:generate`.
-Source code tools can support their own directive comments and this new API
-should help them implement the conventional syntax.
diff --git a/doc/next/6-stdlib/99-minor/image/jpeg/75603.md b/doc/next/6-stdlib/99-minor/image/jpeg/75603.md
deleted file mode 100644
index 43421761fdad82..00000000000000
--- a/doc/next/6-stdlib/99-minor/image/jpeg/75603.md
+++ /dev/null
@@ -1,2 +0,0 @@
-The JPEG encoder and decoder have been replaced with new, faster, more accurate implementations.
-Code that expects specific bit-for-bit outputs from the encoder or decoder may need to be updated.
diff --git a/doc/next/6-stdlib/99-minor/log/slog/65954.md b/doc/next/6-stdlib/99-minor/log/slog/65954.md
deleted file mode 100644
index 631ed665df24a7..00000000000000
--- a/doc/next/6-stdlib/99-minor/log/slog/65954.md
+++ /dev/null
@@ -1,6 +0,0 @@
-The [`NewMultiHandler`](/pkg/log/slog#NewMultiHandler) function creates a
-[`MultiHandler`](/pkg/log/slog#MultiHandler) that invokes all the given Handlers.
-Its `Enable` method reports whether any of the handlers' `Enabled` methods
-return true.
-Its `Handle`, `WithAttr` and `WithGroup` methods call the corresponding method
-on each of the enabled handlers.
diff --git a/doc/next/6-stdlib/99-minor/net/49097.md b/doc/next/6-stdlib/99-minor/net/49097.md
deleted file mode 100644
index bb7947b0a11377..00000000000000
--- a/doc/next/6-stdlib/99-minor/net/49097.md
+++ /dev/null
@@ -1 +0,0 @@
-Added context aware dial functions for TCP, UDP, IP and Unix networks.
diff --git a/doc/next/6-stdlib/99-minor/net/http/67813.md b/doc/next/6-stdlib/99-minor/net/http/67813.md
deleted file mode 100644
index 74b8c7644f831e..00000000000000
--- a/doc/next/6-stdlib/99-minor/net/http/67813.md
+++ /dev/null
@@ -1,4 +0,0 @@
-The new
-[HTTP2Config.StrictMaxConcurrentRequests](/pkg/net/http#HTTP2Config.StrictMaxConcurrentRequests)
-field controls whether a new connection should be opened
-if an existing HTTP/2 connection has exceeded its stream limit.
diff --git a/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md b/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md
deleted file mode 100644
index ef6a4898f26a9d..00000000000000
--- a/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md
+++ /dev/null
@@ -1,2 +0,0 @@
-The HTTP client returned by [Server.Client] will now redirect requests for
-`example.com` and any subdomains to the server being tested.
diff --git a/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md b/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md
deleted file mode 100644
index f6318f85534746..00000000000000
--- a/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md
+++ /dev/null
@@ -1,11 +0,0 @@
-The [ReverseProxy.Director] configuration field is deprecated
-in favor of [ReverseProxy.Rewrite].
-
-A malicious client can remove headers added by a `Director` function
-by designating those headers as hop-by-hop. Since there is no way to address
-this problem within the scope of the `Director` API, we added a new
-`Rewrite` hook in Go 1.20. `Rewrite` hooks are provided with both the
-unmodified inbound request received by the proxy and the outbound request
-which will be sent by the proxy.
-
-Since the `Director` hook is fundamentally unsafe, we are now deprecating it.
diff --git a/doc/next/6-stdlib/99-minor/net/netip/61642.md b/doc/next/6-stdlib/99-minor/net/netip/61642.md
deleted file mode 100644
index 3d79f2e76aee04..00000000000000
--- a/doc/next/6-stdlib/99-minor/net/netip/61642.md
+++ /dev/null
@@ -1 +0,0 @@
-The new [Prefix.Compare] method compares two prefixes.
diff --git a/doc/next/6-stdlib/99-minor/os/70352.md b/doc/next/6-stdlib/99-minor/os/70352.md
deleted file mode 100644
index 5651639dadb499..00000000000000
--- a/doc/next/6-stdlib/99-minor/os/70352.md
+++ /dev/null
@@ -1,4 +0,0 @@
-The new [Process.WithHandle] method provides access to an internal process
-handle on supported platforms (Linux 5.4 or later and Windows). On Linux,
-the process handle is a pidfd. The method returns [ErrNoHandle] on unsupported
-platforms or when no process handle is available.
diff --git a/doc/next/6-stdlib/99-minor/os/73676.md b/doc/next/6-stdlib/99-minor/os/73676.md
deleted file mode 100644
index 70d01f262d86fc..00000000000000
--- a/doc/next/6-stdlib/99-minor/os/73676.md
+++ /dev/null
@@ -1,4 +0,0 @@
-On Windows, the [OpenFile] `flag` parameter can now contain any combination of
-Windows-specific file flags, such as `FILE_FLAG_OVERLAPPED` and
-`FILE_FLAG_SEQUENTIAL_SCAN`, for control of file or device caching behavior,
-access modes, and other special-purpose flags.
\ No newline at end of file
diff --git a/doc/next/6-stdlib/99-minor/testing/71287.md b/doc/next/6-stdlib/99-minor/testing/71287.md
deleted file mode 100644
index 82cac638101099..00000000000000
--- a/doc/next/6-stdlib/99-minor/testing/71287.md
+++ /dev/null
@@ -1,18 +0,0 @@
-The new methods [T.ArtifactDir], [B.ArtifactDir], and [F.ArtifactDir]
-return a directory in which to write test output files (artifacts).
-
-When the `-artifacts` flag is provided to `go test`,
-this directory will be located under the output directory
-(specified with `-outputdir`, or the current directory by default).
-Otherwise, artifacts are stored in a temporary directory
-which is removed after the test completes.
-
-The first call to `ArtifactDir` when `-artifacts` is provided
-writes the location of the directory to the test log.
-
-For example, in a test named `TestArtifacts`,
-`t.ArtifactDir()` emits:
-
-```
-=== ARTIFACTS Test /path/to/artifact/dir
-```
diff --git a/doc/next/7-ports.md b/doc/next/7-ports.md
deleted file mode 100644
index 07445454227bcb..00000000000000
--- a/doc/next/7-ports.md
+++ /dev/null
@@ -1,6 +0,0 @@
-## Ports {#ports}
-
-### Windows
-
-
-As [announced](/doc/go1.25#windows) in the Go 1.25 release notes, the [broken](/doc/go1.24#windows) 32-bit windows/arm port (`GOOS=windows` `GOARCH=arm`) is removed.
diff --git a/lib/fips140/certified.txt b/lib/fips140/certified.txt
new file mode 100644
index 00000000000000..efd3caba85ba62
--- /dev/null
+++ b/lib/fips140/certified.txt
@@ -0,0 +1 @@
+v1.0.0-c2097c7c
diff --git a/lib/fips140/fips140.sum b/lib/fips140/fips140.sum
index 703d1dc60e58ab..5dff95b1b62e9e 100644
--- a/lib/fips140/fips140.sum
+++ b/lib/fips140/fips140.sum
@@ -1,4 +1,7 @@
-# SHA256 checksums of snapshot zip files in this directory.
+# Checksums of snapshot zip files in this directory.
+# Each line is "NAME SHA256HEX H1HASH", where SHA256HEX is the SHA256
+# checksum of the zip file and H1HASH is the module zip hash of its
+# contents, as recorded in go.sum files and the module cache.
# These checksums are included in the FIPS security policy
# (validation instructions sent to the lab) and MUST NOT CHANGE.
# That is, the zip files themselves must not change.
@@ -9,4 +12,5 @@
#
# go test cmd/go/internal/fips140 -update
#
-v1.0.0-c2097c7c.zip daf3614e0406f67ae6323c902db3f953a1effb199142362a039e7526dfb9368b
+v1.0.0-c2097c7c.zip daf3614e0406f67ae6323c902db3f953a1effb199142362a039e7526dfb9368b h1:YapsF69QWGwygaBqdI0rp7IfE5a6tAwepNAHONVcWqQ=
+v1.26.0.zip 9b28f847fdf1db4a36cb2b2f8ec09443c039383f085630a03ecfaddf6db7ea23 h1:dtoPX1ALGGp4rMLzyh6oqIkYRXnXxRkpPWu56l5DFpM=
diff --git a/lib/fips140/inprocess.txt b/lib/fips140/inprocess.txt
index efd3caba85ba62..426c2fb87f5d08 100644
--- a/lib/fips140/inprocess.txt
+++ b/lib/fips140/inprocess.txt
@@ -1 +1 @@
-v1.0.0-c2097c7c
+v1.26.0
diff --git a/lib/fips140/v1.26.0.zip b/lib/fips140/v1.26.0.zip
new file mode 100644
index 00000000000000..f53ade80367a8b
Binary files /dev/null and b/lib/fips140/v1.26.0.zip differ
diff --git a/lib/hg/goreposum.py b/lib/hg/goreposum.py
new file mode 100644
index 00000000000000..1a7d7a444667a9
--- /dev/null
+++ b/lib/hg/goreposum.py
@@ -0,0 +1,64 @@
+# Copyright 2025 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Mercurial extension to add a 'goreposum' command that
+# computes a hash of a remote repo's tag state.
+# Tag definitions can come from the .hgtags file stored in
+# any head of any branch, and the server protocol does not
+# expose the tags directly. However, the protocol does expose
+# the hashes of all the branch heads, so we can use a hash of
+# all those branch names and heads as a conservative snapshot
+# of the entire remote repo state, and use that as the tag sum.
+# Any change on the server then invalidates the tag sum,
+# even if it didn't have anything to do with tags, but at least
+# we will avoid re-cloning a server when there have been no
+# changes at all.
+#
+# Separately, this extension also adds a 'golookup' command that
+# returns the hash of a specific reference, like 'default' or a tag.
+# And golookup of a hash confirms that it still exists on the server.
+# We can use that to revalidate that specific versions still exist and
+# have the same meaning they did the last time we checked.
+#
+# Usage:
+#
+# hg --config "extensions.goreposum=$GOROOT/lib/hg/goreposum.py" goreposum REPOURL
+
+import base64, hashlib, sys
+from mercurial import registrar, ui, hg, node
+from mercurial.i18n import _
+cmdtable = {}
+command = registrar.command(cmdtable)
+@command(b'goreposum', [], _('url'), norepo=True)
+def goreposum(ui, url):
+ """
+ goreposum computes a checksum of all the named state in the remote repo.
+ It hashes together all the branch names and hashes
+ and then all the bookmark names and hashes.
+ Tags are stored in .hgtags files in any of the branches,
+ so the branch metadata includes the tags as well.
+ """
+ h = hashlib.sha256()
+ peer = hg.peer(ui, {}, url)
+ for name, revs in peer.branchmap().items():
+ h.update(name)
+ for r in revs:
+ h.update(b' ')
+ h.update(r)
+ h.update(b'\n')
+ if (b'bookmarks' in peer.listkeys(b'namespaces')):
+ for name, rev in peer.listkeys(b'bookmarks').items():
+ h.update(name)
+ h.update(b'=')
+ h.update(rev)
+ h.update(b'\n')
+ print('r1:'+base64.standard_b64encode(h.digest()).decode('utf-8'))
+
+@command(b'golookup', [], _('url rev'), norepo=True)
+def golookup(ui, url, rev):
+ """
+ golookup looks up a single identifier in the repo,
+ printing its hash.
+ """
+ print(node.hex(hg.peer(ui, {}, url).lookup(rev)).decode('utf-8'))
diff --git a/lib/time/update.bash b/lib/time/update.bash
index 66494752ea3c4b..768c244cf0f374 100755
--- a/lib/time/update.bash
+++ b/lib/time/update.bash
@@ -24,8 +24,8 @@
# in the CL match the update.bash in the CL.
# Versions to use.
-CODE=2025b
-DATA=2025b
+CODE=2026c
+DATA=2026c
set -e
diff --git a/lib/time/zoneinfo.zip b/lib/time/zoneinfo.zip
index b107695d1ef042..46c90a2b806a92 100644
Binary files a/lib/time/zoneinfo.zip and b/lib/time/zoneinfo.zip differ
diff --git a/src/README.vendor b/src/README.vendor
index e6a58d5fb55505..156a81dad4c18d 100644
--- a/src/README.vendor
+++ b/src/README.vendor
@@ -39,12 +39,14 @@ Go from source and use that 'go' binary to update its source tree.
Requirements may be added, updated, and removed with 'go get'.
The vendor directory may be updated with 'go mod vendor'.
+Tree inconsistencies are reported by 'go test cmd/internal/moddeps'.
A typical sequence might be:
cd src # or src/cmd
go get golang.org/x/net@master
go mod tidy
go mod vendor
+ go test cmd/internal/moddeps
Use caution when passing '-u' to 'go get'. The '-u' flag updates
modules providing all transitively imported packages, not only
@@ -53,3 +55,22 @@ the module providing the target package.
Note that 'go mod vendor' only copies packages that are transitively
imported by packages in the current module. If a new package is needed,
it should be imported before running 'go mod vendor'.
+
+Go release cycle considerations
+===============================
+
+Applying changes to packages that are vendored follows the considerations
+written down at go.dev/s/release.
+
+When the Go tree is open for development, a specific change may be pulled in
+at any time that it is needed. During the release freeze, the bar for changes
+in vendored packages is the same as it is for changes in non-vendored packages.
+After a major release is out, minor Go releases follow a more involved process
+documented at go.dev/wiki/MinorReleases#cherry-pick-cls-for-vendored-golangorgx-packages.
+
+In addition to individual updates that happen on demand, all dependencies in
+the std and cmd modules are updated to their latest available versions at least
+twice during every major release cycle. This is done to avoid the possibility of
+some dependencies being left on very old versions and in turn make their eventual
+update more disruptive. This recurring process is tracked in go.dev/issue/36905.
+The golang.org/x/build/cmd/updatestd command exists to assist with that process.
diff --git a/src/archive/tar/format.go b/src/archive/tar/format.go
index 9954b4d9f5548a..32e58a9d9b4e1e 100644
--- a/src/archive/tar/format.go
+++ b/src/archive/tar/format.go
@@ -147,6 +147,12 @@ const (
// Max length of a special file (PAX header, GNU long name or link).
// This matches the limit used by libarchive.
maxSpecialFileSize = 1 << 20
+
+ // Maximum number of sparse file entries.
+ // We should never actually hit this limit
+ // (every sparse encoding will first be limited by maxSpecialFileSize),
+ // but this adds an additional layer of defense.
+ maxSparseFileEntries = 1 << 20
)
// blockPadding computes the number of bytes needed to pad offset up to the
diff --git a/src/archive/tar/reader.go b/src/archive/tar/reader.go
index 16ac2f5b17c28b..e426c72ffe9cc3 100644
--- a/src/archive/tar/reader.go
+++ b/src/archive/tar/reader.go
@@ -45,8 +45,9 @@ func NewReader(r io.Reader) *Reader {
// Any remaining data in the current file is automatically discarded.
// At the end of the archive, Next returns the error io.EOF.
//
-// If Next encounters a non-local name (as defined by [filepath.IsLocal])
+// If Next encounters a non-local file name (as defined by [filepath.IsLocal])
// and the GODEBUG environment variable contains `tarinsecurepath=0`,
+// Only file names are validated, not link targets.
// Next returns the header with an [ErrInsecurePath] error.
// A future version of Go may introduce this behavior by default.
// Programs that want to accept non-local names can ignore
@@ -490,7 +491,8 @@ func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, err
}
s := blk.toGNU().sparse()
spd := make(sparseDatas, 0, s.maxEntries())
- for {
+ totalSize := len(s)
+ for totalSize < maxSpecialFileSize {
for i := 0; i < s.maxEntries(); i++ {
// This termination condition is identical to GNU and BSD tar.
if s.entry(i).offset()[0] == 0x00 {
@@ -501,7 +503,11 @@ func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, err
if p.err != nil {
return nil, p.err
}
- spd = append(spd, sparseEntry{Offset: offset, Length: length})
+ var err error
+ spd, err = appendSparseEntry(spd, sparseEntry{Offset: offset, Length: length})
+ if err != nil {
+ return nil, err
+ }
}
if s.isExtended()[0] > 0 {
@@ -510,10 +516,12 @@ func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, err
return nil, err
}
s = blk.toSparse()
+ totalSize += len(s)
continue
}
return spd, nil // Done
}
+ return nil, errSparseTooLong
}
// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format
@@ -586,7 +594,10 @@ func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) {
if err1 != nil || err2 != nil {
return nil, ErrHeader
}
- spd = append(spd, sparseEntry{Offset: offset, Length: length})
+ spd, err = appendSparseEntry(spd, sparseEntry{Offset: offset, Length: length})
+ if err != nil {
+ return nil, err
+ }
}
return spd, nil
}
@@ -620,12 +631,22 @@ func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) {
if err1 != nil || err2 != nil {
return nil, ErrHeader
}
- spd = append(spd, sparseEntry{Offset: offset, Length: length})
+ spd, err = appendSparseEntry(spd, sparseEntry{Offset: offset, Length: length})
+ if err != nil {
+ return nil, err
+ }
sparseMap = sparseMap[2:]
}
return spd, nil
}
+func appendSparseEntry(spd sparseDatas, ent sparseEntry) (sparseDatas, error) {
+ if len(spd) >= maxSparseFileEntries {
+ return nil, errSparseTooLong
+ }
+ return append(spd, ent), nil
+}
+
// Read reads from the current file in the tar archive.
// It returns (0, io.EOF) when it reaches the end of that file,
// until [Next] is called to advance to the next file.
diff --git a/src/archive/tar/reader_test.go b/src/archive/tar/reader_test.go
index fca53dae741bd5..a324674cb7bb19 100644
--- a/src/archive/tar/reader_test.go
+++ b/src/archive/tar/reader_test.go
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"hash/crc32"
+ "internal/obscuretestdata"
"io"
"maps"
"math"
@@ -25,10 +26,11 @@ import (
func TestReader(t *testing.T) {
vectors := []struct {
- file string // Test input file
- headers []*Header // Expected output headers
- chksums []string // CRC32 checksum of files, leave as nil if not checked
- err error // Expected error to occur
+ file string // Test input file
+ obscured bool // Obscured with obscuretestdata package
+ headers []*Header // Expected output headers
+ chksums []string // CRC32 checksum of files, leave as nil if not checked
+ err error // Expected error to occur
}{{
file: "testdata/gnu.tar",
headers: []*Header{{
@@ -523,8 +525,9 @@ func TestReader(t *testing.T) {
file: "testdata/pax-nul-path.tar",
err: ErrHeader,
}, {
- file: "testdata/neg-size.tar",
- err: ErrHeader,
+ file: "testdata/neg-size.tar.base64",
+ obscured: true,
+ err: ErrHeader,
}, {
file: "testdata/issue10968.tar",
err: ErrHeader,
@@ -629,15 +632,24 @@ func TestReader(t *testing.T) {
}}
for _, v := range vectors {
- t.Run(path.Base(v.file), func(t *testing.T) {
- f, err := os.Open(v.file)
+ t.Run(strings.TrimSuffix(path.Base(v.file), ".base64"), func(t *testing.T) {
+ path := v.file
+ if v.obscured {
+ tf, err := obscuretestdata.DecodeToTempFile(path)
+ if err != nil {
+ t.Fatalf("obscuredtestdata.DecodeToTempFile(%s): %v", path, err)
+ }
+ path = tf
+ }
+
+ f, err := os.Open(path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer f.Close()
var fr io.Reader = f
- if strings.HasSuffix(v.file, ".bz2") {
+ if strings.HasSuffix(v.file, ".bz2") || strings.HasSuffix(v.file, ".bz2.base64") {
fr = bzip2.NewReader(fr)
}
@@ -775,7 +787,7 @@ type readBadSeeker struct{ io.ReadSeeker }
func (rbs *readBadSeeker) Seek(int64, int) (int64, error) { return 0, fmt.Errorf("illegal seek") }
-// TestReadTruncation test the ending condition on various truncated files and
+// TestReadTruncation tests the ending condition on various truncated files and
// that truncated files are still detected even if the underlying io.Reader
// satisfies io.Seeker.
func TestReadTruncation(t *testing.T) {
@@ -1133,6 +1145,17 @@ func TestReadOldGNUSparseMap(t *testing.T) {
input: makeInput(FormatGNU, "",
makeSparseStrings(sparseDatas{{10 << 30, 512}, {20 << 30, 512}})...),
wantMap: sparseDatas{{10 << 30, 512}, {20 << 30, 512}},
+ }, {
+ input: makeInput(FormatGNU, "",
+ makeSparseStrings(func() sparseDatas {
+ var datas sparseDatas
+ // This is more than enough entries to exceed our limit.
+ for i := range int64(1 << 20) {
+ datas = append(datas, sparseEntry{i * 2, (i * 2) + 1})
+ }
+ return datas
+ }())...),
+ wantErr: errSparseTooLong,
}}
for i, v := range vectors {
diff --git a/src/archive/tar/stat_unix.go b/src/archive/tar/stat_unix.go
index f999f56db6c915..891a1a3b4a3e30 100644
--- a/src/archive/tar/stat_unix.go
+++ b/src/archive/tar/stat_unix.go
@@ -19,7 +19,7 @@ func init() {
sysStat = statUnix
}
-// userMap and groupMap caches UID and GID lookups for performance reasons.
+// userMap and groupMap cache UID and GID lookups for performance reasons.
// The downside is that renaming uname or gname by the OS never takes effect.
var userMap, groupMap sync.Map // map[int]string
diff --git a/src/archive/tar/strconv.go b/src/archive/tar/strconv.go
index 217efe9e2ee7f4..d3c28a8c4e3bce 100644
--- a/src/archive/tar/strconv.go
+++ b/src/archive/tar/strconv.go
@@ -312,7 +312,7 @@ func formatPAXRecord(k, v string) (string, error) {
// "%d %s=%s\n" % (size, key, value)
//
// Keys and values should be UTF-8, but the number of bad writers out there
-// forces us to be a more liberal.
+// forces us to be more liberal.
// Thus, we only reject all keys with NUL, and only reject NULs in values
// for the PAX version of the USTAR string fields.
// The key must not contain an '=' character.
diff --git a/src/archive/tar/testdata/gnu-sparse-big.tar b/src/archive/tar/testdata/gnu-sparse-big.tar
deleted file mode 100644
index 1a5cfc96d92983..00000000000000
Binary files a/src/archive/tar/testdata/gnu-sparse-big.tar and /dev/null differ
diff --git a/src/archive/tar/testdata/gnu-sparse-big.tar.base64 b/src/archive/tar/testdata/gnu-sparse-big.tar.base64
new file mode 100644
index 00000000000000..c25505083b1304
--- /dev/null
+++ b/src/archive/tar/testdata/gnu-sparse-big.tar.base64
@@ -0,0 +1,90 @@
+Z251LXNwYXJzZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDAwMDAAMDAwMDAw
+MAAwMDAwMDAwADAwMDAwMDA2MDAwADAwMDAwMDAwMDAwADAyMjIyNwAgUwAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhciAgAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwMDAwADAwMDAw
+MDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAlQL4gAw
+MDAwMDAwMTAwMACAAAAAAAAABKgXxgAwMDAwMDAwMTAwMACAAAAAAAAABvwjqgAwMDAwMDAwMTAw
+MACAAAAAAAAACVAvjgAwMDAwMDAwMTAwMAABgAAAAAAAAA34R1gAAAAAAAAAAAAAAAAAAAAAAACA
+AAAAAAAAC6Q7cgAwMDAwMDAwMTAwMACAAAAAAAAADfhHVgAwMDAwMDAwMTAwMAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1
+Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5AAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAADAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2
+Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAMDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3
+ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OQAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4
+OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5AAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAADAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5
+MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAMDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw
+MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OQAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
diff --git a/src/archive/tar/testdata/neg-size.tar b/src/archive/tar/testdata/neg-size.tar
deleted file mode 100644
index 21edf38cc3c3d9..00000000000000
Binary files a/src/archive/tar/testdata/neg-size.tar and /dev/null differ
diff --git a/src/archive/tar/testdata/neg-size.tar.base64 b/src/archive/tar/testdata/neg-size.tar.base64
new file mode 100644
index 00000000000000..b7e26a05b1e1ce
--- /dev/null
+++ b/src/archive/tar/testdata/neg-size.tar.base64
@@ -0,0 +1,9 @@
+EwMwMBMDLTgyMTk1MDI5NnQTE4NzfINzEzAwcXfhZrsDMDAwAAAAEDAxMRNz9DEwMTAwdBMTg3N8
+g3NzADATc3yDc3P0eFMTc/QxMDEwMHQTE4NzfINzc/QwE3N8g3Nz9HFTMNR0MBMwMHEw9DAAAAAQ
+MDGAABAwMTETc/QxMDH0MHQTMDBx1OFmuwMAAAD/gICAADExNTYyMQAgMBP///+AMTAwdHhTMDB0
+MBMwMHF3MDEwMTAwdBMTg3N8g3Nz9HhTMDB0MBMwMHF34Wa7AzAwMAAAABAwMTETc/QxMDEwMHQT
+E4NzfINzc/QwE3N8g3Nz9HhTE3P0MTAxMDB0ExODc3yDc3MwMBNzfINzczB4UzAwdDATMDBxMDAA
+gAAAEDAxc/QxMDEwMHQTAAAAIOFmuwMwNAAAABAwMTET////gDEwMHR4UzAwdDATMDBxd+FmuwMw
+MDAAAAAQMDExE3ODc3P0eFMTc/QxMDEwMHQTE4NzfINzc/QzMTEwMzM2MjQ4NDYxMjgzODBzfINz
+c/R4UzAwdDATMDBxMDAwAAAAEDAxAAAQMDExE3P0MTAxMDB0EzAwcdThZrsDMDQAAAAQg3N8g3Nz
+9DATc3yDc3P0eFMwMHQwEzAwcTAwMAAAABAwMQAAEDAxMRNz9DEwMTAwdBMwMHgw4Wa7AwAAEDA=
diff --git a/src/archive/tar/testdata/pax-sparse-big.tar b/src/archive/tar/testdata/pax-sparse-big.tar
deleted file mode 100644
index 65d1f8eceb0849..00000000000000
Binary files a/src/archive/tar/testdata/pax-sparse-big.tar and /dev/null differ
diff --git a/src/archive/tar/testdata/pax-sparse-big.tar.base64 b/src/archive/tar/testdata/pax-sparse-big.tar.base64
new file mode 100644
index 00000000000000..640ef32159fd48
--- /dev/null
+++ b/src/archive/tar/testdata/pax-sparse-big.tar.base64
@@ -0,0 +1,108 @@
+UGF4SGVhZGVycy4wL3BheC1zcGFyc2UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDAwMDAAMDAwMDAw
+MAAwMDAwMDAwADAwMDAwMDAwMTcyADAwMDAwMDAwMDAwADAxMjIyNwAgeAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhcgAwMAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAy
+MiBHTlUuc3BhcnNlLm1ham9yPTEKMjIgR05VLnNwYXJzZS5taW5vcj0wCjMwIEdOVS5zcGFyc2Uu
+bmFtZT1wYXgtc3BhcnNlCjM1IEdOVS5zcGFyc2UucmVhbHNpemU9NjAwMDAwMDAwMDAKMTMgc2l6
+ZT0zNTg0CgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEdO
+VVNwYXJzZUZpbGUuMC9wYXgtc3BhcnNlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwMDAwADAwMDAwMDAA
+MDAwMDAwMAAwMDAwMDAwNzAwMAAwMDAwMDAwMDAwMAAwMTM3MzcAIDAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdXN0YXIAMDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDAwMDAwMAAwMDAwMDAw
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANgo5
+OTk5OTk5NDg4CjUxMgoxOTk5OTk5OTQ4OAo1MTIKMjk5OTk5OTk0ODgKNTEyCjM5OTk5OTk5NDg4
+CjUxMgo0OTk5OTk5OTQ4OAo1MTIKNTk5OTk5OTk0ODgKNTEyCgAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAMDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3
+ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OQAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4
+OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5AAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAADAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5
+MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAMDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw
+MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OQAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAx
+MjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5AAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAADAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEy
+MzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
diff --git a/src/archive/tar/testdata/writer-big-long.tar b/src/archive/tar/testdata/writer-big-long.tar
deleted file mode 100644
index 09fc5dd3dd7fc5..00000000000000
Binary files a/src/archive/tar/testdata/writer-big-long.tar and /dev/null differ
diff --git a/src/archive/tar/testdata/writer-big-long.tar.base64 b/src/archive/tar/testdata/writer-big-long.tar.base64
new file mode 100644
index 00000000000000..171d2870685f39
--- /dev/null
+++ b/src/archive/tar/testdata/writer-big-long.tar.base64
@@ -0,0 +1,27 @@
+bG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9u
+Z25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbG9uZ25hbWUvbDAwMDAwMDAAMDAwMDAw
+MAAwMDAwMDAwADAwMDAwMDAwMjU2ADAwMDAwMDAwMDAwADAzMTQyMAAgeAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhcgAwMAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAx
+NTQgcGF0aD1sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25n
+bmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFt
+ZS9sb25nbmFtZS9sb25nbmFtZS9sb25nbmFtZS8xNmdpZy50eHQKMjAgc2l6ZT0xNzE3OTg2OTE4
+NAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGxv
+bmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmdu
+YW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2xvbmduYW1lL2wwMDAwNjQ0ADAwMDE3NTAA
+MDAwMTc1MAAwMDAwMDAwMDAwMAAxMjMzMjc3MDUwNwAwMzY0NjIAIDAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdXN0YXIAMDBndWlsbGF1bWUAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAGd1aWxsYXVtZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDAwMDAwMAAwMDAwMDAw
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go
index 9e484432ea90b6..759000f6b9569a 100644
--- a/src/archive/tar/writer_test.go
+++ b/src/archive/tar/writer_test.go
@@ -8,6 +8,7 @@ import (
"bytes"
"encoding/hex"
"errors"
+ "internal/obscuretestdata"
"io"
"io/fs"
"maps"
@@ -74,8 +75,9 @@ func TestWriter(t *testing.T) {
)
vectors := []struct {
- file string // Optional filename of expected output
- tests []testFnc
+ file string // Optional filename of expected output
+ obscured bool // Whether file is obscured
+ tests []testFnc
}{{
// The writer test file was produced with this command:
// tar (GNU tar) 1.26
@@ -151,7 +153,8 @@ func TestWriter(t *testing.T) {
// bsdtar -xvf writer-big-long.tar
//
// This file is in PAX format.
- file: "testdata/writer-big-long.tar",
+ file: "testdata/writer-big-long.tar.base64",
+ obscured: true,
tests: []testFnc{
testHeader{Header{
Typeflag: TypeReg,
@@ -393,7 +396,8 @@ func TestWriter(t *testing.T) {
testClose{},
},
}, {
- file: "testdata/gnu-sparse-big.tar",
+ file: "testdata/gnu-sparse-big.tar.base64",
+ obscured: true,
tests: []testFnc{
testHeader{Header{
Typeflag: TypeGNUSparse,
@@ -425,7 +429,8 @@ func TestWriter(t *testing.T) {
testClose{nil},
},
}, {
- file: "testdata/pax-sparse-big.tar",
+ file: "testdata/pax-sparse-big.tar.base64",
+ obscured: true,
tests: []testFnc{
testHeader{Header{
Typeflag: TypeReg,
@@ -483,7 +488,7 @@ func TestWriter(t *testing.T) {
return x == y
}
for _, v := range vectors {
- t.Run(path.Base(v.file), func(t *testing.T) {
+ t.Run(strings.TrimSuffix(path.Base(v.file), ".base64"), func(t *testing.T) {
const maxSize = 10 << 10 // 10KiB
buf := new(bytes.Buffer)
tw := NewWriter(iotest.TruncateWriter(buf, maxSize))
@@ -522,7 +527,16 @@ func TestWriter(t *testing.T) {
}
if v.file != "" {
- want, err := os.ReadFile(v.file)
+ path := v.file
+ if v.obscured {
+ tf, err := obscuretestdata.DecodeToTempFile(path)
+ if err != nil {
+ t.Fatalf("obscuretestdata.DecodeToTempFile(%s): %v", path, err)
+ }
+ path = tf
+ }
+
+ want, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile() = %v, want nil", err)
}
diff --git a/src/archive/zip/reader.go b/src/archive/zip/reader.go
index 6b57f767fc849d..8741456d99e9cc 100644
--- a/src/archive/zip/reader.go
+++ b/src/archive/zip/reader.go
@@ -63,7 +63,6 @@ type File struct {
zip *Reader
zipr io.ReaderAt
headerOffset int64 // includes overall ZIP archive baseOffset
- zip64 bool // zip64 extended information extra field presence
}
// OpenReader will open the Zip file specified by name and return a ReadCloser.
@@ -406,10 +405,6 @@ func readDirectoryHeader(f *File, r io.Reader) error {
f.NonUTF8 = f.Flags&0x800 == 0
}
- needUSize := f.UncompressedSize == ^uint32(0)
- needCSize := f.CompressedSize == ^uint32(0)
- needHeaderOffset := f.headerOffset == int64(^uint32(0))
-
// Best effort to find what we need.
// Other zip authors might not even follow the basic format,
// and we'll just ignore the Extra content in that case.
@@ -425,28 +420,23 @@ parseExtras:
switch fieldTag {
case zip64ExtraID:
- f.zip64 = true
-
// update directory values from the zip64 extra block.
// They should only be consulted if the sizes read earlier
// are maxed out.
- // See golang.org/issue/13367.
- if needUSize {
- needUSize = false
+ // See go.dev/issue/13367 and go.dev/issue/31692.
+ if f.UncompressedSize == ^uint32(0) {
if len(fieldBuf) < 8 {
return ErrFormat
}
f.UncompressedSize64 = fieldBuf.uint64()
}
- if needCSize {
- needCSize = false
+ if f.CompressedSize == ^uint32(0) {
if len(fieldBuf) < 8 {
return ErrFormat
}
f.CompressedSize64 = fieldBuf.uint64()
}
- if needHeaderOffset {
- needHeaderOffset = false
+ if f.headerOffset == int64(^uint32(0)) {
if len(fieldBuf) < 8 {
return ErrFormat
}
@@ -509,20 +499,6 @@ parseExtras:
}
}
- // Assume that uncompressed size 2³²-1 could plausibly happen in
- // an old zip32 file that was sharding inputs into the largest chunks
- // possible (or is just malicious; search the web for 42.zip).
- // If needUSize is true still, it means we didn't see a zip64 extension.
- // As long as the compressed size is not also 2³²-1 (implausible)
- // and the header is not also 2³²-1 (equally implausible),
- // accept the uncompressed size 2³²-1 as valid.
- // If nothing else, this keeps archive/zip working with 42.zip.
- _ = needUSize
-
- if needCSize || needHeaderOffset {
- return ErrFormat
- }
-
return nil
}
@@ -605,7 +581,7 @@ func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, baseOffset
d.comment = string(b[:l])
// These values mean that the file can be a zip64 file
- if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
+ if d.directoryRecords == 0xffff || d.directorySize == 0xffffffff || d.directoryOffset == 0xffffffff {
p, err := findDirectory64End(r, directoryEndOffset)
if err == nil && p >= 0 {
directoryEndOffset = p
@@ -834,7 +810,16 @@ func (r *Reader) initFileList() {
continue
}
- for dir := path.Dir(name); dir != "."; dir = path.Dir(dir) {
+ dir := name
+ for {
+ if idx := strings.LastIndex(dir, "/"); idx < 0 {
+ break
+ } else {
+ dir = dir[:idx]
+ }
+ if dirs[dir] {
+ break
+ }
dirs[dir] = true
}
diff --git a/src/archive/zip/reader_test.go b/src/archive/zip/reader_test.go
index cb8a0c28714eff..5ce994e4dd95c5 100644
--- a/src/archive/zip/reader_test.go
+++ b/src/archive/zip/reader_test.go
@@ -9,6 +9,7 @@ import (
"encoding/binary"
"encoding/hex"
"errors"
+ "fmt"
"internal/obscuretestdata"
"io"
"io/fs"
@@ -1874,3 +1875,83 @@ func TestBaseOffsetPlusOverflow(t *testing.T) {
// as the section reader offset & size were < 0.
NewReader(bytes.NewReader(data), int64(len(data))+1875)
}
+
+func BenchmarkReaderOneDeepDir(b *testing.B) {
+ var buf bytes.Buffer
+ zw := NewWriter(&buf)
+
+ for i := range 4000 {
+ name := strings.Repeat("a/", i) + "data"
+ zw.CreateHeader(&FileHeader{
+ Name: name,
+ Method: Store,
+ })
+ }
+
+ if err := zw.Close(); err != nil {
+ b.Fatal(err)
+ }
+ data := buf.Bytes()
+
+ for b.Loop() {
+ zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ b.Fatal(err)
+ }
+ zr.Open("does-not-exist")
+ }
+}
+
+func BenchmarkReaderManyDeepDirs(b *testing.B) {
+ var buf bytes.Buffer
+ zw := NewWriter(&buf)
+
+ for i := range 2850 {
+ name := fmt.Sprintf("%x", i)
+ name = strings.Repeat("/"+name, i+1)[1:]
+
+ zw.CreateHeader(&FileHeader{
+ Name: name,
+ Method: Store,
+ })
+ }
+
+ if err := zw.Close(); err != nil {
+ b.Fatal(err)
+ }
+ data := buf.Bytes()
+
+ for b.Loop() {
+ zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ b.Fatal(err)
+ }
+ zr.Open("does-not-exist")
+ }
+}
+
+func BenchmarkReaderManyShallowFiles(b *testing.B) {
+ var buf bytes.Buffer
+ zw := NewWriter(&buf)
+
+ for i := range 310000 {
+ name := fmt.Sprintf("%v", i)
+ zw.CreateHeader(&FileHeader{
+ Name: name,
+ Method: Store,
+ })
+ }
+
+ if err := zw.Close(); err != nil {
+ b.Fatal(err)
+ }
+ data := buf.Bytes()
+
+ for b.Loop() {
+ zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ b.Fatal(err)
+ }
+ zr.Open("does-not-exist")
+ }
+}
diff --git a/src/archive/zip/struct.go b/src/archive/zip/struct.go
index 3ebf39f5a694a2..f09323a581dade 100644
--- a/src/archive/zip/struct.go
+++ b/src/archive/zip/struct.go
@@ -156,7 +156,11 @@ type FileHeader struct {
// UncompressedSize64 is the uncompressed size of the file in bytes.
UncompressedSize64 uint64
- Extra []byte
+ // Extra are the extensible data fields. The writer automatically includes
+ // the appropriate Zip64 field if necessary, and [Writer.Close] appends the
+ // Central Directory version of the Zip64 field to Extra.
+ Extra []byte
+
ExternalAttrs uint32 // Meaning depends on CreatorVersion
}
@@ -337,11 +341,6 @@ func (h *FileHeader) SetMode(mode fs.FileMode) {
}
}
-// isZip64 reports whether the file size exceeds the 32 bit limit
-func (h *FileHeader) isZip64() bool {
- return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max
-}
-
func (h *FileHeader) hasDataDescriptor() bool {
return h.Flags&0x8 != 0
}
diff --git a/src/archive/zip/testdata/zip64/go126-deflate-zeros-5g.zsparse b/src/archive/zip/testdata/zip64/go126-deflate-zeros-5g.zsparse
new file mode 100644
index 00000000000000..8bd977eaaa2c09
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-deflate-zeros-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-offset-eq-4g.zsparse b/src/archive/zip/testdata/zip64/go126-offset-eq-4g.zsparse
new file mode 100644
index 00000000000000..43a66fb35e7ec3
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-offset-eq-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-offset-past-4g.zsparse b/src/archive/zip/testdata/zip64/go126-offset-past-4g.zsparse
new file mode 100644
index 00000000000000..02884925ae4f95
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-offset-past-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-store-4g-minus-1.zsparse b/src/archive/zip/testdata/zip64/go126-store-4g-minus-1.zsparse
new file mode 100644
index 00000000000000..14d53d243b885c
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-store-4g-minus-1.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-store-4g-minus-2.zsparse b/src/archive/zip/testdata/zip64/go126-store-4g-minus-2.zsparse
new file mode 100644
index 00000000000000..c145edfa426d1c
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-store-4g-minus-2.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-store-5g.zsparse b/src/archive/zip/testdata/zip64/go126-store-5g.zsparse
new file mode 100644
index 00000000000000..f2cd1a1229cda5
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-store-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-store-exact-4g.zsparse b/src/archive/zip/testdata/zip64/go126-store-exact-4g.zsparse
new file mode 100644
index 00000000000000..04111a2e33cc8a
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-store-exact-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/go126-store-just-under-4g.zsparse b/src/archive/zip/testdata/zip64/go126-store-just-under-4g.zsparse
new file mode 100644
index 00000000000000..6cdbf73bb77605
Binary files /dev/null and b/src/archive/zip/testdata/zip64/go126-store-just-under-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-deflate-zeros-5g.zsparse b/src/archive/zip/testdata/zip64/infozip-deflate-zeros-5g.zsparse
new file mode 100644
index 00000000000000..978eb347c8bba2
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-deflate-zeros-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-offset-eq-4g.zsparse b/src/archive/zip/testdata/zip64/infozip-offset-eq-4g.zsparse
new file mode 100644
index 00000000000000..5ac3b232a3b0c2
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-offset-eq-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-offset-past-4g.zsparse b/src/archive/zip/testdata/zip64/infozip-offset-past-4g.zsparse
new file mode 100644
index 00000000000000..5963ad1d5a0865
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-offset-past-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-store-4g-minus-1.zsparse b/src/archive/zip/testdata/zip64/infozip-store-4g-minus-1.zsparse
new file mode 100644
index 00000000000000..180bb49e729a5d
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-store-4g-minus-1.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-store-4g-minus-2.zsparse b/src/archive/zip/testdata/zip64/infozip-store-4g-minus-2.zsparse
new file mode 100644
index 00000000000000..c28800ca4b3d4b
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-store-4g-minus-2.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-store-5g.zsparse b/src/archive/zip/testdata/zip64/infozip-store-5g.zsparse
new file mode 100644
index 00000000000000..57273005e8732c
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-store-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-store-exact-4g.zsparse b/src/archive/zip/testdata/zip64/infozip-store-exact-4g.zsparse
new file mode 100644
index 00000000000000..6129c31f822137
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-store-exact-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/infozip-store-just-under-4g.zsparse b/src/archive/zip/testdata/zip64/infozip-store-just-under-4g.zsparse
new file mode 100644
index 00000000000000..9d653d2a5b7f68
Binary files /dev/null and b/src/archive/zip/testdata/zip64/infozip-store-just-under-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-deflate-zeros-5g.zsparse b/src/archive/zip/testdata/zip64/libarchive-deflate-zeros-5g.zsparse
new file mode 100644
index 00000000000000..9c8beb4a912bb0
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-deflate-zeros-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-offset-eq-4g.zsparse b/src/archive/zip/testdata/zip64/libarchive-offset-eq-4g.zsparse
new file mode 100644
index 00000000000000..f46d5bbb2abcfc
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-offset-eq-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-offset-past-4g.zsparse b/src/archive/zip/testdata/zip64/libarchive-offset-past-4g.zsparse
new file mode 100644
index 00000000000000..c0cb23be17f560
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-offset-past-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-1.zsparse b/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-1.zsparse
new file mode 100644
index 00000000000000..c70aaa1d8486e7
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-1.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-2.zsparse b/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-2.zsparse
new file mode 100644
index 00000000000000..9c45089434f8fd
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-store-4g-minus-2.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-store-5g.zsparse b/src/archive/zip/testdata/zip64/libarchive-store-5g.zsparse
new file mode 100644
index 00000000000000..23ee5242a48a4b
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-store-5g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-store-exact-4g.zsparse b/src/archive/zip/testdata/zip64/libarchive-store-exact-4g.zsparse
new file mode 100644
index 00000000000000..b123767e9427ff
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-store-exact-4g.zsparse differ
diff --git a/src/archive/zip/testdata/zip64/libarchive-store-just-under-4g.zsparse b/src/archive/zip/testdata/zip64/libarchive-store-just-under-4g.zsparse
new file mode 100644
index 00000000000000..c5191da8362380
Binary files /dev/null and b/src/archive/zip/testdata/zip64/libarchive-store-just-under-4g.zsparse differ
diff --git a/src/archive/zip/writer.go b/src/archive/zip/writer.go
index 0a310054e37678..40bc505ea8f17a 100644
--- a/src/archive/zip/writer.go
+++ b/src/archive/zip/writer.go
@@ -42,6 +42,10 @@ type header struct {
}
// NewWriter returns a new [Writer] writing a zip file to w.
+//
+// Note that the exact bytes written to w are not covered by the Go 1
+// compatibility promise. Callers, including tests, should not depend on the
+// exact written bytes.
func NewWriter(w io.Writer) *Writer {
return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}}
}
@@ -89,48 +93,65 @@ func (w *Writer) Close() error {
// write central directory
start := w.cw.count
+ usedZip64 := false
for _, h := range w.dir {
+ // For the Central Directory, we always have the correct sizes.
+ //
+ // Implementations disagree on what triggers the inclusion of a Zip64
+ // extra field: Info-ZIP only writes it if any size or offset EXCEEDS
+ // 4GiB - 1, while libarchive writes it if any size REACHES OR EXCEEDS
+ // 4GiB - 1, or if the offset EXCEEDS 4GiB - 1. The spec is ambiguous.
+ //
+ // We conservatively write Zip64 extra fields if any size or offset
+ // REACHES OR EXCEEDS 4GiB - 1, to maximize compatibility with readers.
+ // There is no ambiguity in parsing, so there is no downside to it.
+ //
+ // The spec is clear though that all and only the fields that REACH OR
+ // EXCEED 4GiB - 1 are included in the Zip64 extra, once it's present.
+ readerVersion := h.ReaderVersion
+ if h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max || h.offset >= uint32max {
+ usedZip64 = true
+ readerVersion = max(readerVersion, zipVersion45)
+ var size uint16
+ var buf [28]byte // 2x uint16 + up to 3x uint64
+ eb := writeBuf(buf[:])
+ eb.uint16(zip64ExtraID)
+ eb.uint16(0) // size to be filled out later
+ if h.UncompressedSize64 >= uint32max {
+ eb.uint64(h.UncompressedSize64)
+ size += 8
+ }
+ if h.CompressedSize64 >= uint32max {
+ eb.uint64(h.CompressedSize64)
+ size += 8
+ }
+ if h.offset >= uint32max {
+ eb.uint64(h.offset)
+ size += 8
+ }
+ sb := writeBuf(buf[2:])
+ sb.uint16(size)
+ h.Extra = append(h.Extra, buf[:4+size]...)
+ }
+
var buf [directoryHeaderLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(directoryHeaderSignature))
b.uint16(h.CreatorVersion)
- b.uint16(h.ReaderVersion)
+ b.uint16(readerVersion)
b.uint16(h.Flags)
b.uint16(h.Method)
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
b.uint32(h.CRC32)
- if h.isZip64() || h.offset >= uint32max {
- // the file needs a zip64 header. store maxint in both
- // 32 bit size fields (and offset later) to signal that the
- // zip64 extra header should be used.
- b.uint32(uint32max) // compressed size
- b.uint32(uint32max) // uncompressed size
-
- // append a zip64 extra block to Extra
- var buf [28]byte // 2x uint16 + 3x uint64
- eb := writeBuf(buf[:])
- eb.uint16(zip64ExtraID)
- eb.uint16(24) // size = 3x uint64
- eb.uint64(h.UncompressedSize64)
- eb.uint64(h.CompressedSize64)
- eb.uint64(h.offset)
- h.Extra = append(h.Extra, buf[:]...)
- } else {
- b.uint32(h.CompressedSize)
- b.uint32(h.UncompressedSize)
- }
-
+ b.uint32(uint32(min(h.CompressedSize64, uint32max)))
+ b.uint32(uint32(min(h.UncompressedSize64, uint32max)))
b.uint16(uint16(len(h.Name)))
b.uint16(uint16(len(h.Extra)))
b.uint16(uint16(len(h.Comment)))
b = b[4:] // skip disk number start and internal file attr (2x uint16)
b.uint32(h.ExternalAttrs)
- if h.offset > uint32max {
- b.uint32(uint32max)
- } else {
- b.uint32(uint32(h.offset))
- }
+ b.uint32(uint32(min(h.offset, uint32max)))
if _, err := w.cw.Write(buf[:]); err != nil {
return err
}
@@ -154,7 +175,11 @@ func (w *Writer) Close() error {
f(size, offset)
}
- if records >= uint16max || size >= uint32max || offset >= uint32max {
+ // Emit the Zip64 EOCD records whenever any individual entry needed a Zip64
+ // extra field, even if the EOCD's own fields fit in 32 bits, matching
+ // Info-ZIP (but not libarchive). See APPNOTE 4.3.9.2: "when Zip64
+ // extensions are in use, the EOCD64 record must be present."
+ if usedZip64 || records >= uint16max || size >= uint32max || offset >= uint32max {
var buf [directory64EndLen + directory64LocLen]byte
b := writeBuf(buf[:])
@@ -179,24 +204,18 @@ func (w *Writer) Close() error {
if _, err := w.cw.Write(buf[:]); err != nil {
return err
}
-
- // store max values in the regular end record to signal
- // that the zip64 values should be used instead
- records = uint16max
- size = uint32max
- offset = uint32max
}
// write end record
var buf [directoryEndLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(directoryEndSignature))
- b = b[4:] // skip over disk number and first disk number (2x uint16)
- b.uint16(uint16(records)) // number of entries this disk
- b.uint16(uint16(records)) // number of entries total
- b.uint32(uint32(size)) // size of directory
- b.uint32(uint32(offset)) // start of directory
- b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
+ b = b[4:] // skip over disk number and first disk number (2x uint16)
+ b.uint16(uint16(min(uint16max, records))) // number of entries this disk
+ b.uint16(uint16(min(uint16max, records))) // number of entries total
+ b.uint32(uint32(min(uint32max, size))) // size of directory
+ b.uint32(uint32(min(uint32max, offset))) // start of directory
+ b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
if _, err := w.cw.Write(buf[:]); err != nil {
return err
}
@@ -394,38 +413,79 @@ func writeHeader(w io.Writer, h *header) error {
return errLongExtra
}
+ // The correct behavior of a streaming writer, implemented by Info-ZIP 3.0,
+ // would be to write 0xFFFFFFFF in the size fields and then write a Zip64
+ // extra field with the sizes at zero (to signal they are stored in a ZIP64
+ // data descriptor, in case the file is > 4GiB).
+ //
+ // We don't do that, and instead write zeroes directly in the size fields,
+ // because that wastes 28 bytes for every file smaller than 4GiB, and
+ // because it would change the encoding of nearly every zip file created by
+ // archive/zip. (No one should rely on it being stable, but still.)
+ //
+ // Anyway, the Local File Header is not that important, as the Central
+ // Directory is authoritative, and there we always write the correct sizes.
+ //
+ // If we do know the sizes, because [Writer.CreateRaw] is used and the data
+ // descriptor flag is not set, then we write them to the header. If either
+ // size reaches 4GiB, we write 0xFFFFFFFF placeholders and a Zip64 extra
+ // field with BOTH sizes, per the spec and matching Info-ZIP. Note this is
+ // different from the Central Directory Zip64 extra field logic, somehow.
+ //
+ // (One final interesting case that doesn't apply to us: if the input is
+ // streaming but the output is seekable, Info-ZIP always writes Zip64 extra
+ // fields, and then goes back and patches in the sizes, even for files < 4GiB.)
+
+ var zip64ExtraInfo []byte
+ readerVersion := h.ReaderVersion
+ noDataDescriptor := h.raw && !h.hasDataDescriptor()
+ if noDataDescriptor && (h.CompressedSize64 > uint32max || h.UncompressedSize64 > uint32max) {
+ readerVersion = max(readerVersion, zipVersion45)
+ zip64ExtraInfo = make([]byte, 20) // 2x uint16 + 2x uint64
+ b := writeBuf(zip64ExtraInfo)
+ b.uint16(zip64ExtraID)
+ b.uint16(16) // size of Zip64 extra field data
+ b.uint64(h.UncompressedSize64)
+ b.uint64(h.CompressedSize64)
+ }
+
var buf [fileHeaderLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(fileHeaderSignature))
- b.uint16(h.ReaderVersion)
+ b.uint16(readerVersion)
b.uint16(h.Flags)
b.uint16(h.Method)
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
- // In raw mode (caller does the compression), the values are either
- // written here or in the trailing data descriptor based on the header
- // flags.
- if h.raw && !h.hasDataDescriptor() {
+ if noDataDescriptor {
b.uint32(h.CRC32)
- b.uint32(uint32(min(h.CompressedSize64, uint32max)))
- b.uint32(uint32(min(h.UncompressedSize64, uint32max)))
+ if zip64ExtraInfo != nil {
+ b.uint32(uint32max)
+ b.uint32(uint32max)
+ } else {
+ b.uint32(uint32(h.CompressedSize64))
+ b.uint32(uint32(h.UncompressedSize64))
+ }
} else {
- // When this package handle the compression, these values are
- // always written to the trailing data descriptor.
b.uint32(0) // crc32
b.uint32(0) // compressed size
b.uint32(0) // uncompressed size
}
b.uint16(uint16(len(h.Name)))
- b.uint16(uint16(len(h.Extra)))
+ b.uint16(uint16(len(h.Extra) + len(zip64ExtraInfo)))
if _, err := w.Write(buf[:]); err != nil {
return err
}
if _, err := io.WriteString(w, h.Name); err != nil {
return err
}
- _, err := w.Write(h.Extra)
- return err
+ if _, err := w.Write(h.Extra); err != nil {
+ return err
+ }
+ if _, err := w.Write(zip64ExtraInfo); err != nil {
+ return err
+ }
+ return nil
}
// CreateRaw adds a file to the zip archive using the provided [FileHeader] and
@@ -597,7 +657,7 @@ func (w *fileWriter) close() error {
fh.CompressedSize64 = uint64(w.compCount.count)
fh.UncompressedSize64 = uint64(w.rawCount.count)
- if fh.isZip64() {
+ if w.CompressedSize64 > uint32max || w.UncompressedSize64 > uint32max {
fh.CompressedSize = uint32max
fh.UncompressedSize = uint32max
fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions
@@ -613,13 +673,13 @@ func (w *fileWriter) writeDataDescriptor() error {
if !w.hasDataDescriptor() {
return nil
}
- // Write data descriptor. This is more complicated than one would
- // think, see e.g. comments in zipfile.c:putextended() and
- // https://bugs.openjdk.org/browse/JDK-7073588.
- // The approach here is to write 8 byte sizes if needed without
- // adding a zip64 extra in the local header (too late anyway).
+ // See the comment in [writeHeader] about how and why we don't signal ZIP64
+ // mode in the local file header. If one of the sizes turns out to exceed
+ // 4GiB, we use the 64-bit sizes anyway, for lack of alternatives.
+ //
+ // See also https://bugs.openjdk.org/browse/JDK-7073588.
var buf []byte
- if w.isZip64() {
+ if w.CompressedSize64 > uint32max || w.UncompressedSize64 > uint32max {
buf = make([]byte, dataDescriptor64Len)
} else {
buf = make([]byte, dataDescriptorLen)
@@ -627,7 +687,7 @@ func (w *fileWriter) writeDataDescriptor() error {
b := writeBuf(buf)
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
b.uint32(w.CRC32)
- if w.isZip64() {
+ if w.CompressedSize64 > uint32max || w.UncompressedSize64 > uint32max {
b.uint64(w.CompressedSize64)
b.uint64(w.UncompressedSize64)
} else {
diff --git a/src/archive/zip/zip64_sparse_test.go b/src/archive/zip/zip64_sparse_test.go
new file mode 100644
index 00000000000000..96260a425a5df6
--- /dev/null
+++ b/src/archive/zip/zip64_sparse_test.go
@@ -0,0 +1,240 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package zip
+
+import (
+ "cmp"
+ "compress/gzip"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "slices"
+)
+
+// A sparseFile represents an archive as a sequence of non-zero byte spans
+// (the LFH headers, the Central Directory, the EOCD records, and any
+// non-zero compressed bodies) plus a total length. Bytes outside any span
+// are implicitly zero. This is the storage format used for goldens under
+// testdata/zip64/ (suffix .zsparse) and the in-memory shape produced by
+// the writer-reproduction harness.
+//
+// On-disk layout (all little-endian):
+//
+// uint64 size
+// uint32 numSpans
+// for each span:
+// uint64 offset
+// uint32 dataLen
+// dataLen bytes
+//
+// Spans are sorted by offset and non-overlapping.
+type sparseFile struct {
+ Size int64
+ Spans []sparseSpan
+}
+
+type sparseSpan struct {
+ Offset int64
+ Data []byte
+}
+
+// ReadAt implements [io.ReaderAt] by serving the underlying spans and
+// synthesizing zero bytes for any gap inside [0, Size).
+func (f *sparseFile) ReadAt(p []byte, off int64) (int, error) {
+ if off < 0 {
+ return 0, errors.New("sparseFile: negative offset")
+ }
+ if off >= f.Size {
+ return 0, io.EOF
+ }
+ end := min(off+int64(len(p)), f.Size)
+ n := int(end - off)
+ clear(p[:n])
+ for _, s := range f.Spans {
+ sEnd := s.Offset + int64(len(s.Data))
+ if sEnd <= off || s.Offset >= end {
+ continue
+ }
+ from := max(s.Offset, off)
+ to := min(sEnd, end)
+ copy(p[from-off:to-off], s.Data[from-s.Offset:to-s.Offset])
+ }
+ if n < len(p) {
+ return n, io.EOF
+ }
+ return n, nil
+}
+
+// materializeTail returns the last keep bytes of the conceptual file as a
+// plain byte slice, suitable for [parseCD].
+func (f *sparseFile) materializeTail(keep int64) (data []byte, baseOff uint64) {
+ if keep > f.Size {
+ keep = f.Size
+ }
+ base := f.Size - keep
+ buf := make([]byte, keep)
+ f.ReadAt(buf, base)
+ return buf, uint64(base)
+}
+
+const sparseChunk = 4096
+
+// scanSparse stream-reads r and builds a sparseFile, treating any contiguous
+// run of zero bytes (rounded to sparseChunk boundaries) as a gap. Adjacent
+// non-zero chunks are coalesced into one span.
+func scanSparse(r io.Reader) (*sparseFile, error) {
+ f := &sparseFile{}
+ var cur *sparseSpan
+ buf := make([]byte, sparseChunk)
+ for {
+ n, err := io.ReadFull(r, buf)
+ if n > 0 {
+ chunk := buf[:n]
+ if isAllZero(chunk) {
+ if cur != nil {
+ f.Spans = append(f.Spans, *cur)
+ cur = nil
+ }
+ } else {
+ if cur == nil {
+ cur = &sparseSpan{Offset: f.Size}
+ }
+ cur.Data = append(cur.Data, chunk...)
+ }
+ f.Size += int64(n)
+ }
+ if err != nil {
+ if err == io.EOF || err == io.ErrUnexpectedEOF {
+ break
+ }
+ return nil, err
+ }
+ }
+ if cur != nil {
+ f.Spans = append(f.Spans, *cur)
+ }
+ return f, nil
+}
+
+// writeSparse serializes f to w in the on-disk format described on
+// [sparseFile].
+func writeSparse(w io.Writer, f *sparseFile) error {
+ var hdr [12]byte
+ le.PutUint64(hdr[:8], uint64(f.Size))
+ le.PutUint32(hdr[8:12], uint32(len(f.Spans)))
+ if _, err := w.Write(hdr[:]); err != nil {
+ return err
+ }
+ for _, s := range f.Spans {
+ var b [12]byte
+ le.PutUint64(b[:8], uint64(s.Offset))
+ le.PutUint32(b[8:12], uint32(len(s.Data)))
+ if _, err := w.Write(b[:]); err != nil {
+ return err
+ }
+ if _, err := w.Write(s.Data); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// readSparse parses the on-disk format from r.
+func readSparse(r io.Reader) (*sparseFile, error) {
+ var hdr [12]byte
+ if _, err := io.ReadFull(r, hdr[:]); err != nil {
+ return nil, err
+ }
+ f := &sparseFile{
+ Size: int64(le.Uint64(hdr[:8])),
+ }
+ n := le.Uint32(hdr[8:12])
+ if n > 1<<20 {
+ return nil, fmt.Errorf("sparseFile: implausible span count %d", n)
+ }
+ f.Spans = make([]sparseSpan, n)
+ for i := range f.Spans {
+ var b [12]byte
+ if _, err := io.ReadFull(r, b[:]); err != nil {
+ return nil, err
+ }
+ f.Spans[i].Offset = int64(le.Uint64(b[:8]))
+ sz := le.Uint32(b[8:12])
+ f.Spans[i].Data = make([]byte, sz)
+ if _, err := io.ReadFull(r, f.Spans[i].Data); err != nil {
+ return nil, err
+ }
+ }
+ if !slices.IsSortedFunc(f.Spans, func(a, b sparseSpan) int {
+ return cmp.Compare(a.Offset, b.Offset)
+ }) {
+ return nil, errors.New("sparseFile: spans not sorted")
+ }
+ return f, nil
+}
+
+// readSparseFile reads a sparse file from path. The file is expected to be
+// gzip-compressed; the outer gzip wrap shrinks goldens that contain non-zero
+// compressed bodies (e.g., the deflate-zeros entries) by 100x because
+// deflate-of-zeros is highly repetitive. Small Store goldens benefit too:
+// gzip's header overhead is ~30 bytes, well under the bytes saved on a 4 KB
+// sparse representation.
+func readSparseFile(path string) (*sparseFile, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ zr, err := gzip.NewReader(f)
+ if err != nil {
+ return nil, err
+ }
+ defer zr.Close()
+ return readSparse(zr)
+}
+
+// isAllZero reports whether every byte in b is 0.
+func isAllZero(b []byte) bool {
+ for _, c := range b {
+ if c != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+// sparseBuffer accumulates writes into a [sparseFile], dropping any
+// chunkSize-byte chunk that is all-zero. This makes capturing the result
+// of pushing multi-GiB streams of zeros through the writer almost free —
+// the only bytes that end up retained are the LFHs, the Central
+// Directory, the EOCD records, and any non-zero compressed body.
+type sparseBuffer struct {
+ f sparseFile
+ cur *sparseSpan
+}
+
+func (t *sparseBuffer) Write(p []byte) (int, error) {
+ n := len(p)
+ for len(p) > 0 {
+ k := len(p)
+ if k > sparseChunk {
+ k = sparseChunk
+ }
+ chunk := p[:k]
+ if isAllZero(chunk) {
+ t.cur = nil
+ } else {
+ if t.cur == nil {
+ t.f.Spans = append(t.f.Spans, sparseSpan{Offset: t.f.Size})
+ t.cur = &t.f.Spans[len(t.f.Spans)-1]
+ }
+ t.cur.Data = append(t.cur.Data, chunk...)
+ }
+ t.f.Size += int64(k)
+ p = p[k:]
+ }
+ return n, nil
+}
diff --git a/src/archive/zip/zip64_test.go b/src/archive/zip/zip64_test.go
new file mode 100644
index 00000000000000..9252e6ac3de3ac
--- /dev/null
+++ b/src/archive/zip/zip64_test.go
@@ -0,0 +1,614 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package zip
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+)
+
+// TestZip64WriterCDGoldens checks that the archive/zip Writer emits a Central
+// Directory that matches the Zip64 conventions used by Info-ZIP, libarchive,
+// and the pre-CL archive/zip writer (go126-*), for archives at or above 4 GiB,
+// except where we intentionally diverged.
+//
+// For each golden in testdata/zip64/*.zsparse (see [sparseFile] for the
+// committed format), the test:
+// 1. Parses the golden's CD into a producer-independent snapshot — which
+// fields hold 0xFFFFFFFF placeholders, which Zip64 extra sub-fields are
+// present and in what order, and the EOCD/EOCD64 values.
+// 2. Verifies the production [NewReader] parses the same archive.
+// 3. Replays the same entries through a fresh [Writer] into a [sparseBuffer]
+// and parses our own CD.
+// 4. Verifies the production [NewReader] parses our reproduced archive too.
+// 5. Compares the two snapshots field-by-field, ignoring producer-specific
+// details (creator version, external attrs, non-Zip64 extras, absolute
+// byte offsets that depend on LFH/data-descriptor layout).
+func TestZip64WriterCDGoldens(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping in short mode; each golden replays a multi-GiB write")
+ }
+
+ matches, err := filepath.Glob("testdata/zip64/*.zsparse")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(matches) == 0 {
+ t.Fatal("missing Zip64 goldens in testdata/zip64")
+ }
+
+ // Tail materialized for parseCD. Goldens have ≤ 2 entries; their CD
+ // plus EOCD records fits in well under 1 MiB.
+ const tailKeep = 1 << 20
+
+ // archive/zip's writer takes the most defensive position on every
+ // spec-fuzzy point: it always emits the Zip64 extra at the 0xFFFFFFFF
+ // boundary (matching libarchive but more conservative than Info-ZIP) AND
+ // emits EOCD64 whenever any entry has a Zip64 extra in its CD record
+ // (matching Info-ZIP but more conservative than libarchive). The go126-
+ // goldens are output of an older archive/zip writer, and the format
+ // deliberately diverges; they are kept here so the reader-side check
+ // enforces backwards compatibility with archives produced by our own past
+ // writer, and to ensure we only diverge where intended.
+ expectedDiff := map[string]bool{
+ // Info-ZIP treats a CD size field of exactly 0xFFFFFFFF as a real
+ // value and omits the Zip64 extra; archive/zip defensively emits
+ // the Zip64 extra with USize64+CSize64.
+ "infozip-store-4g-minus-1": true,
+
+ // Info-ZIP treats a CD offset field of exactly 0xFFFFFFFF as a real
+ // value and omits the Zip64 extra for offset; archive/zip defensively
+ // emits the Zip64 extra with the offset sub-field.
+ "infozip-offset-eq-4g": true,
+
+ // libarchive's writer emits EOCD64 only on EOCD-level overflow (CD
+ // size/offset > 4GiB, records > 0xFFFF); archive/zip also emits
+ // EOCD64 when any per-entry CD record uses a Zip64 extra, even if
+ // the EOCD fields fit in 32 bits.
+ "libarchive-deflate-zeros-5g": true,
+
+ // libarchive's LFH always carries a UT timestamp extra (~9 bytes),
+ // so its dirOffset for a body of 4GiB-59 lands just past 0xFFFFFFFF
+ // and it emits EOCD64. archive/zip's streaming LFH has no such
+ // extras and stays under uint32max.
+ "libarchive-store-just-under-4g": true,
+
+ // The old archive/zip writer differs from the current writer on
+ // every Zip64-using entry: it always wrote a fixed 24-byte Zip64
+ // extra with all three sub-fields (usize, csize, offset) and set
+ // both 32-bit size fields to 0xFFFFFFFF whenever the per-entry
+ // trigger fired; it also set the EOCD records/size/offset to the
+ // placeholder values whenever EOCD64 was present.
+ "go126-store-5g": true,
+ "go126-deflate-zeros-5g": true,
+ "go126-store-4g-minus-1": true,
+ "go126-store-4g-minus-2": true,
+ "go126-store-exact-4g": true,
+ "go126-offset-past-4g": true,
+ "go126-offset-eq-4g": true,
+ "go126-store-just-under-4g": false,
+ }
+
+ for _, path := range matches {
+ name := strings.TrimSuffix(filepath.Base(path), ".zsparse")
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+ goldenSF, err := readSparseFile(path)
+ if err != nil {
+ t.Fatalf("read golden: %v", err)
+ }
+ goldenData, goldenBase := goldenSF.materializeTail(tailKeep)
+ golden, err := parseCD(goldenData, goldenBase)
+ if err != nil {
+ t.Fatalf("parse golden CD: %v", err)
+ }
+
+ // Verify the production Reader can parse the full golden.
+ checkReaderMatchesSnapshot(t, "golden", goldenSF, golden)
+
+ oursSF := reproduceCD(t, golden)
+ oursData, oursBase := oursSF.materializeTail(tailKeep)
+ got, err := parseCD(oursData, oursBase)
+ if err != nil {
+ t.Fatalf("parse reproduced CD: %v\nbytes:\n%s", err, hexDump(oursData))
+ }
+ // Verify the production Reader can parse archive/zip's own
+ // output and gets the same view of the entries.
+ checkReaderMatchesSnapshot(t, "reproduced", oursSF, got)
+
+ if expectedDiff[name] {
+ var cap captureReporter
+ compareCDSnapshots(&cap, golden, got)
+ if !cap.failed {
+ t.Errorf("expected this golden to fail equivalence, but it passed")
+ } else {
+ t.Logf("expected mismatch:\n%s", indent(cap.msg.String(), " "))
+ }
+ return
+ }
+ compareCDSnapshots(t, golden, got)
+ })
+ }
+}
+
+// errReporter is the subset of [testing.TB] that [compareCDSnapshots] uses.
+// The captureReporter implementation lets the test capture mismatches for
+// expected-failure cases instead of propagating them to the outer t.
+type errReporter interface {
+ Errorf(format string, args ...any)
+ Helper()
+}
+
+type captureReporter struct {
+ failed bool
+ msg strings.Builder
+}
+
+func (c *captureReporter) Errorf(format string, args ...any) {
+ c.failed = true
+ fmt.Fprintf(&c.msg, format+"\n", args...)
+}
+
+func (c *captureReporter) Helper() {}
+
+// checkReaderMatchesSnapshot opens the archive backed by the sparseFile
+// using the production [NewReader] and asserts that the entry list it
+// returns matches the [cdSnapshot] (entry count, names, resolved 64-bit
+// sizes).
+func checkReaderMatchesSnapshot(t *testing.T, label string, f *sparseFile, snap *cdSnapshot) {
+ t.Helper()
+ zr, err := NewReader(f, f.Size)
+ if err != nil {
+ t.Fatalf("%s: NewReader: %v", label, err)
+ }
+ if g, w := len(zr.File), len(snap.Entries); g != w {
+ t.Errorf("%s: NewReader returned %d files, parseCD found %d", label, g, w)
+ return
+ }
+ for i, f := range zr.File {
+ want := &snap.Entries[i]
+ if f.Name != want.Name {
+ t.Errorf("%s entry %d: Name = %q, want %q", label, i, f.Name, want.Name)
+ }
+ if f.UncompressedSize64 != want.USize64 {
+ t.Errorf("%s entry %d %q: UncompressedSize64 = %d, want %d", label, i, want.Name, f.UncompressedSize64, want.USize64)
+ }
+ if f.CompressedSize64 != want.CSize64 {
+ t.Errorf("%s entry %d %q: CompressedSize64 = %d, want %d", label, i, want.Name, f.CompressedSize64, want.CSize64)
+ }
+ }
+}
+
+// indent prefixes every line of s with prefix.
+func indent(s, prefix string) string {
+ if s == "" {
+ return s
+ }
+ lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
+ for i, l := range lines {
+ lines[i] = prefix + l
+ }
+ return strings.Join(lines, "\n") + "\n"
+}
+
+// reproduceCD writes a zip archive with the same logical entries as golden
+// into a [sparseBuffer] (which drops all-zero chunks, so pushing multi-GiB
+// streams of zeros through the writer is essentially free) and returns the
+// resulting [sparseFile].
+//
+// For entries where compressed == uncompressed (Store, or other 1:1 cases)
+// we drive the Writer through [Writer.CreateHeader] so that the data
+// descriptor, offset accounting, and Close-time CD emission all exercise
+// the production streaming path. The CRC32 hasher is replaced with
+// [fakeHash32] to avoid hashing many GiB of zeros.
+//
+// For entries where compressed ≪ uncompressed (Method=Deflate over zeros),
+// actually deflating multi-GiB streams at test time is prohibitively slow,
+// so we fall back to [Writer.CreateRaw] and declare the sizes directly.
+// The Central Directory output is identical either way.
+func reproduceCD(t *testing.T, golden *cdSnapshot) *sparseFile {
+ t.Helper()
+ sb := &sparseBuffer{}
+ w := NewWriter(sb)
+ for i, e := range golden.Entries {
+ if e.CSize64 == e.USize64 {
+ fh := &FileHeader{Name: e.Name, Method: e.Method}
+ fw, err := w.CreateHeader(fh)
+ if err != nil {
+ t.Fatalf("CreateHeader[%d %q]: %v", i, e.Name, err)
+ }
+ fw.(*fileWriter).crc32 = fakeHash32{}
+ if _, err := io.CopyN(fw, zeros{}, int64(e.USize64)); err != nil {
+ t.Fatalf("CopyN[%d %q]: %v", i, e.Name, err)
+ }
+ continue
+ }
+ fh := &FileHeader{
+ Name: e.Name,
+ Method: e.Method,
+ CompressedSize64: e.CSize64,
+ UncompressedSize64: e.USize64,
+ }
+ fw, err := w.CreateRaw(fh)
+ if err != nil {
+ t.Fatalf("CreateRaw[%d %q]: %v", i, e.Name, err)
+ }
+ if _, err := io.CopyN(fw, zeros{}, int64(e.CSize64)); err != nil {
+ t.Fatalf("CopyN[%d %q]: %v", i, e.Name, err)
+ }
+ }
+ if err := w.Close(); err != nil {
+ t.Fatalf("Close: %v", err)
+ }
+ return &sb.f
+}
+
+// compareCDSnapshots asserts that got matches want on Zip64-relevant fields.
+//
+// Per-entry size fields (RawCSize, RawUSize, CSize64, USize64) are compared
+// exactly — we feed them in from the golden when reproducing, so the writer
+// has no excuse to disagree. Per-entry RawOffset and the EOCD records/size/
+// offset fields are compared only as placeholder-or-not: their absolute
+// values depend on producer-specific LFH layout (Info-ZIP packs sizes into
+// the LFH; archive/zip's streaming path uses a data descriptor; libarchive
+// adds UT extras) and that's not what this test is pinning down.
+func compareCDSnapshots(t errReporter, want, got *cdSnapshot) {
+ t.Helper()
+ if g, w := len(got.Entries), len(want.Entries); g != w {
+ t.Errorf("entry count = %d, want %d", g, w)
+ return
+ }
+ for i := range want.Entries {
+ we, ge := &want.Entries[i], &got.Entries[i]
+ // csize and usize come from the declared FileHeader values, so the
+ // raw 32-bit fields must match exactly (real value vs. placeholder
+ // choice and, when not placeholder, the value itself).
+ if we.RawCSize != ge.RawCSize {
+ t.Errorf("entry %d %q: RawCSize = %#08x, want %#08x", i, we.Name, ge.RawCSize, we.RawCSize)
+ }
+ if we.RawUSize != ge.RawUSize {
+ t.Errorf("entry %d %q: RawUSize = %#08x, want %#08x", i, we.Name, ge.RawUSize, we.RawUSize)
+ }
+ // Resolved csize/usize must match — we fed them in from the golden.
+ if we.CSize64 != ge.CSize64 {
+ t.Errorf("entry %d %q: CSize64 = %d, want %d", i, we.Name, ge.CSize64, we.CSize64)
+ }
+ if we.USize64 != ge.USize64 {
+ t.Errorf("entry %d %q: USize64 = %d, want %d", i, we.Name, ge.USize64, we.USize64)
+ }
+ // Offset is layout-dependent. Compare placeholder-or-not, not value.
+ if isPlaceholder32(we.RawOffset) != isPlaceholder32(ge.RawOffset) {
+ t.Errorf("entry %d %q: RawOffset placeholder = %#08x, want %#08x", i, we.Name, ge.RawOffset, we.RawOffset)
+ }
+
+ // Zip64 sub-field presence/order, must match exactly.
+ if !slices.Equal(we.Z64ExtraFields, ge.Z64ExtraFields) {
+ t.Errorf("entry %d %q: Zip64 sub-field order = %v, want %v", i, we.Name, ge.Z64ExtraFields, we.Z64ExtraFields)
+ }
+ // ReaderVersion ≥ 45 whenever a Zip64 extra is present.
+ if len(we.Z64ExtraFields) > 0 && ge.ReaderVersion < zipVersion45 {
+ t.Errorf("entry %d %q: ReaderVersion = %d, want ≥ %d (Zip64 extra present)", i, we.Name, ge.ReaderVersion, zipVersion45)
+ }
+ }
+
+ // EOCD: compare placeholder-or-not for each field. Exact values are
+ // layout-dependent.
+ if isPlaceholder16(want.EOCD.Records) != isPlaceholder16(got.EOCD.Records) {
+ t.Errorf("EOCD records placeholder = %#x, want %#x", got.EOCD.Records, want.EOCD.Records)
+ }
+ if isPlaceholder32(want.EOCD.Size) != isPlaceholder32(got.EOCD.Size) {
+ t.Errorf("EOCD size placeholder = %#x, want %#x", got.EOCD.Size, want.EOCD.Size)
+ }
+ if isPlaceholder32(want.EOCD.Offset) != isPlaceholder32(got.EOCD.Offset) {
+ t.Errorf("EOCD offset placeholder = %#x, want %#x", got.EOCD.Offset, want.EOCD.Offset)
+ }
+
+ if got.HasEOCD64 != want.HasEOCD64 {
+ t.Errorf("EOCD64 present = %v, want %v", got.HasEOCD64, want.HasEOCD64)
+ }
+ if want.HasEOCD64 && got.HasEOCD64 {
+ if got.EOCD64.Records != want.EOCD64.Records {
+ t.Errorf("EOCD64 records = %d, want %d", got.EOCD64.Records, want.EOCD64.Records)
+ }
+ // EOCD64.Size and EOCD64.Offset are layout-dependent.
+ }
+}
+
+func isPlaceholder32(v uint32) bool { return v == uint32max }
+func isPlaceholder16(v uint16) bool { return v == uint16max }
+
+// CD snapshot types and parser
+
+// zip64SubID identifies one of the three sub-fields that may appear in a
+// Zip64 extended-information extra field, in the spec-defined order.
+type zip64SubID int
+
+const (
+ z64USize zip64SubID = iota + 1
+ z64CSize
+ z64Offset
+)
+
+func (s zip64SubID) String() string {
+ switch s {
+ case z64USize:
+ return "usize"
+ case z64CSize:
+ return "csize"
+ case z64Offset:
+ return "offset"
+ }
+ return fmt.Sprintf("zip64SubID(%d)", int(s))
+}
+
+type cdEntry struct {
+ Name string
+ Method uint16
+ ReaderVersion uint16
+
+ // Raw 32-bit fields from the CD record. A value of 0xFFFFFFFF indicates
+ // the real value is in the Zip64 extended-information extra field.
+ RawCSize uint32
+ RawUSize uint32
+ RawOffset uint32
+
+ // Resolved 64-bit values (from the 32-bit field if not a placeholder,
+ // otherwise from the Zip64 extra).
+ CSize64 uint64
+ USize64 uint64
+ Offset64 uint64
+
+ // Sub-fields present in the Zip64 extra, in the order they appear.
+ Z64ExtraFields []zip64SubID
+}
+
+type eocdRec struct {
+ Records uint16 // 0xFFFF if placeholder
+ Size uint32 // 0xFFFFFFFF if placeholder
+ Offset uint32 // 0xFFFFFFFF if placeholder
+}
+
+type eocd64Rec struct {
+ Records uint64
+ Size uint64
+ Offset uint64
+}
+
+type cdSnapshot struct {
+ Entries []cdEntry
+ EOCD eocdRec
+ HasEOCD64 bool
+ EOCD64 eocd64Rec
+}
+
+var le = binary.LittleEndian
+
+// parseCD parses the Central Directory and EOCD records of a zip archive
+// from its raw bytes. data must be the tail of the archive, with baseOffset
+// indicating where data[0] sits in the original archive (0 for whole-archive
+// input).
+func parseCD(data []byte, baseOffset uint64) (*cdSnapshot, error) {
+ sigOff, err := findEOCD(data)
+ if err != nil {
+ return nil, err
+ }
+ snap := &cdSnapshot{}
+ snap.EOCD.Records = le.Uint16(data[sigOff+10:])
+ snap.EOCD.Size = le.Uint32(data[sigOff+12:])
+ snap.EOCD.Offset = le.Uint32(data[sigOff+16:])
+
+ dirOffset := uint64(snap.EOCD.Offset)
+ nRecords := uint64(snap.EOCD.Records)
+
+ // toData converts an absolute archive offset to a data slice offset,
+ // returning false if it lies before our captured tail.
+ toData := func(absOff uint64) (uint64, bool) {
+ if absOff < baseOffset {
+ return 0, false
+ }
+ return absOff - baseOffset, true
+ }
+
+ // Look for an EOCD64 locator immediately preceding the EOCD record.
+ if sigOff >= directory64LocLen {
+ locOff := sigOff - directory64LocLen
+ if le.Uint32(data[locOff:]) == directory64LocSignature {
+ eocd64Off := le.Uint64(data[locOff+8:])
+ eocd64DataOff, ok := toData(eocd64Off)
+ if !ok {
+ return nil, fmt.Errorf("zip: EOCD64 at %#x before captured tail (base %#x)", eocd64Off, baseOffset)
+ }
+ if eocd64DataOff+directory64EndLen > uint64(len(data)) {
+ return nil, errors.New("zip: EOCD64 offset out of range")
+ }
+ if le.Uint32(data[eocd64DataOff:]) != directory64EndSignature {
+ return nil, errors.New("zip: EOCD64 signature mismatch")
+ }
+ snap.HasEOCD64 = true
+ snap.EOCD64.Records = le.Uint64(data[eocd64DataOff+32:])
+ snap.EOCD64.Size = le.Uint64(data[eocd64DataOff+40:])
+ snap.EOCD64.Offset = le.Uint64(data[eocd64DataOff+48:])
+ dirOffset = snap.EOCD64.Offset
+ nRecords = snap.EOCD64.Records
+ }
+ }
+
+ off, ok := toData(dirOffset)
+ if !ok {
+ return nil, fmt.Errorf("zip: CD at %#x before captured tail (base %#x)", dirOffset, baseOffset)
+ }
+ for i := uint64(0); i < nRecords; i++ {
+ if off+directoryHeaderLen > uint64(len(data)) {
+ return nil, fmt.Errorf("zip: CD entry %d out of range", i)
+ }
+ rec := data[off:]
+ if le.Uint32(rec) != directoryHeaderSignature {
+ return nil, fmt.Errorf("zip: bad CD signature at offset %d", off)
+ }
+ var e cdEntry
+ e.ReaderVersion = le.Uint16(rec[6:])
+ e.Method = le.Uint16(rec[10:])
+ e.RawCSize = le.Uint32(rec[20:])
+ e.RawUSize = le.Uint32(rec[24:])
+ nameLen := uint64(le.Uint16(rec[28:]))
+ extraLen := uint64(le.Uint16(rec[30:]))
+ commLen := uint64(le.Uint16(rec[32:]))
+ e.RawOffset = le.Uint32(rec[42:])
+
+ recLen := uint64(directoryHeaderLen) + nameLen + extraLen + commLen
+ if off+recLen > uint64(len(data)) {
+ return nil, fmt.Errorf("zip: CD entry %d truncated", i)
+ }
+ nameOff := off + directoryHeaderLen
+ extraOff := nameOff + nameLen
+ e.Name = string(data[nameOff:extraOff])
+ extra := data[extraOff : extraOff+extraLen]
+
+ e.CSize64 = uint64(e.RawCSize)
+ e.USize64 = uint64(e.RawUSize)
+ e.Offset64 = uint64(e.RawOffset)
+
+ // Walk extra fields; consume the Zip64 sub-field if present.
+ // Per the spec and Info-ZIP convention, the Zip64 extra contains
+ // 8-byte values for exactly the size/offset fields whose 32-bit
+ // counterpart is 0xFFFFFFFF, in the order: USize, CSize, Offset.
+ for len(extra) >= 4 {
+ tag := le.Uint16(extra)
+ size := uint64(le.Uint16(extra[2:]))
+ if 4+size > uint64(len(extra)) {
+ break
+ }
+ field := extra[4 : 4+size]
+ extra = extra[4+size:]
+ if tag != zip64ExtraID {
+ continue
+ }
+ if e.RawUSize == uint32max && len(field) >= 8 {
+ e.USize64 = le.Uint64(field)
+ e.Z64ExtraFields = append(e.Z64ExtraFields, z64USize)
+ field = field[8:]
+ }
+ if e.RawCSize == uint32max && len(field) >= 8 {
+ e.CSize64 = le.Uint64(field)
+ e.Z64ExtraFields = append(e.Z64ExtraFields, z64CSize)
+ field = field[8:]
+ }
+ if e.RawOffset == uint32max && len(field) >= 8 {
+ e.Offset64 = le.Uint64(field)
+ e.Z64ExtraFields = append(e.Z64ExtraFields, z64Offset)
+ field = field[8:]
+ }
+ }
+
+ snap.Entries = append(snap.Entries, e)
+ off += recLen
+ }
+ return snap, nil
+}
+
+// findEOCD locates the EOCD record by scanning back from the end of data,
+// matching both the signature and the trailing comment-length field.
+func findEOCD(data []byte) (uint64, error) {
+ if len(data) < directoryEndLen {
+ return 0, errors.New("zip: too short for EOCD")
+ }
+ maxComment := uint16max
+ lo := len(data) - directoryEndLen
+ hi := lo
+ if hi > maxComment {
+ lo = hi - maxComment
+ } else {
+ lo = 0
+ }
+ for i := hi; i >= lo; i-- {
+ if le.Uint32(data[i:]) != directoryEndSignature {
+ continue
+ }
+ cl := int(le.Uint16(data[i+20:]))
+ if i+directoryEndLen+cl == len(data) {
+ return uint64(i), nil
+ }
+ }
+ return 0, errors.New("zip: EOCD not found")
+}
+
+// hexDump returns a short hex dump of data for failure messages.
+func hexDump(data []byte) string {
+ if len(data) > 4096 {
+ data = data[len(data)-4096:]
+ }
+ var b strings.Builder
+ for i := 0; i < len(data); i += 16 {
+ end := min(i+16, len(data))
+ fmt.Fprintf(&b, "%04x % x\n", i, data[i:end])
+ }
+ return b.String()
+}
+
+// TestZip64LFHBothPlaceholders covers the [Writer.CreateRaw] + no-data-
+// descriptor path where the entry's uncompressed or compressed size exceeds
+// 4 GiB. The Local File Header carries a Zip64 extra with both 8-byte
+// USize64 and CSize64 sub-fields (matching Info-ZIP), so per APPNOTE 4.5.3
+// both 32-bit size fields in the LFH must be the 0xFFFFFFFF placeholder —
+// even if only one of the sizes actually overflows.
+func TestZip64LFHBothPlaceholders(t *testing.T) {
+ var buf bytes.Buffer
+ w := NewWriter(&buf)
+ fh := &FileHeader{
+ Name: "x",
+ Method: Deflate,
+ CompressedSize64: 1024,
+ UncompressedSize64: 5 << 30, // > 4 GiB
+ }
+ fw, err := w.CreateRaw(fh)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := io.CopyN(fw, zeros{}, int64(fh.CompressedSize64)); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ b := buf.Bytes()
+ if got := le.Uint32(b[14:18]); got != fh.CRC32 {
+ t.Errorf("LFH CRC32 = %#x, want %#x", got, fh.CRC32)
+ }
+ if got := le.Uint32(b[18:22]); got != uint32max {
+ t.Errorf("LFH CompressedSize = %#x, want %#x (placeholder)", got, uint32(uint32max))
+ }
+ if got := le.Uint32(b[22:26]); got != uint32max {
+ t.Errorf("LFH UncompressedSize = %#x, want %#x (placeholder)", got, uint32(uint32max))
+ }
+
+ // The Zip64 LFH extra should carry both 64-bit sub-fields in
+ // USize64-then-CSize64 order.
+ nameLen := uint64(le.Uint16(b[26:28]))
+ extraLen := uint64(le.Uint16(b[28:30]))
+ if want := uint64(20); extraLen != want {
+ t.Fatalf("LFH extra length = %d, want %d", extraLen, want)
+ }
+ extra := b[30+nameLen : 30+nameLen+extraLen]
+ if tag := le.Uint16(extra[:2]); tag != zip64ExtraID {
+ t.Errorf("Zip64 extra tag = %#x, want %#x", tag, zip64ExtraID)
+ }
+ if dataLen := le.Uint16(extra[2:4]); dataLen != 16 {
+ t.Errorf("Zip64 extra data length = %d, want 16", dataLen)
+ }
+ if got := le.Uint64(extra[4:12]); got != fh.UncompressedSize64 {
+ t.Errorf("Zip64 USize64 = %d, want %d", got, fh.UncompressedSize64)
+ }
+ if got := le.Uint64(extra[12:20]); got != fh.CompressedSize64 {
+ t.Errorf("Zip64 CSize64 = %d, want %d", got, fh.CompressedSize64)
+ }
+}
diff --git a/src/archive/zip/zip_test.go b/src/archive/zip/zip_test.go
index 5a6689a0eb1163..1f36b365ecbbbc 100644
--- a/src/archive/zip/zip_test.go
+++ b/src/archive/zip/zip_test.go
@@ -14,7 +14,7 @@ import (
"hash"
"internal/testenv"
"io"
- "runtime"
+ "math/bits"
"slices"
"strings"
"testing"
@@ -494,12 +494,15 @@ func suffixIsZip64(t *testing.T, zip sizedReaderAt) bool {
// Zip64 is required if the total size of the records is uint32max.
func TestZip64LargeDirectory(t *testing.T) {
- if runtime.GOARCH == "wasm" {
- t.Skip("too slow on wasm")
+ if testenv.CPUIsSlow() {
+ t.Skip("too slow")
}
if testing.Short() {
t.Skip("skipping in short mode")
}
+ if bits.UintSize == 32 {
+ t.Skip("skipping on 32-bit platforms")
+ }
t.Parallel()
// gen returns a func that writes a zip with a wantLen bytes
// of central directory.
@@ -538,15 +541,30 @@ func TestZip64LargeDirectory(t *testing.T) {
}
t.Run("uint32max-1_NoZip64", func(t *testing.T) {
t.Parallel()
- if generatesZip64(t, gen(uint32max-1)) {
+ buf := new(rleBuffer)
+ w := NewWriter(buf)
+ gen(uint32max - 1)(w)
+ if suffixIsZip64(t, buf) {
t.Error("unexpected zip64")
}
+ if _, err := NewReader(buf, buf.Size()); err != nil {
+ t.Errorf("NewReader: %v", err)
+ }
})
t.Run("uint32max_HasZip64", func(t *testing.T) {
t.Parallel()
- if !generatesZip64(t, gen(uint32max)) {
+ buf := new(rleBuffer)
+ w := NewWriter(buf)
+ gen(uint32max)(w)
+ if !suffixIsZip64(t, buf) {
t.Error("expected zip64")
}
+ // Round-trip through NewReader. With CD size exactly 0xFFFFFFFF,
+ // records well below 0xFFFF, and dirOffset == 0, the only EOCD
+ // field that holds the placeholder is directorySize.
+ if _, err := NewReader(buf, buf.Size()); err != nil {
+ t.Errorf("NewReader: %v", err)
+ }
})
}
diff --git a/src/bufio/bufio.go b/src/bufio/bufio.go
index 141a9a1a2a2305..f9b762be8d4966 100644
--- a/src/bufio/bufio.go
+++ b/src/bufio/bufio.go
@@ -30,7 +30,7 @@ var (
// Reader implements buffering for an io.Reader object.
// A new Reader is created by calling [NewReader] or [NewReaderSize];
-// alternatively the zero value of a Reader may be used after calling [Reset]
+// alternatively the zero value of a Reader may be used after calling [Reader.Reset]
// on it.
type Reader struct {
buf []byte
diff --git a/src/builtin/builtin.go b/src/builtin/builtin.go
index afa2a10f9091dc..be53077d9cb495 100644
--- a/src/builtin/builtin.go
+++ b/src/builtin/builtin.go
@@ -122,6 +122,10 @@ type Type int
// invocation.
type Type1 int
+// TypeOrExpr is here for the purposes of documentation only. It is a stand-in
+// for either a Go type or an expression.
+type TypeOrExpr int
+
// IntegerType is here for the purposes of documentation only. It is a stand-in
// for any integer type: int, uint, int8 etc.
type IntegerType int
@@ -220,10 +224,15 @@ func max[T cmp.Ordered](x T, y ...T) T
// min will return NaN.
func min[T cmp.Ordered](x T, y ...T) T
-// The new built-in function allocates memory. The first argument is a type,
-// not a value, and the value returned is a pointer to a newly
-// allocated zero value of that type.
-func new(Type) *Type
+// The built-in function new allocates a new, initialized variable and returns
+// a pointer to it. It accepts a single argument, which may be either a type
+// or an expression.
+// If the argument is a type T, then new(T) allocates a variable of type T
+// initialized to its zero value.
+// Otherwise, the argument is an expression x and new(x) allocates a variable
+// of the type of x initialized to the value of x. If that value is an untyped
+// constant, it is first implicitly converted to its default type.
+func new(TypeOrExpr) *Type
// The complex built-in function constructs a complex value from two
// floating-point values. The real and imaginary parts must be of the same
diff --git a/src/bytes/buffer.go b/src/bytes/buffer.go
index 9684513942da88..03f17d624daa23 100644
--- a/src/bytes/buffer.go
+++ b/src/bytes/buffer.go
@@ -77,6 +77,18 @@ func (b *Buffer) String() string {
return string(b.buf[b.off:])
}
+// Peek returns the next n bytes without advancing the buffer.
+// If Peek returns fewer than n bytes, it also returns [io.EOF].
+// The slice is only valid until the next call to a read or write method.
+// The slice aliases the buffer content at least until the next buffer modification,
+// so immediate changes to the slice will affect the result of future reads.
+func (b *Buffer) Peek(n int) ([]byte, error) {
+ if b.Len() < n {
+ return b.buf[b.off:], io.EOF
+ }
+ return b.buf[b.off : b.off+n], nil
+}
+
// empty reports whether the unread portion of the buffer is empty.
func (b *Buffer) empty() bool { return len(b.buf) <= b.off }
@@ -299,9 +311,9 @@ func (b *Buffer) WriteByte(c byte) error {
}
// WriteRune appends the UTF-8 encoding of Unicode code point r to the
-// buffer, returning its length and an error, which is always nil but is
-// included to match [bufio.Writer]'s WriteRune. The buffer is grown as needed;
-// if it becomes too large, WriteRune will panic with [ErrTooLarge].
+// buffer, returning the number of bytes written and a nil error. The nil
+// error is included to match [bufio.Writer]'s WriteRune. The buffer is grown
+// as needed; if it becomes too large, WriteRune will panic with [ErrTooLarge].
func (b *Buffer) WriteRune(r rune) (n int, err error) {
// Compare as uint32 to correctly handle negative runes.
if uint32(r) < utf8.RuneSelf {
diff --git a/src/bytes/buffer_test.go b/src/bytes/buffer_test.go
index b46ba1204eb806..9c1ba0a838054f 100644
--- a/src/bytes/buffer_test.go
+++ b/src/bytes/buffer_test.go
@@ -531,6 +531,40 @@ func TestReadString(t *testing.T) {
}
}
+var peekTests = []struct {
+ buffer string
+ skip int
+ n int
+ expected string
+ err error
+}{
+ {"", 0, 0, "", nil},
+ {"aaa", 0, 3, "aaa", nil},
+ {"foobar", 0, 2, "fo", nil},
+ {"a", 0, 2, "a", io.EOF},
+ {"helloworld", 4, 3, "owo", nil},
+ {"helloworld", 5, 5, "world", nil},
+ {"helloworld", 5, 6, "world", io.EOF},
+ {"helloworld", 10, 1, "", io.EOF},
+}
+
+func TestPeek(t *testing.T) {
+ for _, test := range peekTests {
+ buf := NewBufferString(test.buffer)
+ buf.Next(test.skip)
+ bytes, err := buf.Peek(test.n)
+ if string(bytes) != test.expected {
+ t.Errorf("expected %q, got %q", test.expected, bytes)
+ }
+ if err != test.err {
+ t.Errorf("expected error %v, got %v", test.err, err)
+ }
+ if buf.Len() != len(test.buffer)-test.skip {
+ t.Errorf("bad length after peek: %d, want %d", buf.Len(), len(test.buffer)-test.skip)
+ }
+ }
+}
+
func BenchmarkReadString(b *testing.B) {
const n = 32 << 10
diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go
index 87183c01955829..b0daa5dfc41c20 100644
--- a/src/bytes/bytes.go
+++ b/src/bytes/bytes.go
@@ -89,6 +89,7 @@ func ContainsRune(b []byte, r rune) bool {
}
// ContainsFunc reports whether any of the UTF-8-encoded code points r within b satisfy f(r).
+// It stops as soon as a call to f returns true.
func ContainsFunc(b []byte, f func(rune) bool) bool {
return IndexFunc(b, f) >= 0
}
@@ -98,15 +99,6 @@ func IndexByte(b []byte, c byte) int {
return bytealg.IndexByte(b, c)
}
-func indexBytePortable(s []byte, c byte) int {
- for i, b := range s {
- if b == c {
- return i
- }
- }
- return -1
-}
-
// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s.
func LastIndex(s, sep []byte) int {
n := len(sep)
@@ -246,7 +238,7 @@ func IndexAny(s []byte, chars string) int {
}
return IndexRune(s, r)
}
- if len(s) > 8 {
+ if shouldUseASCIISet(len(s)) {
if as, isASCII := makeASCIISet(chars); isASCII {
for i, c := range s {
if as.contains(c) {
@@ -301,7 +293,7 @@ func LastIndexAny(s []byte, chars string) int {
// Avoid scanning all of s.
return -1
}
- if len(s) > 8 {
+ if shouldUseASCIISet(len(s)) {
if as, isASCII := makeASCIISet(chars); isASCII {
for i := len(s) - 1; i >= 0; i-- {
if as.contains(s[i]) {
@@ -388,9 +380,7 @@ func genSplit(s, sep []byte, sepSave, n int) [][]byte {
if n < 0 {
n = Count(s, sep) + 1
}
- if n > len(s)+1 {
- n = len(s) + 1
- }
+ n = min(n, len(s)+1)
a := make([][]byte, n)
n--
@@ -935,15 +925,22 @@ func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int {
return -1
}
-// asciiSet is a 32-byte value, where each bit represents the presence of a
-// given ASCII character in the set. The 128-bits of the lower 16 bytes,
-// starting with the least-significant bit of the lowest word to the
-// most-significant bit of the highest word, map to the full range of all
-// 128 ASCII characters. The 128-bits of the upper 16 bytes will be zeroed,
-// ensuring that any non-ASCII character will be reported as not in the set.
-// This allocates a total of 32 bytes even though the upper half
-// is unused to avoid bounds checks in asciiSet.contains.
-type asciiSet [8]uint32
+// asciiSet is a 256-byte lookup table for fast ASCII character membership testing.
+// Each element corresponds to an ASCII character value, with true indicating the
+// character is in the set. Using bool instead of byte allows the compiler to
+// eliminate the comparison instruction, as bool values are guaranteed to be 0 or 1.
+//
+// The full 256-element table is used rather than a 128-element table to avoid
+// additional operations in the lookup path. Alternative approaches were tested:
+// - [128]bool with explicit bounds check (if c >= 128): introduces branches
+// that cause pipeline stalls, resulting in ~70% slower performance
+// - [128]bool with masking (c&0x7f): eliminates bounds checks but the AND
+// operation still costs ~10% performance compared to direct indexing
+//
+// The 256-element array allows direct indexing with no bounds checks, no branches,
+// and no masking operations, providing optimal performance. The additional 128 bytes
+// of memory is a worthwhile tradeoff for the simpler, faster code.
+type asciiSet [256]bool
// makeASCIISet creates a set of ASCII characters and reports whether all
// characters in chars are ASCII.
@@ -953,14 +950,24 @@ func makeASCIISet(chars string) (as asciiSet, ok bool) {
if c >= utf8.RuneSelf {
return as, false
}
- as[c/32] |= 1 << (c % 32)
+ as[c] = true
}
return as, true
}
// contains reports whether c is inside the set.
func (as *asciiSet) contains(c byte) bool {
- return (as[c/32] & (1 << (c % 32))) != 0
+ return as[c]
+}
+
+// shouldUseASCIISet returns whether to use the lookup table optimization.
+// The threshold of 8 bytes balances initialization cost against per-byte
+// search cost, performing well across all charset sizes.
+//
+// More complex heuristics (e.g., different thresholds per charset size)
+// add branching overhead that eats away any theoretical improvements.
+func shouldUseASCIISet(bufLen int) bool {
+ return bufLen > 8
}
// containsRune is a simplified version of strings.ContainsRune
@@ -1413,3 +1420,16 @@ func CutSuffix(s, suffix []byte) (before []byte, found bool) {
}
return s[:len(s)-len(suffix)], true
}
+
+// CutLast slices s around the last instance of sep,
+// returning the text before and after sep.
+// The found result reports whether sep appears in s.
+// If sep does not appear in s, CutLast returns s, nil, false.
+//
+// CutLast returns slices of the original slice s, not copies.
+func CutLast(s, sep []byte) (before, after []byte, found bool) {
+ if i := LastIndex(s, sep); i >= 0 {
+ return s[:i], s[i+len(sep):], true
+ }
+ return s, nil, false
+}
diff --git a/src/bytes/bytes_test.go b/src/bytes/bytes_test.go
index f18915c879e097..20d333217e81e2 100644
--- a/src/bytes/bytes_test.go
+++ b/src/bytes/bytes_test.go
@@ -323,10 +323,6 @@ func TestIndexByte(t *testing.T) {
if pos != tt.i {
t.Errorf(`IndexByte(%q, '%c') = %v`, tt.a, b, pos)
}
- posp := IndexBytePortable(a, b)
- if posp != tt.i {
- t.Errorf(`indexBytePortable(%q, '%c') = %v`, tt.a, b, posp)
- }
}
}
@@ -617,8 +613,18 @@ func BenchmarkIndexByte(b *testing.B) {
benchBytes(b, indexSizes, bmIndexByte(IndexByte))
}
+// indexBytePortable use as the baseline for performance comparisons.
+func indexBytePortable(s []byte, c byte) int {
+ for i, b := range s {
+ if b == c {
+ return i
+ }
+ }
+ return -1
+}
+
func BenchmarkIndexBytePortable(b *testing.B) {
- benchBytes(b, indexSizes, bmIndexByte(IndexBytePortable))
+ benchBytes(b, indexSizes, bmIndexByte(indexBytePortable))
}
func bmIndexByte(index func([]byte, byte) int) func(b *testing.B, n int) {
@@ -961,7 +967,7 @@ func TestSplit(t *testing.T) {
if tt.n < 0 {
b := sliceOfString(Split([]byte(tt.s), []byte(tt.sep)))
if !slices.Equal(result, b) {
- t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
+ t.Errorf("Split disagrees with SplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
}
}
if len(a) > 0 {
@@ -1023,7 +1029,7 @@ func TestSplitAfter(t *testing.T) {
if tt.n < 0 {
b := sliceOfString(SplitAfter([]byte(tt.s), []byte(tt.sep)))
if !slices.Equal(result, b) {
- t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
+ t.Errorf("SplitAfter disagrees with SplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
}
}
}
@@ -1224,7 +1230,7 @@ func TestMap(t *testing.T) {
// Run a couple of awful growth/shrinkage tests
a := tenRunes('a')
- // 1. Grow. This triggers two reallocations in Map.
+ // 1. Grow. This triggers two reallocations in Map.
maxRune := func(r rune) rune { return unicode.MaxRune }
m := Map(maxRune, []byte(a))
expect := tenRunes(unicode.MaxRune)
@@ -1970,6 +1976,27 @@ func TestCut(t *testing.T) {
}
}
+func TestCutLast(t *testing.T) {
+ tests := []struct {
+ s, sep string
+ before, after string
+ found bool
+ }{
+ {"a/b/c", "/", "a/b", "c", true},
+ {"a//b//c", "//", "a//b", "c", true},
+ {"abc", "/", "abc", "", false},
+ {"abc", "", "abc", "", true},
+ {"", "", "", "", true},
+ {"/abc", "/", "", "abc", true},
+ {"abc/", "/", "abc", "", true},
+ }
+ for _, tt := range tests {
+ if before, after, found := CutLast([]byte(tt.s), []byte(tt.sep)); string(before) != tt.before || string(after) != tt.after || found != tt.found {
+ t.Errorf("CutLast(%q, %q) = %q, %q, %v; want %q, %q, %v", tt.s, tt.sep, before, after, found, tt.before, tt.after, tt.found)
+ }
+ }
+}
+
var cutPrefixTests = []struct {
s, sep string
after string
diff --git a/src/bytes/export_test.go b/src/bytes/export_test.go
deleted file mode 100644
index b65428d9ce8489..00000000000000
--- a/src/bytes/export_test.go
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bytes
-
-// Export func for testing
-var IndexBytePortable = indexBytePortable
diff --git a/src/cmd/api/api_test.go b/src/cmd/api/api_test.go
index 98ebf168ef3949..784bdb74e428e0 100644
--- a/src/cmd/api/api_test.go
+++ b/src/cmd/api/api_test.go
@@ -232,7 +232,6 @@ var warmupCache = sync.OnceFunc(func() {
// Warm up the import cache in parallel.
var wg sync.WaitGroup
for _, context := range contexts {
- context := context
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/src/cmd/api/main_test.go b/src/cmd/api/main_test.go
index ed366be4e77267..f217319f1579a0 100644
--- a/src/cmd/api/main_test.go
+++ b/src/cmd/api/main_test.go
@@ -17,6 +17,7 @@ import (
"go/parser"
"go/token"
"go/types"
+ "internal/buildcfg"
"internal/testenv"
"io"
"log"
@@ -116,12 +117,20 @@ func Check(t *testing.T) {
for _, c := range contexts {
c.Compiler = build.Default.Compiler
+
+ // Include baseline goexperiment.* tool tags.
+ baseline, err := buildcfg.ParseGOEXPERIMENT(c.GOOS, c.GOARCH, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, exp := range baseline.Enabled() {
+ c.ToolTags = append(c.ToolTags, "goexperiment."+exp)
+ }
}
walkers := make([]*Walker, len(contexts))
var wg sync.WaitGroup
for i, context := range contexts {
- i, context := i, context
wg.Add(1)
go func() {
defer wg.Done()
@@ -390,7 +399,7 @@ func (w *Walker) parseFile(dir, file string) (*ast.File, error) {
return f, nil
}
- f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
+ f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments|parser.SkipObjectResolution)
if err != nil {
return nil, err
}
@@ -1154,14 +1163,15 @@ func needApproval(filename string) bool {
func (w *Walker) collectDeprecated() {
isDeprecated := func(doc *ast.CommentGroup) bool {
- if doc != nil {
- for _, c := range doc.List {
- if strings.HasPrefix(c.Text, "// Deprecated:") {
- return true
- }
- }
- }
- return false
+ // Look for "Deprecated:" (case-sensitive) at the beginning (not middle) of a paragraph.
+ // It's typically found in the last paragraph, but it's not required to be the last one.
+ // The colon is typically followed by a space, but it can also be a newline, as was the
+ // case at https://go.dev/pkg/go/build#AllowBinary for example.
+ //
+ // See https://go.dev/wiki/Deprecated and https://go.dev/ref/mod#go-mod-file-module-deprecation.
+ text := doc.Text()
+ return strings.HasPrefix(text, "Deprecated: ") || strings.Contains(text, "\n\nDeprecated: ") ||
+ strings.HasPrefix(text, "Deprecated:\n") || strings.Contains(text, "\n\nDeprecated:\n")
}
w.deprecated = make(map[token.Pos]bool)
diff --git a/src/cmd/api/testdata/src/pkg/issue79145/golden.txt b/src/cmd/api/testdata/src/pkg/issue79145/golden.txt
new file mode 100644
index 00000000000000..a62e6b70018a09
--- /dev/null
+++ b/src/cmd/api/testdata/src/pkg/issue79145/golden.txt
@@ -0,0 +1,5 @@
+pkg issue79145, const Tau = 6.28319 // 314159265358979323846264338327950288419716939937510582097494459/50000000000000000000000000000000000000000000000000000000000000
+pkg issue79145, const Tau ideal-float
+pkg issue79145, const Old //deprecated
+pkg issue79145, const Old = true
+pkg issue79145, const Old ideal-bool
diff --git a/src/cmd/api/testdata/src/pkg/issue79145/issue79145.go b/src/cmd/api/testdata/src/pkg/issue79145/issue79145.go
new file mode 100644
index 00000000000000..bf6e5386e27fc6
--- /dev/null
+++ b/src/cmd/api/testdata/src/pkg/issue79145/issue79145.go
@@ -0,0 +1,14 @@
+package issue79145
+
+// Tau is the constant τ, which equals to 6.283185... or 2π. Tau is not
+// Deprecated: please find more information at https://oeis.org/A019692.
+const Tau = 2 * pi
+
+const pi = 3.14159265358979323846264338327950288419716939937510582097494459 // https://oeis.org/A000796; like math.Pi
+
+/*
+Old is something old.
+
+Deprecated: Use New instead.
+*/
+const Old = true
diff --git a/src/cmd/asm/internal/arch/arch.go b/src/cmd/asm/internal/arch/arch.go
index 8481a8f378f2cb..dd955bd3a87b24 100644
--- a/src/cmd/asm/internal/arch/arch.go
+++ b/src/cmd/asm/internal/arch/arch.go
@@ -92,7 +92,8 @@ func jumpX86(word string) bool {
func jumpRISCV(word string) bool {
switch word {
case "BEQ", "BEQZ", "BGE", "BGEU", "BGEZ", "BGT", "BGTU", "BGTZ", "BLE", "BLEU", "BLEZ",
- "BLT", "BLTU", "BLTZ", "BNE", "BNEZ", "CALL", "JAL", "JALR", "JMP":
+ "BLT", "BLTU", "BLTZ", "BNE", "BNEZ", "CALL", "CBEQZ", "CBNEZ", "CJ", "CJALR", "CJR",
+ "JAL", "JALR", "JMP":
return true
}
return false
@@ -274,6 +275,15 @@ func archArm64() *Arch {
for i := arm64.REG_V0; i <= arm64.REG_V31; i++ {
register[obj.Rconv(i)] = int16(i)
}
+ for i := arm64.REG_Z0; i <= arm64.REG_Z31; i++ {
+ register[obj.Rconv(i)] = int16(i)
+ }
+ for i := arm64.REG_P0; i <= arm64.REG_P15; i++ {
+ register[obj.Rconv(i)] = int16(i)
+ }
+ for i := arm64.REG_PN0; i <= arm64.REG_PN15; i++ {
+ register[obj.Rconv(i)] = int16(i)
+ }
// System registers.
for i := 0; i < len(arm64.SystemReg); i++ {
@@ -291,9 +301,12 @@ func archArm64() *Arch {
delete(register, "R28")
register["g"] = arm64.REG_R28
registerPrefix := map[string]bool{
- "F": true,
- "R": true,
- "V": true,
+ "F": true,
+ "R": true,
+ "V": true,
+ "Z": true,
+ "P": true,
+ "PN": true,
}
instructions := make(map[string]obj.As)
diff --git a/src/cmd/asm/internal/arch/arm64.go b/src/cmd/asm/internal/arch/arm64.go
index d562e5907d6fc1..24020a336b7536 100644
--- a/src/cmd/asm/internal/arch/arm64.go
+++ b/src/cmd/asm/internal/arch/arm64.go
@@ -182,6 +182,18 @@ func arm64RegisterNumber(name string, n int16) (int16, bool) {
if 0 <= n && n <= 31 {
return arm64.REG_V0 + n, true
}
+ case "Z":
+ if 0 <= n && n <= 31 {
+ return arm64.REG_Z0 + n, true
+ }
+ case "P":
+ if 0 <= n && n <= 15 {
+ return arm64.REG_P0 + n, true
+ }
+ case "PN":
+ if 0 <= n && n <= 15 {
+ return arm64.REG_PN0 + n, true
+ }
}
return 0, false
}
@@ -197,9 +209,17 @@ func ARM64RegisterShift(reg, op, count int16) (int64, error) {
// ARM64RegisterArrangement constructs an ARM64 vector register arrangement.
func ARM64RegisterArrangement(reg int16, name, arng string) (int64, error) {
- var curQ, curSize uint16
- if name[0] != 'V' {
- return 0, errors.New("expect V0 through V31; found: " + name)
+ var curQ, curSize, prefix uint16
+ if name[0] != 'V' && name[0] != 'Z' && name[0] != 'P' {
+ return 0, errors.New("expect V0-V31, Z0-Z31, or P0-P15; found: " + name)
+ }
+ switch name[0] {
+ case 'V':
+ prefix = 0
+ case 'Z':
+ prefix = 1
+ case 'P':
+ prefix = 2
}
if reg < 0 {
return 0, errors.New("invalid register number: " + name)
@@ -229,30 +249,23 @@ func ARM64RegisterArrangement(reg int16, name, arng string) (int64, error) {
case "D2":
curSize = 3
curQ = 1
+ case "B":
+ curSize = 1
+ curQ = 2
+ case "H":
+ curSize = 2
+ curQ = 2
+ case "S":
+ curSize = 3
+ curQ = 2
+ case "D":
+ curSize = 1
+ curQ = 3
+ case "Q":
+ curSize = 2
+ curQ = 3
default:
return 0, errors.New("invalid arrangement in ARM64 register list")
}
- return (int64(curQ) & 1 << 30) | (int64(curSize&3) << 10), nil
-}
-
-// ARM64RegisterListOffset generates offset encoding according to AArch64 specification.
-func ARM64RegisterListOffset(firstReg, regCnt int, arrangement int64) (int64, error) {
- offset := int64(firstReg)
- switch regCnt {
- case 1:
- offset |= 0x7 << 12
- case 2:
- offset |= 0xa << 12
- case 3:
- offset |= 0x6 << 12
- case 4:
- offset |= 0x2 << 12
- default:
- return 0, errors.New("invalid register numbers in ARM64 register list")
- }
- offset |= arrangement
- // arm64 uses the 60th bit to differentiate from other archs
- // For more details, refer to: obj/arm64/list7.go
- offset |= 1 << 60
- return offset, nil
+ return (int64(prefix) << 32) | (int64(curQ) & 3 << 30) | (int64(curSize&3) << 10), nil
}
diff --git a/src/cmd/asm/internal/arch/loong64.go b/src/cmd/asm/internal/arch/loong64.go
index 21263d34331513..4f62f5210b5740 100644
--- a/src/cmd/asm/internal/arch/loong64.go
+++ b/src/cmd/asm/internal/arch/loong64.go
@@ -75,48 +75,51 @@ var loong64LasxArngExtMap = map[string]int16{
// Loong64RegisterExtension constructs an Loong64 register with extension or arrangement.
func Loong64RegisterExtension(a *obj.Addr, ext string, reg, num int16, isAmount, isIndex bool) error {
var ok bool
- var arng_type int16
- var simd_type int16
+ var arngType int16
+ var simdType int16
+ var simdReg int16
switch {
case reg >= loong64.REG_V0 && reg <= loong64.REG_V31:
- simd_type = loong64.LSX
+ simdType = loong64.LSX
+ simdReg = reg - loong64.REG_V0
case reg >= loong64.REG_X0 && reg <= loong64.REG_X31:
- simd_type = loong64.LASX
+ simdType = loong64.LASX
+ simdReg = reg - loong64.REG_X0
default:
return errors.New("Loong64 extension: invalid LSX/LASX register: " + fmt.Sprintf("%d", reg))
}
if isIndex {
- arng_type, ok = loong64ElemExtMap[ext]
+ arngType, ok = loong64ElemExtMap[ext]
if !ok {
return errors.New("Loong64 extension: invalid LSX/LASX arrangement type: " + ext)
}
a.Reg = loong64.REG_ELEM
- a.Reg += ((reg & loong64.EXT_REG_MASK) << loong64.EXT_REG_SHIFT)
- a.Reg += ((arng_type & loong64.EXT_TYPE_MASK) << loong64.EXT_TYPE_SHIFT)
- a.Reg += ((simd_type & loong64.EXT_SIMDTYPE_MASK) << loong64.EXT_SIMDTYPE_SHIFT)
+ a.Reg += ((simdReg & loong64.EXT_REG_MASK) << loong64.EXT_REG_SHIFT)
+ a.Reg += ((arngType & loong64.EXT_TYPE_MASK) << loong64.EXT_TYPE_SHIFT)
+ a.Reg += ((simdType & loong64.EXT_SIMDTYPE_MASK) << loong64.EXT_SIMDTYPE_SHIFT)
a.Index = num
} else {
- switch simd_type {
+ switch simdType {
case loong64.LSX:
- arng_type, ok = loong64LsxArngExtMap[ext]
+ arngType, ok = loong64LsxArngExtMap[ext]
if !ok {
return errors.New("Loong64 extension: invalid LSX arrangement type: " + ext)
}
case loong64.LASX:
- arng_type, ok = loong64LasxArngExtMap[ext]
+ arngType, ok = loong64LasxArngExtMap[ext]
if !ok {
return errors.New("Loong64 extension: invalid LASX arrangement type: " + ext)
}
}
a.Reg = loong64.REG_ARNG
- a.Reg += ((reg & loong64.EXT_REG_MASK) << loong64.EXT_REG_SHIFT)
- a.Reg += ((arng_type & loong64.EXT_TYPE_MASK) << loong64.EXT_TYPE_SHIFT)
- a.Reg += ((simd_type & loong64.EXT_SIMDTYPE_MASK) << loong64.EXT_SIMDTYPE_SHIFT)
+ a.Reg += ((simdReg & loong64.EXT_REG_MASK) << loong64.EXT_REG_SHIFT)
+ a.Reg += ((arngType & loong64.EXT_TYPE_MASK) << loong64.EXT_TYPE_SHIFT)
+ a.Reg += ((simdType & loong64.EXT_SIMDTYPE_MASK) << loong64.EXT_SIMDTYPE_SHIFT)
}
return nil
diff --git a/src/cmd/asm/internal/arch/riscv64.go b/src/cmd/asm/internal/arch/riscv64.go
index 891d2be0e5cf20..b7ee8c27cc0bb9 100644
--- a/src/cmd/asm/internal/arch/riscv64.go
+++ b/src/cmd/asm/internal/arch/riscv64.go
@@ -47,6 +47,22 @@ func IsRISCV64CSRO(op obj.As) (imm bool, ok bool) {
return
}
+// IsRISCV64PseudoCSRO reports whether the op is a pseudo instruction
+// that uses CSR symbolic names, whether that instruction expects a register
+// or an immediate source operand and what the expected index of the operand
+// containing the CSR name should be.
+func IsRISCV64PseudoCSRO(op obj.As) (imm bool, index int, ok bool) {
+ switch op {
+ case riscv.ACSRCI, riscv.ACSRSI, riscv.ACSRWI:
+ imm, index, ok = true, 1, true
+ case riscv.ACSRC, riscv.ACSRS, riscv.ACSRW:
+ imm, index, ok = false, 1, true
+ case riscv.ACSRR:
+ imm, index, ok = false, 0, true
+ }
+ return
+}
+
var riscv64SpecialOperand map[string]riscv.SpecialOperand
// RISCV64SpecialOperand returns the internal representation of a special operand.
@@ -69,6 +85,10 @@ func RISCV64SpecialOperand(name string) riscv.SpecialOperand {
}
riscv64SpecialOperand[csrName] = riscv.SpecialOperand(int(csrCode) + int(riscv.SPOP_CSR_BEGIN))
}
+ // Add the FENCE operands
+ for opd := riscv.SPOP_FENCE_BEGIN + 1; opd < riscv.SPOP_FENCE_END; opd++ {
+ riscv64SpecialOperand[opd.String()] = opd
+ }
}
if opd, ok := riscv64SpecialOperand[name]; ok {
return opd
diff --git a/src/cmd/asm/internal/asm/asm.go b/src/cmd/asm/internal/asm/asm.go
index 0f75edf4e5d254..731fb26eb638c9 100644
--- a/src/cmd/asm/internal/asm/asm.go
+++ b/src/cmd/asm/internal/asm/asm.go
@@ -15,6 +15,7 @@ import (
"cmd/asm/internal/flags"
"cmd/asm/internal/lex"
"cmd/internal/obj"
+ "cmd/internal/obj/arm64"
"cmd/internal/obj/ppc64"
"cmd/internal/obj/riscv"
"cmd/internal/obj/x86"
@@ -593,6 +594,10 @@ func (p *Parser) branch(addr *obj.Addr, target *obj.Prog) {
addr.Val = target
}
+func isARM64SVE(op obj.As) bool {
+ return op > arm64.ASVESTART
+}
+
// asmInstruction assembles an instruction.
// MOVW R9, (R10)
func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
@@ -660,6 +665,25 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
prog.AddRestSource(a[1])
break
}
+ } else if p.arch.Family == sys.RISCV64 {
+ // RISCV64 pseudo instructions that reference CSRs with symbolic names.
+ if isImm, specialIndex, ok := arch.IsRISCV64PseudoCSRO(op); ok {
+ if a[0].Type != obj.TYPE_CONST && isImm {
+ p.errorf("invalid value for first operand to %s instruction, must be a 5 bit unsigned immediate", op)
+ return
+ }
+ if a[specialIndex].Type != obj.TYPE_SPECIAL {
+ p.errorf("invalid value for first or second operand to %s instruction, must be a CSR name", op)
+ return
+ }
+ prog.AddRestSourceArgs([]obj.Addr{a[specialIndex]})
+ if specialIndex == 0 {
+ prog.To = a[1]
+ } else {
+ prog.From = a[0]
+ }
+ break
+ }
}
prog.From = a[0]
prog.To = a[1]
@@ -741,6 +765,11 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
// For ARM64 CASP-like instructions, its 2nd destination operand is register pair(Rt, Rt+1) that can
// not fit into prog.RegTo2, so save it to the prog.RestArgs.
prog.AddRestDest(a[2])
+ case isARM64SVE(op):
+ // SVE instructions, see arm64/goops_gen.go
+ prog.From = a[0]
+ prog.AddRestSource(a[1])
+ prog.To = a[2]
default:
prog.From = a[0]
prog.Reg = p.getRegister(prog, op, &a[1])
@@ -843,6 +872,13 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
break
}
if p.arch.Family == sys.ARM64 {
+ if isARM64SVE(op) {
+ // SVE instructions, see arm64/goops_gen.go
+ prog.From = a[0]
+ prog.AddRestSourceArgs([]obj.Addr{a[1], a[2]})
+ prog.To = a[3]
+ break
+ }
prog.From = a[0]
prog.Reg = p.getRegister(prog, op, &a[1])
prog.AddRestSource(a[2])
@@ -891,6 +927,13 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
p.errorf("can't handle %s instruction with 4 operands", op)
return
case 5:
+ if p.arch.Family == sys.ARM64 && isARM64SVE(op) {
+ // SVE instructions, see arm64/goops_gen.go
+ prog.From = a[0]
+ prog.AddRestSourceArgs([]obj.Addr{a[1], a[2], a[3]})
+ prog.To = a[4]
+ break
+ }
if p.arch.Family == sys.PPC64 {
prog.From = a[0]
// Second arg is always a register type on ppc64.
@@ -914,6 +957,13 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
p.errorf("can't handle %s instruction with 5 operands", op)
return
case 6:
+ if p.arch.Family == sys.ARM64 && isARM64SVE(op) {
+ // SVE instructions, see arm64/goops_gen.go
+ prog.From = a[0]
+ prog.AddRestSourceArgs([]obj.Addr{a[1], a[2], a[3], a[4]})
+ prog.To = a[5]
+ break
+ }
if p.arch.Family == sys.ARM && arch.IsARMMRC(op) {
// Strange special case: MCR, MRC.
prog.To.Type = obj.TYPE_CONST
diff --git a/src/cmd/asm/internal/asm/endtoend_test.go b/src/cmd/asm/internal/asm/endtoend_test.go
index afaf02815f94a2..e2f23ff61240a0 100644
--- a/src/cmd/asm/internal/asm/endtoend_test.go
+++ b/src/cmd/asm/internal/asm/endtoend_test.go
@@ -38,7 +38,7 @@ func testEndToEnd(t *testing.T, goarch, file string) {
ctxt.IsAsm = true
defer ctxt.Bso.Flush()
failed := false
- ctxt.DiagFunc = func(format string, args ...interface{}) {
+ ctxt.DiagFunc = func(format string, args ...any) {
failed = true
t.Errorf(format, args...)
}
@@ -193,12 +193,17 @@ Diff:
top := pList.Firstpc
var text *obj.LSym
ok = true
- ctxt.DiagFunc = func(format string, args ...interface{}) {
+ ctxt.DiagFunc = func(format string, args ...any) {
t.Errorf(format, args...)
ok = false
}
obj.Flushplist(ctxt, pList, nil)
+ if !ok {
+ // If we've encountered errors, the output is unlikely to be sane.
+ t.FailNow()
+ }
+
for p := top; p != nil; p = p.Link {
if p.As == obj.ATEXT {
text = p.From.Sym
@@ -294,7 +299,7 @@ func testErrors(t *testing.T, goarch, file string, flags ...string) {
failed := false
var errBuf bytes.Buffer
parser.errorWriter = &errBuf
- ctxt.DiagFunc = func(format string, args ...interface{}) {
+ ctxt.DiagFunc = func(format string, args ...any) {
failed = true
s := fmt.Sprintf(format, args...)
if !strings.HasSuffix(s, "\n") {
@@ -409,6 +414,20 @@ func TestARM64Encoder(t *testing.T) {
testEndToEnd(t, "arm64", "arm64enc")
}
+func TestARM64SVEEncoder(t *testing.T) {
+ if !buildcfg.Experiment.SIMD {
+ t.Skip("test requires GOEXPERIMENT=simd")
+ }
+ testEndToEnd(t, "arm64", "arm64sveenc")
+}
+
+func TestARM64SVEErrors(t *testing.T) {
+ if !buildcfg.Experiment.SIMD {
+ t.Skip("test requires GOEXPERIMENT=simd")
+ }
+ testErrors(t, "arm64", "arm64sveerror")
+}
+
func TestARM64Errors(t *testing.T) {
testErrors(t, "arm64", "arm64error")
}
@@ -467,6 +486,7 @@ func TestLOONG64Encoder(t *testing.T) {
testEndToEnd(t, "loong64", "loong64enc3")
testEndToEnd(t, "loong64", "loong64enc4")
testEndToEnd(t, "loong64", "loong64enc5")
+ testEndToEnd(t, "loong64", "loong64enc6")
testEndToEnd(t, "loong64", "loong64")
}
@@ -485,16 +505,35 @@ func TestPPC64EndToEnd(t *testing.T) {
}
}
-func TestRISCVEndToEnd(t *testing.T) {
- testEndToEnd(t, "riscv64", "riscv64")
+func testRISCV64AllProfiles(t *testing.T, testFn func(t *testing.T)) {
+ t.Helper()
+
+ defer func(orig int) { buildcfg.GORISCV64 = orig }(buildcfg.GORISCV64)
+
+ for _, goriscv64 := range []int{20, 22, 23} {
+ t.Run(fmt.Sprintf("rva%vu64", goriscv64), func(t *testing.T) {
+ buildcfg.GORISCV64 = goriscv64
+ testFn(t)
+ })
+ }
+}
+
+func TestRISCV64EndToEnd(t *testing.T) {
+ testRISCV64AllProfiles(t, func(t *testing.T) {
+ testEndToEnd(t, "riscv64", "riscv64")
+ })
}
-func TestRISCVErrors(t *testing.T) {
- testErrors(t, "riscv64", "riscv64error")
+func TestRISCV64Errors(t *testing.T) {
+ testRISCV64AllProfiles(t, func(t *testing.T) {
+ testErrors(t, "riscv64", "riscv64error")
+ })
}
-func TestRISCVValidation(t *testing.T) {
- testErrors(t, "riscv64", "riscv64validation")
+func TestRISCV64Validation(t *testing.T) {
+ testRISCV64AllProfiles(t, func(t *testing.T) {
+ testErrors(t, "riscv64", "riscv64validation")
+ })
}
func TestS390XEndToEnd(t *testing.T) {
diff --git a/src/cmd/asm/internal/asm/parse.go b/src/cmd/asm/internal/asm/parse.go
index 545f6c7553351b..bf69f9d1bd7083 100644
--- a/src/cmd/asm/internal/asm/parse.go
+++ b/src/cmd/asm/internal/asm/parse.go
@@ -78,7 +78,7 @@ func NewParser(ctxt *obj.Link, ar *arch.Arch, lexer lex.TokenReader) *Parser {
// and turn it into a recoverable panic.
var panicOnError bool
-func (p *Parser) errorf(format string, args ...interface{}) {
+func (p *Parser) errorf(format string, args ...any) {
if panicOnError {
panic(fmt.Errorf(format, args...))
}
@@ -90,7 +90,7 @@ func (p *Parser) errorf(format string, args ...interface{}) {
if p.lex != nil {
// Put file and line information on head of message.
format = "%s:%d: " + format + "\n"
- args = append([]interface{}{p.lex.File(), p.lineNum}, args...)
+ args = append([]any{p.lex.File(), p.lineNum}, args...)
}
fmt.Fprintf(p.errorWriter, format, args...)
p.errorCount++
@@ -423,6 +423,70 @@ func (p *Parser) operand(a *obj.Addr) {
}
}
+ // Detect AC_PREGSEL operand syntax: [selreg, $idximm](preg.T2)
+ if tok.ScanToken == '[' && p.arch.Family == sys.ARM64 {
+ pos := p.inputPos
+ if pos+5 < len(p.input) && p.input[pos].ScanToken == scanner.Ident &&
+ p.input[pos+1].ScanToken == ',' && p.input[pos+2].ScanToken == '$' &&
+ p.input[pos+3].ScanToken == scanner.Int && p.input[pos+4].ScanToken == ']' &&
+ p.input[pos+5].ScanToken == '(' {
+
+ selregStr := p.input[pos].String()
+ immStr := p.input[pos+3].String()
+
+ pos += 6 // past '('
+ if pos < len(p.input) && p.input[pos].ScanToken == scanner.Ident {
+ pregStr := p.input[pos].String()
+ pos++
+ ext := ""
+ if pos < len(p.input) && p.input[pos].ScanToken == '.' {
+ pos++
+ if pos < len(p.input) && p.input[pos].ScanToken == scanner.Ident {
+ ext = p.input[pos].String()
+ pos++
+ }
+ }
+ if pos < len(p.input) && p.input[pos].ScanToken == ')' {
+ // Matches AC_PREGSEL syntax!
+ p.inputPos = pos + 1 // consume the parsed tokens
+
+ // Encode components
+ selreg, ok1 := p.registerReference(selregStr)
+ preg, ok2 := p.registerReference(pregStr)
+ idximm := int64(p.atoi(immStr))
+
+ if !ok1 || !ok2 {
+ p.errorf("invalid registers in pregsel operand")
+ return
+ }
+
+ var arng2 int
+ switch ext {
+ case "B":
+ arng2 = arm64.ARNG_B
+ case "H":
+ arng2 = arm64.ARNG_H
+ case "S":
+ arng2 = arm64.ARNG_S
+ case "D":
+ arng2 = arm64.ARNG_D
+ case "Q":
+ arng2 = arm64.ARNG_Q
+ default:
+ p.errorf("invalid P register arrangement: %s", ext)
+ return
+ }
+
+ // Pack values into a.Offset
+ a.Type = obj.TYPE_REG
+ a.Offset = int64(preg&31) | int64((arng2&15)<<5) | int64((selreg&31)<<9) | (idximm << 14) | (int64(1) << 62)
+ p.expectOperandEnd()
+ return
+ }
+ }
+ }
+ }
+
// Special register list syntax for arm: [R1,R3-R7]
if tok.ScanToken == '[' {
if prefix != 0 {
@@ -472,6 +536,30 @@ func (p *Parser) operand(a *obj.Addr) {
return
}
+ // Detect (VL*imm) or (-VL*imm) pattern in ARM64
+ if p.arch.Family == sys.ARM64 && len(p.input) >= 5 && p.input[0].ScanToken == '(' {
+ pos := 1
+ sign := int64(1)
+ if p.input[pos].ScanToken == '-' {
+ sign = -1
+ pos++
+ } else if p.input[pos].ScanToken == '+' {
+ pos++
+ }
+ if pos+3 < len(p.input) && p.input[pos].String() == "VL" && p.input[pos+1].ScanToken == '*' && p.input[pos+2].ScanToken == scanner.Int && p.input[pos+3].ScanToken == ')' {
+ imm := int64(p.atoi(p.input[pos+2].String()))
+ imm *= sign
+ a.Offset = imm
+ a.Scale = -32768 // bit 15 signals multiple of Vector Length
+
+ // Remove the (VL*imm) tokens and let operand parse the remaining sequence (reg.T)
+ p.input = p.input[pos+4:]
+ p.inputPos = 0
+ p.operand(a)
+ return
+ }
+ }
+
// Constant.
haveConstant := false
switch tok.ScanToken {
@@ -579,7 +667,7 @@ func (p *Parser) atRegisterExtension() bool {
switch p.arch.Family {
case sys.ARM64, sys.Loong64:
// R1.xxx
- return p.peek() == '.'
+ return p.peek() == '.' || p.peek() == '['
default:
return false
}
@@ -743,7 +831,7 @@ func (p *Parser) registerExtension(a *obj.Addr, name string, prefix rune) {
if p.peek() == lex.LSH {
// (Rn)(Rm<<2), the shifted offset register.
ext = "LSL"
- } else {
+ } else if p.peek() == '.' {
// (Rn)(Rm.UXTW<1), the extended offset register.
// Rm.UXTW<<3, the extended register.
p.get('.')
@@ -775,7 +863,7 @@ func (p *Parser) registerExtension(a *obj.Addr, name string, prefix rune) {
switch p.arch.Family {
case sys.ARM64:
- err := arm64.ARM64RegisterExtension(a, ext, reg, num, isAmount, isIndex)
+ err := arm64.EncodeRegisterExtension(a, ext, reg, num, isAmount, isIndex)
if err != nil {
p.errorf("%v", err)
}
@@ -971,8 +1059,133 @@ func (p *Parser) registerIndirect(a *obj.Addr, prefix rune) {
if !ok {
p.errorf("indirect through non-register %s", tok)
}
+ // SVE extended addressing support
+ var ext string
+ var mod int64
+ var amount int64
+ isSVEMemExt := false
+
+ if p.arch.Family == sys.ARM64 && (p.peek() == '.' || p.peek() == lex.LSH) {
+ // The index is a vector register, or contains a shift, then this is SVE extended addressing.
+ isSVEMemExt = true
+ if p.peek() == '.' {
+ p.get('.')
+ tok2 := p.next()
+ str := tok2.String()
+ if str == "UXTW" || str == "SXTW" {
+ switch str {
+ case "UXTW":
+ mod = 1
+ case "SXTW":
+ mod = 2
+ }
+ } else {
+ ext = str
+ if p.peek() == '.' {
+ p.get('.')
+ tok3 := p.next()
+ modStr := tok3.String()
+ switch modStr {
+ case "UXTW":
+ mod = 1
+ case "SXTW":
+ mod = 2
+ default:
+ p.errorf("unknown modifier %s", modStr)
+ }
+ }
+ }
+ }
+ if p.peek() == lex.LSH {
+ p.get(lex.LSH)
+ tok4 := p.get(scanner.Int)
+ amount, _ = strconv.ParseInt(tok4.String(), 10, 16)
+ }
+ }
p.get(')')
a.Type = obj.TYPE_MEM
+
+ if isSVEMemExt {
+ encodedR1 := r1
+ if ext != "" {
+ if r1 >= arm64.REG_Z0 && r1 <= arm64.REG_Z31 {
+ var arng int
+ switch ext {
+ case "B":
+ arng = arm64.ARNG_B
+ case "H":
+ arng = arm64.ARNG_H
+ case "S":
+ arng = arm64.ARNG_S
+ case "D":
+ arng = arm64.ARNG_D
+ case "Q":
+ arng = arm64.ARNG_Q
+ default:
+ p.errorf("unknown arrangement %s", ext)
+ }
+ if arng != 0 {
+ encodedR1 = arm64.REG_ZARNG + (r1 & 31) + int16((arng&15)<<5)
+ }
+ } else {
+ p.errorf("arrangement not allowed for this register")
+ }
+ }
+
+ if p.peek() != '(' {
+ a.Reg = encodedR1
+ return
+ }
+ p.get('(')
+ tok5 := p.next()
+ r2, ok := p.registerReference(tok5.String())
+ if !ok {
+ p.errorf("expected register")
+ }
+ var ext2 string
+ if p.peek() == '.' {
+ p.get('.')
+ tok6 := p.next()
+ ext2 = tok6.String()
+ }
+ p.get(')')
+
+ encodedR2 := r2
+ if ext2 != "" {
+ if r2 >= arm64.REG_Z0 && r2 <= arm64.REG_Z31 {
+ var arng int
+ switch ext2 {
+ case "B":
+ arng = arm64.ARNG_B
+ case "H":
+ arng = arm64.ARNG_H
+ case "S":
+ arng = arm64.ARNG_S
+ case "D":
+ arng = arm64.ARNG_D
+ case "Q":
+ arng = arm64.ARNG_Q
+ default:
+ p.errorf("unknown arrangement %s", ext2)
+ }
+ if arng != 0 {
+ encodedR2 = arm64.REG_ZARNG + (r2 & 31) + int16((arng&15)<<5)
+ }
+ } else {
+ p.errorf("arrangement not allowed for this register")
+ }
+ }
+
+ a.Index = encodedR2 // Base
+ a.Reg = encodedR1 // Index
+ a.Offset = 0 // Ensure offset is 0
+
+ var scaleValue int16 = -32768 // Bit 15 set
+ scaleValue |= int16(amount << 12)
+ scaleValue |= int16(mod << 9)
+ a.Scale = scaleValue
+ return
+ }
if r1 < 0 {
// Pseudo-register reference.
if r2 != 0 {
@@ -1117,13 +1330,26 @@ ListLoop:
}
switch p.arch.Family {
case sys.ARM64:
- // Vn.T
+ // Vn.T, Zn.T, Pn.T
name := tok.String()
r, ok := p.registerReference(name)
if !ok {
p.errorf("invalid register: %s", name)
}
- reg := r - p.arch.Register["V0"]
+ var registerBase int16
+ registerCntMask := 31
+ switch name[0] {
+ case 'V':
+ registerBase = p.arch.Register["V0"]
+ case 'Z':
+ registerBase = p.arch.Register["Z0"]
+ case 'P':
+ registerBase = p.arch.Register["P0"]
+ registerCntMask = 15
+ default:
+ p.errorf("invalid register in register list: %s", name)
+ }
+ reg := r - registerBase
p.get('.')
tok := p.next()
ext := tok.String()
@@ -1141,8 +1367,33 @@ ListLoop:
} else if nextReg != int(reg) {
p.errorf("incontiguous register in ARM64 register list: %s", name)
}
- regCnt++
- nextReg = (nextReg + 1) % 32
+ if p.peek() == '-' && name[0] == 'Z' {
+ // register range in SVE
+ p.next()
+ nameHi := p.next().String()
+ if nameHi[0] != 'Z' {
+ p.errorf("invalid register in register range: %s", nameHi)
+ }
+ rHi, ok2 := p.registerReference(nameHi)
+ if !ok2 {
+ p.errorf("invalid register: %s", nameHi)
+ }
+ regHi := rHi - registerBase
+ p.get('.')
+ extHi := p.next().String()
+ hiArrangement, err := arch.ARM64RegisterArrangement(regHi, nameHi, extHi)
+ if err != nil {
+ p.errorf("%v", err)
+ }
+ if hiArrangement != arrangement {
+ p.errorf("inconsistent arrangement in ARM64 register list range")
+ }
+ // Scale stores the range size - 1.
+ a.Scale = regHi - reg
+ } else {
+ regCnt++
+ nextReg = (nextReg + 1) & registerCntMask
+ }
case sys.ARM:
// Parse the upper and lower bounds.
lo := p.registerNumber(tok.String())
@@ -1174,7 +1425,7 @@ ListLoop:
case sys.ARM:
a.Offset = int64(bits)
case sys.ARM64:
- offset, err := arch.ARM64RegisterListOffset(firstReg, regCnt, arrangement)
+ offset, err := arm64.RegisterListOffset(firstReg, regCnt, arrangement, a.Scale)
if err != nil {
p.errorf("%v", err)
}
diff --git a/src/cmd/asm/internal/asm/testdata/amd64dynlinkerror.s b/src/cmd/asm/internal/asm/testdata/amd64dynlinkerror.s
index 4bf58a39a43266..8b104307cd2697 100644
--- a/src/cmd/asm/internal/asm/testdata/amd64dynlinkerror.s
+++ b/src/cmd/asm/internal/asm/testdata/amd64dynlinkerror.s
@@ -169,3 +169,8 @@ TEXT ·a34(SB), 0, $0-0
SHLXQ AX, CX, R15
ADDQ $1, R15
RET
+
+// Ensure from3 get GOT-rewritten without errors.
+TEXT ·a35(SB), 0, $0-0
+ VGF2P8AFFINEQB $0, runtime·writeBarrier(SB), Z1, Z1
+ RET
diff --git a/src/cmd/asm/internal/asm/testdata/amd64enc.s b/src/cmd/asm/internal/asm/testdata/amd64enc.s
index 5bba292dee43f6..c93c479735ae39 100644
--- a/src/cmd/asm/internal/asm/testdata/amd64enc.s
+++ b/src/cmd/asm/internal/asm/testdata/amd64enc.s
@@ -2425,6 +2425,15 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
MOVQ $-249346713, (R11) // 49c703674523f1
MOVQ $-249346713, DX // 48c7c2674523f1
MOVQ $-249346713, R11 // 49c7c3674523f1
+ // Positive immediates that fit in 32 bits use the shorter 5-byte
+ // MOV r64, imm32 (zero-extended) form rather than the 7-byte sign-
+ // extended c7 /0 form. See Ziq_rp in cmd/internal/obj/x86/asm6.go.
+ MOVQ $305419896, DX // ba78563412
+ MOVQ $305419896, R11 // 41bb78563412
+ MOVQ $2147483648, DX // ba00000080
+ MOVQ $2147483648, R11 // 41bb00000080
+ MOVQ $4294967295, DX // baffffffff
+ MOVQ $4294967295, R11 // 41bbffffffff
MOVQ DX, (BX) // 488913
MOVQ R11, (BX) // 4c891b
MOVQ DX, (R11) // 498913
diff --git a/src/cmd/asm/internal/asm/testdata/arm.s b/src/cmd/asm/internal/asm/testdata/arm.s
index 93edc8854ead47..fcc6ac32df4d0d 100644
--- a/src/cmd/asm/internal/asm/testdata/arm.s
+++ b/src/cmd/asm/internal/asm/testdata/arm.s
@@ -1089,10 +1089,23 @@ jmp_label_3:
RSC $0xffffff4b, R5 // RSC $4294967115, R5 // b4b0e0e30b50e5e0
RSC.S $0xffffffb5, R2, R3 // RSC.S $4294967221, R2, R3 // 4ab0e0e30b30f2e0
RSC.S $0xffffff4a, R5 // RSC.S $4294967114, R5 // b5b0e0e30b50f5e0
- AND $0xffffffaa, R2, R3 // AND $4294967210, R2, R3 // 55b0e0e30b3002e0
- AND $0xffffff55, R5 // AND $4294967125, R5 // aab0e0e30b5005e0
- AND.S $0xffffffab, R2, R3 // AND.S $4294967211, R2, R3 // 54b0e0e30b3012e0
- AND.S $0xffffff54, R5 // AND.S $4294967124, R5 // abb0e0e30b5015e0
+ AND $0xffffffaa, R2, R3 // AND $4294967210, R2, R3 // 5530c2e3
+ AND $0xffffff55, R5 // AND $4294967125, R5 // aa50c5e3
+ AND.S $0xffffffab, R2, R3 // AND.S $4294967211, R2, R3 // 5430d2e3
+ AND.S $0xffffff54, R5 // AND.S $4294967124, R5 // ab50d5e3
+ // AND with a negative-rotated immediate: emit a single BIC,
+ // not the two-instruction MVN+AND synthesis (which clobbers R11).
+ AND $0xfffffffc, R0, R1 // AND $4294967292, R0, R1 // 0310c0e3
+ AND $0xfffffff0, R0, R1 // AND $4294967280, R0, R1 // 0f10c0e3
+ AND $0xffffff00, R0, R1 // AND $4294967040, R0, R1 // ff10c0e3
+ AND $0xfffffffc, R8 // AND $4294967292, R8 // 0380c8e3
+ AND.S $0xfffffffc, R0, R1 // AND.S $4294967292, R0, R1 // 0310d0e3
+ // AND with an irregular immediate (neither rot8 nor ~rot8 fit)
+ // still goes through the literal-pool synthesis: MOVW pool(PC),
+ // R11; AND R11, Rn, Rd. Encoded form depends on pool placement,
+ // so just round-trip the disassembly.
+ AND $0xaaaaaaaa, R0, R1 // AND $2863311530, R0, R1
+ AND.S $0xaaaaaaaa, R0, R1 // AND.S $2863311530, R0, R1
ORR $0xffffffaa, R2, R3 // ORR $4294967210, R2, R3 // 55b0e0e30b3082e1
ORR $0xffffff55, R5 // ORR $4294967125, R5 // aab0e0e30b5085e1
ORR.S $0xffffffab, R2, R3 // ORR.S $4294967211, R2, R3 // 54b0e0e30b3092e1
diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s
index 109a3d8316678b..be4bf0c65a58e9 100644
--- a/src/cmd/asm/internal/asm/testdata/arm64.s
+++ b/src/cmd/asm/internal/asm/testdata/arm64.s
@@ -110,8 +110,50 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VADDP V1.D2, V2.D2, V3.D2 // 43bce14e
VAND V21.B8, V12.B8, V3.B8 // 831d350e
VCMEQ V1.H4, V2.H4, V3.H4 // 438c612e
+ VCMEQ $0, V2.H4, V3.H4 // 4398600e
+ VCMGE V1.H4, V2.H4, V3.H4 // 433c610e
+ VCMGE $0, V2.H4, V3.H4 // 4388602e
+ VCMGT V1.H4, V2.H4, V3.H4 // 4334610e
+ VCMGT $0, V2.H4, V3.H4 // 4388600e
+ VCMHI V1.H4, V2.H4, V3.H4 // 4334612e
+ VCMHS V1.H4, V2.H4, V3.H4 // 433c612e
+ VCMLE $0, V2.H4, V3.H4 // 4398602e
+ VCMLT $0, V2.H4, V3.H4 // 43a8600e
+ VFCMEQ V1.S4, V2.S4, V3.S4 // 43e4214e
+ VFCMEQ $(0.0), V2.S4, V3.S4 // 43d8a04e
+ VFCMGE V1.S4, V2.S4, V3.S4 // 43e4216e
+ VFCMGE $(0.0), V2.S4, V3.S4 // 43c8a06e
+ VFCMGT V1.S4, V2.S4, V3.S4 // 43e4a16e
+ VFCMGT $(0.0), V2.S4, V3.S4 // 43c8a04e
+ VFCMLE $(0.0), V2.S4, V3.S4 // 43d8a06e
+ VFCMLT $(0.0), V2.S4, V3.S4 // 43e8a04e
+ VFADD V0.S4, V0.S4, V1.S4 // 01d4204e
+ VFADD V0.D2, V0.D2, V1.D2 // 01d4604e
+ VFSUB V0.S4, V0.S4, V1.S4 // 01d4a04e
+ VFSUB V0.D2, V0.D2, V1.D2 // 01d4e04e
+ VFMUL V0.S4, V0.S4, V1.S4 // 01dc206e
+ VFMUL V0.D2, V0.D2, V1.D2 // 01dc606e
+ VFDIV V0.S4, V0.S4, V1.S4 // 01fc206e
+ VFDIV V0.D2, V0.D2, V1.D2 // 01fc606e
+ VSQADD V0.S4, V0.S4, V1.S4 // 010ca04e
+ VSQADD V0.D2, V0.D2, V1.D2 // 010ce04e
+ VUQADD V0.S4, V0.S4, V1.S4 // 010ca06e
+ VUQADD V0.D2, V0.D2, V1.D2 // 010ce06e
+ VSQSUB V0.S4, V0.S4, V1.S4 // 012ca04e
+ VSQSUB V0.D2, V0.D2, V1.D2 // 012ce04e
+ VUQSUB V0.S4, V0.S4, V1.S4 // 012ca06e
+ VUQSUB V0.D2, V0.D2, V1.D2 // 012ce06e
+ VMUL V0.S4, V0.S4, V1.S4 // 019ca04e
+ VMLA V0.S4, V0.S4, V1.S4 // 0194a04e
+ VMLS V0.S4, V0.S4, V1.S4 // 0194a06e
VORR V5.B16, V4.B16, V3.B16 // 831ca54e
+ VBIC V0.B8, V1.B8, V2.B8 // 221c600e
+ VORN V0.B16, V1.B16, V2.B16 // 221ce04e
VADD V16.S4, V5.S4, V9.S4 // a984b04e
+ VSHADD V0.S4, V1.S4, V2.S4 // 2204a04e
+ VSRHADD V0.S4, V1.S4, V2.S4 // 2214a04e
+ VUHADD V0.S4, V1.S4, V2.S4 // 2204a06e
+ VURHADD V0.S4, V1.S4, V2.S4 // 2214a06e
VEOR V0.B16, V1.B16, V0.B16 // 201c206e
VADDV V0.S4, V0 // 00b8b14e
VMOVI $82, V0.B16 // 40e6024f
@@ -137,6 +179,41 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VUSHR $8, V1.H8, V2.H8 // 2204186f
VUSHR $2, V1.B8, V2.B8 // 22040e2f
VUSHR $2, V1.B16, V2.B16 // 22040e6f
+ VSHRN $7, V1.H8, V0.B8 // 2084090f
+ VSHRN $15, V1.S4, V0.H4 // 2084110f
+ VSHRN $31, V1.D2, V0.S2 // 2084210f
+ VSHRN2 $7, V1.H8, V0.B16 // 2084094f
+ VSHRN2 $15, V1.S4, V0.H8 // 2084114f
+ VSHRN2 $31, V1.D2, V0.S4 // 2084214f
+ VSSHR $2, V0.B8, V1.B8 // 01040e0f
+ VSSHR $2, V0.B16, V1.B16 // 01040e4f
+ VSSHR $8, V0.H4, V1.H4 // 0104180f
+ VSSHR $8, V0.H8, V1.H8 // 0104184f
+ VSSHR $16, V0.S2, V1.S2 // 0104300f
+ VSSHR $16, V0.S4, V1.S4 // 0104304f
+ VSSHR $32, V0.D2, V1.D2 // 0104604f
+ VSRSHR $2, V0.B8, V1.B8 // 01240e0f
+ VSRSHR $2, V0.B16, V1.B16 // 01240e4f
+ VSRSHR $8, V0.H4, V1.H4 // 0124180f
+ VSRSHR $8, V0.H8, V1.H8 // 0124184f
+ VSRSHR $16, V0.S2, V1.S2 // 0124300f
+ VSRSHR $16, V0.S4, V1.S4 // 0124304f
+ VSRSHR $32, V0.D2, V1.D2 // 0124604f
+ VFSQRT V0.S2, V1.S2 // 01f8a12e
+ VFSQRT V0.S4, V1.S4 // 01f8a16e
+ VFSQRT V0.D2, V1.D2 // 01f8e16e
+ VFRINTN V0.S2, V1.S2 // 0188210e
+ VFRINTN V0.S4, V1.S4 // 0188214e
+ VFRINTN V0.D2, V1.D2 // 0188614e
+ VFRINTP V0.S2, V1.S2 // 0188a10e
+ VFRINTP V0.S4, V1.S4 // 0188a14e
+ VFRINTP V0.D2, V1.D2 // 0188e14e
+ VFRINTM V0.S2, V1.S2 // 0198210e
+ VFRINTM V0.S4, V1.S4 // 0198214e
+ VFRINTM V0.D2, V1.D2 // 0198614e
+ VFRINTZ V0.S2, V1.S2 // 0198a10e
+ VFRINTZ V0.S4, V1.S4 // 0198a14e
+ VFRINTZ V0.D2, V1.D2 // 0198e14e
VSHL $56, V1.D2, V2.D2 // 2254784f
VSHL $24, V1.S4, V2.S4 // 2254384f
VSHL $24, V1.S2, V2.S2 // 2254380f
@@ -144,6 +221,48 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VSHL $8, V1.H8, V2.H8 // 2254184f
VSHL $2, V1.B8, V2.B8 // 22540a0f
VSHL $2, V1.B16, V2.B16 // 22540a4f
+ VSQSHL $56, V1.D2, V2.D2 // 2274784f
+ VSQSHL $24, V1.S4, V2.S4 // 2274384f
+ VSQSHL $24, V1.S2, V2.S2 // 2274380f
+ VSQSHL $8, V1.H4, V2.H4 // 2274180f
+ VSQSHL $8, V1.H8, V2.H8 // 2274184f
+ VSQSHL $2, V1.B8, V2.B8 // 22740a0f
+ VSQSHL $2, V1.B16, V2.B16 // 22740a4f
+ VUQSHL $56, V1.D2, V2.D2 // 2274786f
+ VUQSHL $24, V1.S4, V2.S4 // 2274386f
+ VUQSHL $24, V1.S2, V2.S2 // 2274382f
+ VUQSHL $8, V1.H4, V2.H4 // 2274182f
+ VUQSHL $8, V1.H8, V2.H8 // 2274186f
+ VUQSHL $2, V1.B8, V2.B8 // 22740a2f
+ VUQSHL $2, V1.B16, V2.B16 // 22740a6f
+ VSSHL V1.D2, V2.D2, V3.D2 // 4344e14e
+ VSSHL V1.S4, V2.S4, V3.S4 // 4344a14e
+ VSSHL V1.S2, V2.S2, V3.S2 // 4344a10e
+ VSSHL V1.H4, V2.H4, V3.H4 // 4344610e
+ VSSHL V1.H8, V2.H8, V3.H8 // 4344614e
+ VSSHL V1.B8, V2.B8, V3.B8 // 4344210e
+ VSSHL V1.B16, V2.B16, V3.B16 // 4344214e
+ VUSHL V1.D2, V2.D2, V3.D2 // 4344e16e
+ VUSHL V1.S4, V2.S4, V3.S4 // 4344a16e
+ VUSHL V1.S2, V2.S2, V3.S2 // 4344a12e
+ VUSHL V1.H4, V2.H4, V3.H4 // 4344612e
+ VUSHL V1.H8, V2.H8, V3.H8 // 4344616e
+ VUSHL V1.B8, V2.B8, V3.B8 // 4344212e
+ VUSHL V1.B16, V2.B16, V3.B16 // 4344216e
+ VSQSHL V1.D2, V2.D2, V3.D2 // 434ce14e
+ VSQSHL V1.S4, V2.S4, V3.S4 // 434ca14e
+ VSQSHL V1.S2, V2.S2, V3.S2 // 434ca10e
+ VSQSHL V1.H4, V2.H4, V3.H4 // 434c610e
+ VSQSHL V1.H8, V2.H8, V3.H8 // 434c614e
+ VSQSHL V1.B8, V2.B8, V3.B8 // 434c210e
+ VSQSHL V1.B16, V2.B16, V3.B16 // 434c214e
+ VUQSHL V1.D2, V2.D2, V3.D2 // 434ce16e
+ VUQSHL V1.S4, V2.S4, V3.S4 // 434ca16e
+ VUQSHL V1.S2, V2.S2, V3.S2 // 434ca12e
+ VUQSHL V1.H4, V2.H4, V3.H4 // 434c612e
+ VUQSHL V1.H8, V2.H8, V3.H8 // 434c616e
+ VUQSHL V1.B8, V2.B8, V3.B8 // 434c212e
+ VUQSHL V1.B16, V2.B16, V3.B16 // 434c216e
VSRI $56, V1.D2, V2.D2 // 2244486f
VSRI $24, V1.S4, V2.S4 // 2244286f
VSRI $24, V1.S2, V2.S2 // 2244282f
@@ -203,6 +322,36 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VUXTL2 V30.H8, V30.S4 // dea7106f
VUXTL2 V29.S4, V2.D2 // a2a7206f
VUXTL2 V30.B16, V2.H8 // c2a7086f
+ VSXTL V1.B8, V2.H8 // 22a4080f
+ VSXTL V1.H4, V2.S4 // 22a4100f
+ VSXTL V1.S2, V2.D2 // 22a4200f
+ VSXTL2 V1.B16, V2.H8 // 22a4084f
+ VSXTL2 V1.H8, V2.S4 // 22a4104f
+ VSXTL2 V1.S4, V2.D2 // 22a4204f
+ VXTN V1.H8, V2.B8 // 2228210e
+ VXTN V1.S4, V2.H4 // 2228610e
+ VXTN V1.D2, V2.S2 // 2228a10e
+ VXTN2 V1.H8, V2.B16 // 2228214e
+ VXTN2 V1.S4, V2.H8 // 2228614e
+ VXTN2 V1.D2, V2.S4 // 2228a14e
+ VSQXTN V1.H8, V2.B8 // 2248210e
+ VSQXTN V1.S4, V2.H4 // 2248610e
+ VSQXTN V1.D2, V2.S2 // 2248a10e
+ VSQXTN2 V1.H8, V2.B16 // 2248214e
+ VSQXTN2 V1.S4, V2.H8 // 2248614e
+ VSQXTN2 V1.D2, V2.S4 // 2248a14e
+ VSQXTUN V1.H8, V2.B8 // 2228212e
+ VSQXTUN V1.S4, V2.H4 // 2228612e
+ VSQXTUN V1.D2, V2.S2 // 2228a12e
+ VSQXTUN2 V1.H8, V2.B16 // 2228216e
+ VSQXTUN2 V1.S4, V2.H8 // 2228616e
+ VSQXTUN2 V1.D2, V2.S4 // 2228a16e
+ VUQXTN V1.H8, V2.B8 // 2248212e
+ VUQXTN V1.S4, V2.H4 // 2248612e
+ VUQXTN V1.D2, V2.S2 // 2248a12e
+ VUQXTN2 V1.H8, V2.B16 // 2248216e
+ VUQXTN2 V1.S4, V2.H8 // 2248616e
+ VUQXTN2 V1.D2, V2.S4 // 2248a16e
VBIT V21.B16, V25.B16, V4.B16 // 241fb56e
VBSL V23.B16, V3.B16, V7.B16 // 671c776e
VCMTST V2.B8, V29.B8, V2.B8 // a28f220e
@@ -226,6 +375,15 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VUSHLL $7, V30.B8, V30.H8 // dea70f2f
VUSHLL $15, V30.H4, V29.S4 // dda71f2f
VUSHLL2 $31, V30.S4, V2.D2 // c2a73f6f
+ VSSHLL $0, V1.B8, V2.H8 // 22a4080f
+ VSSHLL $0, V1.H4, V2.S4 // 22a4100f
+ VSSHLL $0, V1.S2, V2.D2 // 22a4200f
+ VSSHLL2 $0, V1.B16, V2.H8 // 22a4084f
+ VSSHLL2 $0, V1.H8, V2.S4 // 22a4104f
+ VSSHLL2 $0, V1.S4, V2.D2 // 22a4204f
+ VSSHLL $7, V1.B8, V2.H8 // 22a40f0f
+ VSSHLL $15, V1.H4, V2.S4 // 22a41f0f
+ VSSHLL2 $31, V1.S4, V2.D2 // 22a43f4f
VBIF V0.B8, V30.B8, V1.B8 // c11fe02e
VBIF V30.B16, V0.B16, V2.B16 // 021cfe6e
FMOVS $(4.0), F0 // 0010221e
@@ -258,6 +416,23 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VUMIN V3.H8, V2.H8, V1.H8 // 416c636e
VUMIN V3.S2, V2.S2, V1.S2 // 416ca32e
VUMIN V3.S4, V2.S4, V1.S4 // 416ca36e
+ VSMAX V3.S4, V2.S4, V1.S4 // 4164a34e
+ VSMIN V3.S4, V2.S4, V1.S4 // 416ca34e
+ VSMAXP V3.S4, V2.S4, V1.S4 // 41a4a34e
+ VSMINP V3.S4, V2.S4, V1.S4 // 41aca34e
+ VUMAXP V3.S4, V2.S4, V1.S4 // 41a4a36e
+ VUMINP V3.S4, V2.S4, V1.S4 // 41aca36e
+ VFMAX V3.S4, V2.S4, V1.S4 // 41f4234e
+ VFMIN V3.S4, V2.S4, V1.S4 // 41f4a34e
+ VFMAXNM V3.S4, V2.S4, V1.S4 // 41c4234e
+ VFMINNM V3.S4, V2.S4, V1.S4 // 41c4a34e
+ VFMAXP V3.S4, V2.S4, V1.S4 // 41f4236e
+ VFADDP V3.S4, V2.S4, V1.S4 // 41d4236e
+ VFADDP V3.S2, V2.S2, V1.S2 // 41d4232e
+ VFADDP V3.D2, V2.D2, V1.D2 // 41d4636e
+ VFMINP V3.S4, V2.S4, V1.S4 // 41f4a36e
+ VFMAXNMP V3.S4, V2.S4, V1.S4 // 41c4236e
+ VFMINNMP V3.S4, V2.S4, V1.S4 // 41c4a36e
FCCMPS LT, F1, F2, $1 // 41b4211e
FMADDS F1, F3, F2, F4 // 440c011f
FMADDD F4, F5, F4, F4 // 8414441f
@@ -283,10 +458,14 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
PRFM 8(R12), PLIL3STRM // 8d0580f9
PRFM (R8), $25 // 190180f9
PRFM 8(R9), $30 // 3e0580f9
+ RPRFM (R1), R2, PLDKEEP // 3848a2f8
+ RPRFM (RSP), R4, PSTSTRM // fd4ba4f8
+ RPRFM (R6), R12, $25 // d978acf8
NOOP // 1f2003d5
HINT $0 // 1f2003d5
DMB $1
SVC
+ SB // ff3003d5
// encryption
SHA256H V9.S4, V3, V2 // 6240095e
@@ -309,11 +488,76 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
VPMULL2 V2.D2, V1.D2, V4.Q1 // 24e0e24e
VPMULL V2.B8, V1.B8, V3.H8 // 23e0220e
VPMULL2 V2.B16, V1.B16, V4.H8 // 24e0224e
+ VUMULL V0.H4, V1.H4, V21.S4 // 35c0602e
+ VUMULL2 V0.H8, V1.H8, V22.S4 // 36c0606e
+ VUMULL V22.H4, V31.H4, V23.S4 // f7c3762e
+ VUMULL2 V22.H8, V31.H8, V24.S4 // f8c3766e
+ VUMLAL V0.H4, V1.H4, V21.S4 // 3580602e
+ VUMLAL2 V0.H8, V1.H8, V22.S4 // 3680606e
+ VUMLSL V0.H4, V1.H4, V21.S4 // 35a0602e
+ VUMLSL2 V0.H8, V1.H8, V22.S4 // 36a0606e
+ VSMULL V0.B8, V1.B8, V5.H8 // 25c0200e
+ VSMULL2 V0.B16, V1.B16, V6.H8 // 26c0204e
+ VSMULL V0.S2, V1.S2, V7.D2 // 27c0a00e
+ VSMULL2 V0.S4, V1.S4, V8.D2 // 28c0a04e
+ VSMLAL V0.H4, V1.H4, V21.S4 // 3580600e
+ VSMLAL2 V0.H8, V1.H8, V22.S4 // 3680604e
+ VSMLSL V0.H4, V1.H4, V21.S4 // 35a0600e
+ VSMLSL2 V0.H8, V1.H8, V22.S4 // 36a0604e
VEOR3 V2.B16, V7.B16, V12.B16, V25.B16 // 990907ce
VBCAX V1.B16, V2.B16, V26.B16, V31.B16 // 5f0722ce
VREV32 V5.B16, V5.B16 // a508206e
VREV64 V2.S2, V3.S2 // 4308a00e
VREV64 V2.S4, V3.S4 // 4308a04e
+ VREV16 V7.B16, V5.B16 // e518204e
+ VREV16 V7.B8, V5.B8 // e518200e
+
+ VABS V0.B8, V1.B8 // 01b8200e
+ VABS V0.B16, V1.B16 // 01b8204e
+ VABS V0.H4, V1.H4 // 01b8600e
+ VABS V0.H8, V1.H8 // 01b8604e
+ VABS V0.S2, V1.S2 // 01b8a00e
+ VABS V0.S4, V1.S4 // 01b8a04e
+ VABS V0.D2, V1.D2 // 01b8e04e
+ VCLS V0.B8, V1.B8 // 0148200e
+ VCLS V0.B16, V1.B16 // 0148204e
+ VCLS V0.H4, V1.H4 // 0148600e
+ VCLS V0.H8, V1.H8 // 0148604e
+ VCLS V0.S2, V1.S2 // 0148a00e
+ VCLS V0.S4, V1.S4 // 0148a04e
+ VCLZ V0.B8, V1.B8 // 0148202e
+ VCLZ V0.B16, V1.B16 // 0148206e
+ VCLZ V0.H4, V1.H4 // 0148602e
+ VCLZ V0.H8, V1.H8 // 0148606e
+ VCLZ V0.S2, V1.S2 // 0148a02e
+ VCLZ V0.S4, V1.S4 // 0148a06e
+ VNEG V0.B8, V1.B8 // 01b8202e
+ VNEG V0.B16, V1.B16 // 01b8206e
+ VNEG V0.H4, V1.H4 // 01b8602e
+ VNEG V0.H8, V1.H8 // 01b8606e
+ VNEG V0.S2, V1.S2 // 01b8a02e
+ VNEG V0.S4, V1.S4 // 01b8a06e
+ VNEG V0.D2, V1.D2 // 01b8e06e
+ VFABS V0.S2, V1.S2 // 01f8a00e
+ VFABS V0.S4, V1.S4 // 01f8a04e
+ VFABS V0.D2, V1.D2 // 01f8e04e
+ VFNEG V0.S2, V1.S2 // 01f8a02e
+ VFNEG V0.S4, V1.S4 // 01f8a06e
+ VFNEG V0.D2, V1.D2 // 01f8e06e
+ VSQABS V0.B8, V1.B8 // 0178200e
+ VSQABS V0.B16, V1.B16 // 0178204e
+ VSQABS V0.H4, V1.H4 // 0178600e
+ VSQABS V0.H8, V1.H8 // 0178604e
+ VSQABS V0.S2, V1.S2 // 0178a00e
+ VSQABS V0.S4, V1.S4 // 0178a04e
+ VSQNEG V0.B8, V1.B8 // 0178202e
+ VSQNEG V0.B16, V1.B16 // 0178206e
+ VSQNEG V0.H4, V1.H4 // 0178602e
+ VSQNEG V0.H8, V1.H8 // 0178606e
+ VSQNEG V0.S2, V1.S2 // 0178a02e
+ VSQNEG V0.S4, V1.S4 // 0178a06e
+ VNOT V0.B8, V1.B8 // 0158202e
+ VNOT V0.B16, V1.B16 // 0158206e
// logical ops
//
@@ -400,6 +644,8 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
MOVD $0x11110000, R1 // MOVD $286326784, R1 // 2122a2d2
MOVD $0xaaaa0000aaaa1111, R1 // MOVD $-6149102338357718767, R1 // 212282d24155b5f24155f5f2
MOVD $0x1111ffff1111aaaa, R1 // MOVD $1230045644216969898, R1 // a1aa8a922122a2f22122e2f2
+ MOVD $0xaaaaaaaaaaaaaaab, R1 // MOVD $-6148914691236517205, R1 // e1f301b2615595f2
+ MOVD $0x0ff019940ff00ff0, R1 // MOVD $1148446028692721648, R1 // e19f0cb28132c3f2
MOVD $0, R1 // e1031faa
MOVD $-1, R1 // 01008092
MOVD $0x210000, R0 // MOVD $2162688, R0 // 2004a0d2
@@ -589,30 +835,30 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
MOVD R3, 4095(R17) // 3bfe3f91630300f9
// large aligned offset, use two instructions(add+ldr/str).
- MOVB R1, 0x1001(R2) // MOVB R1, 4097(R2) // 5b04409161070039
+ MOVB R1, 0x1001(R2) // MOVB R1, 4097(R2) // 5b08009161ff3f39
MOVB R1, 0xffffff(R2) // MOVB R1, 16777215(R2) // 5bfc7f9161ff3f39
- MOVH R1, 0x2002(R2) // MOVH R1, 8194(R2) // 5b08409161070079
+ MOVH R1, 0x2002(R2) // MOVH R1, 8194(R2) // 5b10009161ff3f79
MOVH R1, 0x1000ffe(R2) // MOVH R1, 16781310(R2) // 5bfc7f9161ff3f79
- MOVW R1, 0x4004(R2) // MOVW R1, 16388(R2) // 5b104091610700b9
+ MOVW R1, 0x4004(R2) // MOVW R1, 16388(R2) // 5b20009161ff3fb9
MOVW R1, 0x1002ffc(R2) // MOVW R1, 16789500(R2) // 5bfc7f9161ff3fb9
- MOVD R1, 0x8008(R2) // MOVD R1, 32776(R2) // 5b204091610700f9
+ MOVD R1, 0x8008(R2) // MOVD R1, 32776(R2) // 5b40009161ff3ff9
MOVD R1, 0x1006ff8(R2) // MOVD R1, 16805880(R2) // 5bfc7f9161ff3ff9
- FMOVS F1, 0x4004(R2) // FMOVS F1, 16388(R2) // 5b104091610700bd
+ FMOVS F1, 0x4004(R2) // FMOVS F1, 16388(R2) // 5b20009161ff3fbd
FMOVS F1, 0x1002ffc(R2) // FMOVS F1, 16789500(R2) // 5bfc7f9161ff3fbd
- FMOVD F1, 0x8008(R2) // FMOVD F1, 32776(R2) // 5b204091610700fd
+ FMOVD F1, 0x8008(R2) // FMOVD F1, 32776(R2) // 5b40009161ff3ffd
FMOVD F1, 0x1006ff8(R2) // FMOVD F1, 16805880(R2) // 5bfc7f9161ff3ffd
- MOVB 0x1001(R1), R2 // MOVB 4097(R1), R2 // 3b04409162078039
+ MOVB 0x1001(R1), R2 // MOVB 4097(R1), R2 // 3b08009162ffbf39
MOVB 0xffffff(R1), R2 // MOVB 16777215(R1), R2 // 3bfc7f9162ffbf39
- MOVH 0x2002(R1), R2 // MOVH 8194(R1), R2 // 3b08409162078079
+ MOVH 0x2002(R1), R2 // MOVH 8194(R1), R2 // 3b10009162ffbf79
MOVH 0x1000ffe(R1), R2 // MOVH 16781310(R1), R2 // 3bfc7f9162ffbf79
- MOVW 0x4004(R1), R2 // MOVW 16388(R1), R2 // 3b104091620780b9
+ MOVW 0x4004(R1), R2 // MOVW 16388(R1), R2 // 3b20009162ffbfb9
MOVW 0x1002ffc(R1), R2 // MOVW 16789500(R1), R2 // 3bfc7f9162ffbfb9
- MOVD 0x8008(R1), R2 // MOVD 32776(R1), R2 // 3b204091620740f9
+ MOVD 0x8008(R1), R2 // MOVD 32776(R1), R2 // 3b40009162ff7ff9
MOVD 0x1006ff8(R1), R2 // MOVD 16805880(R1), R2 // 3bfc7f9162ff7ff9
- FMOVS 0x4004(R1), F2 // FMOVS 16388(R1), F2 // 3b104091620740bd
+ FMOVS 0x4004(R1), F2 // FMOVS 16388(R1), F2 // 3b20009162ff7fbd
FMOVS 0x1002ffc(R1), F2 // FMOVS 16789500(R1), F2 // 3bfc7f9162ff7fbd
- FMOVD 0x8008(R1), F2 // FMOVD 32776(R1), F2 // 3b204091620740fd
+ FMOVD 0x8008(R1), F2 // FMOVD 32776(R1), F2 // 3b40009162ff7ffd
FMOVD 0x1006ff8(R1), F2 // FMOVD 16805880(R1), F2 // 3bfc7f9162ff7ffd
// very large or unaligned offset uses constant pool.
@@ -630,6 +876,8 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
FMOVS F1, 0x44332211(R2) // FMOVS F1, 1144201745(R2)
FMOVD F1, 0x1007000(R2) // FMOVD F1, 16805888(R2)
FMOVD F1, 0x44332211(R2) // FMOVD F1, 1144201745(R2)
+ FMOVQ F1, 0x1003000(R2) // FMOVQ F1, 16789504(R2)
+ FMOVQ F1, 0x44332211(R2) // FMOVQ F1, 1144201745(R2)
MOVB 0x1000000(R1), R2 // MOVB 16777216(R1), R2
MOVB 0x44332211(R1), R2 // MOVB 1144201745(R1), R2
@@ -643,6 +891,8 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
FMOVS 0x44332211(R1), F2 // FMOVS 1144201745(R1), F2
FMOVD 0x1000000(R1), F2 // FMOVD 16777216(R1), F2
FMOVD 0x44332211(R1), F2 // FMOVD 1144201745(R1), F2
+ FMOVQ 0x1000000(R1), F2 // FMOVQ 16777216(R1), F2
+ FMOVQ 0x44332211(R1), F2 // FMOVQ 1144201745(R1), F2
// shifted or extended register offset.
MOVD (R2)(R6.SXTW), R4 // 44c866f8
@@ -692,6 +942,12 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8
FMOVD F4, (R2)(R6) // 446826fc
FMOVD F4, (R2)(R6<<3) // 447826fc
+// global address
+ FMOVQ F5, x+8(SB)
+ FMOVQ F5, x(SB)
+ FMOVQ x+8(SB), F5
+ FMOVQ x(SB), F5
+
// vmov
VMOV V8.S[1], R1 // 013d0c0e
VMOV V0.D[0], R11 // 0b3c084e
@@ -1902,4 +2158,42 @@ next:
AUTIA1716 // 9f2103d5
AUTIB1716 // df2103d5
+// Vector Reduction Instructions
+ VSMAXV V0.B8, V0 // 00a8300e
+ VSMAXV V0.B16, V0 // 00a8304e
+ VSMAXV V0.H4, V0 // 00a8700e
+ VSMAXV V0.H8, V0 // 00a8704e
+ VSMAXV V0.S4, V0 // 00a8b04e
+ VSMINV V0.B8, V0 // 00a8310e
+ VSMINV V0.B16, V0 // 00a8314e
+ VSMINV V0.H4, V0 // 00a8710e
+ VSMINV V0.H8, V0 // 00a8714e
+ VSMINV V0.S4, V0 // 00a8b14e
+ VUMAXV V0.B8, V0 // 00a8302e
+ VUMAXV V0.B16, V0 // 00a8306e
+ VUMAXV V0.H4, V0 // 00a8702e
+ VUMAXV V0.H8, V0 // 00a8706e
+ VUMAXV V0.S4, V0 // 00a8b06e
+ VUMINV V0.B8, V0 // 00a8312e
+ VUMINV V0.B16, V0 // 00a8316e
+ VUMINV V0.H4, V0 // 00a8712e
+ VUMINV V0.H8, V0 // 00a8716e
+ VUMINV V0.S4, V0 // 00a8b16e
+ VFMAXV V0.S4, V0 // 00f8306e
+ VFMINV V0.S4, V0 // 00f8b06e
+ VFMAXNMV V0.S4, V0 // 00c8306e
+ VFMINNMV V0.S4, V0 // 00c8b06e
+ VFCVTZS V1.S4, V2.S4 // 22b8a14e
+ VFCVTZS V1.D2, V2.D2 // 22b8e14e
+ VFCVTZU V1.S4, V2.S4 // 22b8a16e
+ VFCVTZU V1.D2, V2.D2 // 22b8e16e
+ VSCVTF V1.S4, V2.S4 // 22d8214e
+ VSCVTF V1.D2, V2.D2 // 22d8614e
+ VUCVTF V1.S4, V2.S4 // 22d8216e
+ VUCVTF V1.D2, V2.D2 // 22d8616e
+ VFCVTN V1.D2, V2.S2 // 2268610e
+ VFCVTN2 V1.D2, V2.S4 // 2268614e
+ VFCVTL V1.S2, V2.D2 // 2278610e
+ VFCVTL2 V1.S4, V2.D2 // 2278614e
+
END
diff --git a/src/cmd/asm/internal/asm/testdata/arm64enc.s b/src/cmd/asm/internal/asm/testdata/arm64enc.s
index cc002a1584d2c7..76151a3570fe33 100644
--- a/src/cmd/asm/internal/asm/testdata/arm64enc.s
+++ b/src/cmd/asm/internal/asm/testdata/arm64enc.s
@@ -426,18 +426,18 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
YIELD // 3f2003d5
//TODO FABD F0, F5, F11 // abd4a07e
//TODO VFABD V30.S2, V8.S2, V24.S2 // 18d5be2e
- //TODO VFABS V5.S4, V24.S4 // b8f8a04e
+ VFABS V5.S4, V24.S4 // b8f8a04e
FABSS F2, F28 // 5cc0201e
FABSD F0, F14 // 0ec0601e
//TODO FACGE F25, F16, F0 // 00ee797e
//TODO VFACGE V11.S2, V15.S2, V9.S2 // e9ed2b2e
//TODO FACGT F20, F16, F27 // 1beef47e
//TODO VFACGT V15.S4, V25.S4, V22.S4 // 36efaf6e
- //TODO VFADD V21.D2, V10.D2, V21.D2 // 55d5754e
+ VFADD V21.D2, V10.D2, V21.D2 // 55d5754e
FADDS F12, F2, F10 // 4a282c1e
FADDD F24, F14, F12 // cc29781e
//TODO VFADDP V4.D2, F13 // 8dd8707e
- //TODO VFADDP V30.S4, V3.S4, V11.S4 // 6bd43e6e
+ VFADDP V30.S4, V3.S4, V11.S4 // 6bd43e6e
FCCMPS LE, F17, F12, $14 // 8ed5311e
FCCMPD HI, F11, F15, $15 // ef856b1e
FCCMPES HS, F28, F13, $13 // bd253c1e
@@ -547,34 +547,34 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
FCVTZUS F12, R29 // 9d01399e
FCVTZUDW F27, R22 // 7603791e
FCVTZUD F25, R22 // 3603799e
- //TODO VFDIV V6.D2, V1.D2, V27.D2 // 3bfc666e
+ VFDIV V6.D2, V1.D2, V27.D2 // 3bfc666e
FDIVS F16, F10, F20 // 5419301e
FDIVD F11, F25, F30 // 3e1b6b1e
FMADDS F15, F2, F8, F1 // 01090f1f
FMADDD F15, F21, F25, F9 // 29574f1f
- //TODO VFMAX V23.D2, V27.D2, V14.D2 // 6ef7774e
+ VFMAX V23.D2, V27.D2, V14.D2 // 6ef7774e
FMAXS F5, F28, F27 // 9b4b251e
FMAXD F12, F31, F31 // ff4b6c1e
- //TODO VFMAXNM V3.D2, V12.D2, V27.D2 // 9bc5634e
+ VFMAXNM V3.D2, V12.D2, V27.D2 // 9bc5634e
FMAXNMS F11, F24, F12 // 0c6b2b1e
FMAXNMD F20, F6, F16 // d068741e
//TODO VFMAXNMP V3.S2, F2 // 62c8307e
- //TODO VFMAXNMP V25.S2, V4.S2, V2.S2 // 82c4392e
+ VFMAXNMP V25.S2, V4.S2, V2.S2 // 82c4392e
//TODO VFMAXNMV V14.S4, F15 // cfc9306e
//TODO VFMAXP V3.S2, F27 // 7bf8307e
- //TODO VFMAXP V29.S2, V30.S2, V9.S2 // c9f73d2e
+ VFMAXP V29.S2, V30.S2, V9.S2 // c9f73d2e
//TODO VFMAXV V13.S4, F14 // aef9306e
- //TODO VFMIN V19.D2, V30.D2, V7.D2 // c7f7f34e
+ VFMIN V19.D2, V30.D2, V7.D2 // c7f7f34e
FMINS F26, F18, F30 // 5e5a3a1e
FMIND F29, F4, F21 // 95587d1e
- //TODO VFMINNM V21.S4, V5.S4, V1.S4 // a1c4b54e
+ VFMINNM V21.S4, V5.S4, V1.S4 // a1c4b54e
FMINNMS F23, F20, F1 // 817a371e
FMINNMD F8, F3, F24 // 7878681e
//TODO VFMINNMP V16.D2, F12 // 0ccaf07e
- //TODO VFMINNMP V10.S4, V25.S4, V27.S4 // 3bc7aa6e
+ VFMINNMP V10.S4, V25.S4, V27.S4 // 3bc7aa6e
//TODO VFMINNMV V8.S4, F3 // 03c9b06e
//TODO VFMINP V10.S2, F20 // 54f9b07e
- //TODO VFMINP V1.D2, V10.D2, V3.D2 // 43f5e16e
+ VFMINP V1.D2, V10.D2, V3.D2 // 43f5e16e
//TODO VFMINV V11.S4, F9 // 69f9b06e
//TODO VFMLA V6.S[0], F2, F14 // 4e10865f
//TODO VFMLA V28.S[2], V2.S2, V30.S2 // 5e189c0f
@@ -596,14 +596,14 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
FMSUBD F11, F7, F15, F31 // ff9d4b1f
//TODO VFMUL V9.S[2], F21, F19 // b39a895f
//TODO VFMUL V26.S[2], V26.S2, V2.S2 // 429b9a0f
- //TODO VFMUL V21.D2, V17.D2, V25.D2 // 39de756e
+ VFMUL V21.D2, V17.D2, V25.D2 // 39de756e
FMULS F0, F6, F24 // d808201e
FMULD F5, F29, F9 // a90b651e
//TODO VFMULX V26.S[2], F20, F8 // 889a9a7f
//TODO VFMULX V12.D[1], V21.D2, V31.D2 // bf9acc6f
//TODO FMULX F16, F1, F31 // 3fdc705e
//TODO VFMULX V29.S2, V13.S2, V31.S2 // bfdd3d0e
- //TODO VFNEG V18.S2, V12.S2 // 4cfaa02e
+ VFNEG V18.S2, V12.S2 // 4cfaa02e
FNEGS F16, F5 // 0542211e
FNEGD F31, F31 // ff43611e
FNMADDS F17, F22, F6, F20 // d458311f
@@ -623,26 +623,26 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
//TODO VFRINTI V21.D2, V31.D2 // bf9ae16e
FRINTIS F17, F17 // 31c2271e
FRINTID F9, F15 // 2fc1671e
- //TODO VFRINTM V9.D2, V27.D2 // 3b99614e
+ VFRINTM V9.D2, V27.D2 // 3b99614e
FRINTMS F24, F16 // 1043251e
FRINTMD F5, F2 // a240651e
- //TODO VFRINTN V30.S4, V2.S4 // c28b214e
+ VFRINTN V30.S4, V2.S4 // c28b214e
FRINTNS F26, F14 // 4e43241e
FRINTND F28, F12 // 8c43641e
- //TODO VFRINTP V27.D2, V31.D2 // 7f8be14e
+ VFRINTP V27.D2, V31.D2 // 7f8be14e
FRINTPS F27, F4 // 64c3241e
FRINTPD F6, F22 // d6c0641e
//TODO VFRINTX V25.D2, V0.D2 // 209b616e
FRINTXS F26, F10 // 4a43271e
FRINTXD F16, F12 // 0c42671e
- //TODO VFRINTZ V25.S4, V27.S4 // 3b9ba14e
+ VFRINTZ V25.S4, V27.S4 // 3b9ba14e
FRINTZS F3, F28 // 7cc0251e
FRINTZD F24, F6 // 06c3651e
//TODO FRSQRTE F29, F5 // a5dbe17e
//TODO VFRSQRTE V18.S2, V1.S2 // 41daa12e
//TODO FRSQRTS F17, F7, F24 // f8fcf15e
//TODO VFRSQRTS V14.S2, V10.S2, V24.S2 // 58fdae0e
- //TODO VFSQRT V2.D2, V21.D2 // 55f8e16e
+ VFSQRT V2.D2, V21.D2 // 55f8e16e
FSQRTS F0, F9 // 09c0211e
FSQRTD F14, F27 // dbc1611e
FSUBS F25, F23, F0 // e03a391e
@@ -762,5 +762,6 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
VMOV V0.B[1], R11 // 0b3c030e
VMOV V1.H[3], R12 // 2c3c0e0e
VUSHR $6, V22.H8, V23.H8 // d7061a6f
+ SB // ff3003d5
RET
diff --git a/src/cmd/asm/internal/asm/testdata/arm64error.s b/src/cmd/asm/internal/asm/testdata/arm64error.s
index ce88e3ca540f13..25dd523479d189 100644
--- a/src/cmd/asm/internal/asm/testdata/arm64error.s
+++ b/src/cmd/asm/internal/asm/testdata/arm64error.s
@@ -141,12 +141,27 @@ TEXT errors(SB),$0
VPMULL2 V1.H4, V2.H4, V3.Q1 // ERROR "operand mismatch"
VPMULL2 V1.D1, V2.D1, V3.Q1 // ERROR "operand mismatch"
VPMULL2 V1.B8, V2.B8, V3.H8 // ERROR "operand mismatch"
+ VPMULL2 V0.H8, V1.H8, V21.S4 // ERROR "invalid arrangement"
+ VUMULL V0.D1, V1.D1, V7.Q1 // ERROR "invalid arrangement"
+ VUMULL2 V0.B8, V1.B8, V21.H8 // ERROR "operand mismatch"
+ VUMLAL2 V0.D2, V1.D2, V7.Q1 // ERROR "invalid arrangement"
+ VUMLAL V0.H4, V1.H4, V21.H4 // ERROR "operand mismatch"
+ VUMLSL V0.H8, V1.H8, V21.H8 // ERROR "operand mismatch"
+ VSMULL2 V0.D2, V1.D2, V7.Q1 // ERROR "invalid arrangement"
+ VSMULL V0.B8, V1.B8, V5.B8 // ERROR "operand mismatch"
+ VSMLAL2 V0.D2, V1.D2, V7.Q1 // ERROR "invalid arrangement"
+ VSMLAL V0.S2, V1.S2, V21.S2 // ERROR "operand mismatch"
+ VSMLSL2 V0.B8, V1.B8, V6.H8 // ERROR "operand mismatch"
VEXT $8, V1.B16, V2.B8, V2.B16 // ERROR "invalid arrangement"
VEXT $8, V1.H8, V2.H8, V2.H8 // ERROR "invalid arrangement"
VRBIT V1.B16, V2.B8 // ERROR "invalid arrangement"
VRBIT V1.H4, V2.H4 // ERROR "invalid arrangement"
VUSHR $56, V1.D2, V2.H4 // ERROR "invalid arrangement"
VUSHR $127, V1.D2, V2.D2 // ERROR "shift out of range"
+ VSSHR $127, V1.D2, V2.D2 // ERROR "shift out of range"
+ VSSHR $56, V1.D2, V2.H4 // ERROR "invalid arrangement"
+ VSRSHR $127, V1.D2, V2.D2 // ERROR "shift out of range"
+ VSRSHR $56, V1.D2, V2.H4 // ERROR "invalid arrangement"
VLD1.P (R8)(R9.SXTX<<2), [V2.B16] // ERROR "invalid extended register"
VLD1.P (R8)(R9<<2), [V2.B16] // ERROR "invalid extended register"
VST1.P [V1.B16], (R8)(R9.UXTW) // ERROR "invalid extended register"
@@ -363,6 +378,8 @@ TEXT errors(SB),$0
VUXTL V30.D2, V30.H8 // ERROR "operand mismatch"
VUXTL2 V20.B8, V21.H8 // ERROR "operand mismatch"
VUXTL V3.D2, V4.B8 // ERROR "operand mismatch"
+ VSXTL V3.D2, V4.B8 // ERROR "operand mismatch"
+ VSXTL2 V20.B8, V21.H8 // ERROR "operand mismatch"
VUZP1 V0.B8, V30.B8, V1.B16 // ERROR "operand mismatch"
VUZP2 V0.Q1, V30.Q1, V1.Q1 // ERROR "invalid arrangement"
VUSHLL $0, V30.D2, V30.H8 // ERROR "operand mismatch"
@@ -377,8 +394,26 @@ TEXT errors(SB),$0
VUMIN V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
VUMAX V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
VUMIN V1.H4, V2.S4, V3.H4 // ERROR "operand mismatch"
+ VSMAX V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VSMIN V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VSMAX V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
+ VSMIN V1.H4, V2.S4, V3.H4 // ERROR "operand mismatch"
+ VSMAXP V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VSMINP V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VUMAXP V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VUMINP V1.D2, V2.D2, V3.D2 // ERROR "invalid arrangement"
+ VFMAX V1.B8, V2.B8, V3.B8 // ERROR "invalid arrangement"
+ VFMIN V1.B16, V2.B16, V3.B16 // ERROR "invalid arrangement"
+ VFADDP V1.B8, V2.B8, V3.B8 // ERROR "invalid arrangement"
+ VFADDP V1.B16, V2.B16, V3.B16 // ERROR "invalid arrangement"
+ VFADDP V1.H4, V2.H4, V3.H4 // ERROR "invalid arrangement"
+ VFADDP V1.H8, V2.H8, V3.H8 // ERROR "invalid arrangement"
VSLI $64, V7.D2, V8.D2 // ERROR "shift out of range"
VUSRA $0, V7.D2, V8.D2 // ERROR "shift out of range"
+ VSSHL V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
+ VUSHL V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
+ VSQSHL V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
+ VUQSHL V1.B8, V2.B8, V3.B16 // ERROR "operand mismatch"
CASPD (R3, R4), (R2), (R8, R9) // ERROR "source register pair must start from even register"
CASPD (R2, R3), (R2), (R9, R10) // ERROR "destination register pair must start from even register"
CASPD (R2, R4), (R2), (R8, R9) // ERROR "source register pair must be contiguous"
@@ -428,4 +463,66 @@ TEXT errors(SB),$0
AUTIBSP C // ERROR "illegal combination"
AUTIA1716 $45 // ERROR "illegal combination"
AUTIB1716 R0 // ERROR "illegal combination"
+ SB $1 // ERROR "illegal combination"
+ RPRFM (R1), RSP, PLDKEEP // ERROR "illegal combination"
+ RPRFM 2(RSP), R4, PSTSTRM // ERROR "illegal combination"
+ RPRFM (R2), R3, $100 // ERROR "range prefetch immediate must be 0 to 63"
+ RPRFM (R5), R6, PLDL1KEEP // ERROR "illegal range prefetch operand"
+
+ // VMUL family invalid arrangement tests
+ VMUL V0.D2, V0.D2, V1.D2 // ERROR "invalid arrangement"
+ VMLA V0.D2, V0.D2, V1.D2 // ERROR "invalid arrangement"
+ VMLS V0.D2, V0.D2, V1.D2 // ERROR "invalid arrangement"
+ // VMUL family operand mismatch tests
+ VMUL V0.S4, V0.D2, V1.S4 // ERROR "operand mismatch"
+ VMLA V0.H4, V0.S4, V1.H4 // ERROR "operand mismatch"
+ VMLS V0.B8, V0.H4, V1.B8 // ERROR "operand mismatch"
+ // VBIC/VORN family invalid arrangement tests
+ VBIC V0.S4, V0.S4, V1.S4 // ERROR "invalid arrangement"
+ VBIC V0.H4, V0.H4, V1.H4 // ERROR "invalid arrangement"
+ VBIC V0.D2, V0.D2, V1.D2 // ERROR "invalid arrangement"
+ VORN V0.S4, V0.S4, V1.S4 // ERROR "invalid arrangement"
+ VORN V0.H4, V0.H4, V1.H4 // ERROR "invalid arrangement"
+ VORN V0.D2, V0.D2, V1.D2 // ERROR "invalid arrangement"
+ VSMAXV V0.B8, V0.S4 // ERROR "illegal combination"
+ VUMAXV V0.S4, V0.H8 // ERROR "illegal combination"
+ VFMAXV V0.S4, V0.D2 // ERROR "illegal combination"
+ VFMAXV V0.D2, V0 // ERROR "invalid arrangement"
+ VFMINV V0.D2, V0 // ERROR "invalid arrangement"
+ VFMAXNMV V0.D2, V0 // ERROR "invalid arrangement"
+ VFMINNMV V0.D2, V0 // ERROR "invalid arrangement"
+ VFMAXV V0.S2, V0 // ERROR "invalid arrangement"
+ VFMINV V0.S2, V0 // ERROR "invalid arrangement"
+ VFMAXNMV V0.S2, V0 // ERROR "invalid arrangement"
+ VFMINNMV V0.S2, V0 // ERROR "invalid arrangement"
+ VFMAXV V0.H4, V0 // ERROR "invalid arrangement"
+ VFMAXV V0.H8, V0 // ERROR "invalid arrangement"
+ VFMINV V0.H4, V0 // ERROR "invalid arrangement"
+ VFMINV V0.H8, V0 // ERROR "invalid arrangement"
+ VFMAXNMV V0.H4, V0 // ERROR "invalid arrangement"
+ VFMAXNMV V0.H8, V0 // ERROR "invalid arrangement"
+ VFMINNMV V0.H4, V0 // ERROR "invalid arrangement"
+ VFMINNMV V0.H8, V0 // ERROR "invalid arrangement"
+
+ // Shifts and misc. conversion error test cases
+ VXTN V1.B8, V0.B8 // ERROR "invalid arrangement"
+ VXTN2 V1.H8, V0.B8 // ERROR "invalid arrangement"
+ VSHRN $8, V1.B8, V0.B8 // ERROR "invalid arrangement"
+ VSHRN $8, V1.S4, V0.S4 // ERROR "invalid arrangement"
+ VSHRN $8, V1.H8, V0.H8 // ERROR "invalid arrangement"
+ VSHRN2 $8, V1.B8, V0.B16 // ERROR "invalid arrangement"
+ VSHRN2 $8, V1.S4, V0.S4 // ERROR "invalid arrangement"
+ VSHRN2 $8, V1.H8, V0.H8 // ERROR "invalid arrangement"
+ VSSHLL $0, V1.D2, V2.H8 // ERROR "operand mismatch"
+ VSSHLL2 $0, V1.B8, V2.H8 // ERROR "operand mismatch"
+ VSSHLL $8, V1.B8, V2.H8 // ERROR "shift amount out of range"
+ VSQXTN V1.B8, V0.B8 // ERROR "invalid arrangement"
+ VSQXTN2 V1.H8, V0.B8 // ERROR "invalid arrangement"
+ VSQXTUN V1.B8, V0.B8 // ERROR "invalid arrangement"
+ VUQXTN V1.B8, V0.B8 // ERROR "invalid arrangement"
+ VFCVTZS V1.B8, V2.B8 // ERROR "invalid arrangement"
+ VFCVTZU V1.H4, V2.H4 // ERROR "invalid arrangement"
+ VSCVTF V1.B8, V2.B8 // ERROR "invalid arrangement"
+ VFCVTN V1.B8, V2.B8 // ERROR "invalid arrangement"
+ VFCVTL V1.H4, V2.S4 // ERROR "operand mismatch"
RET
diff --git a/src/cmd/asm/internal/asm/testdata/arm64sveenc.s b/src/cmd/asm/internal/asm/testdata/arm64sveenc.s
new file mode 100644
index 00000000000000..8fb275c59d57e9
--- /dev/null
+++ b/src/cmd/asm/internal/asm/testdata/arm64sveenc.s
@@ -0,0 +1,1401 @@
+// Code generated by 'instgen -o=$GOROOT # from go install golang.org/x/arch/arm64/instgen@latest'. DO NOT EDIT.
+
+#include "../../../../../runtime/textflag.h"
+
+TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
+ PAND P4.B, P2.B, P1.Z, P14.B // 4e440425
+ PANDS P4.B, P2.B, P1.Z, P14.B // 4e444425
+ PBIC P4.B, P2.B, P1.Z, P14.B // 5e440425
+ PBICS P4.B, P2.B, P1.Z, P14.B // 5e444425
+ PBRKA P5.B, P9.Z, P2.B // a2641025
+ PBRKAS P5.B, P9.Z, P4.B // a4645025
+ PBRKB P5.B, P9.Z, P2.B // a2649025
+ PBRKBS P5.B, P9.Z, P4.B // a464d025
+ PBRKN P4.B, P2.B, P1.Z, P4.B // 44441825
+ PBRKNS P4.B, P2.B, P1.Z, P4.B // 44445825
+ PBRKPA P4.B, P2.B, P1.Z, P14.B // 4ec40425
+ PBRKPAS P4.B, P2.B, P1.Z, P14.B // 4ec44425
+ PBRKPB P4.B, P2.B, P1.Z, P14.B // 5ec40425
+ PBRKPBS P4.B, P2.B, P1.Z, P14.B // 5ec44425
+ PEOR P4.B, P2.B, P1.Z, P14.B // 4e460425
+ PEORS P4.B, P2.B, P1.Z, P14.B // 4e464425
+ PNAND P4.B, P2.B, P1.Z, P14.B // 5e468425
+ PNANDS P4.B, P2.B, P1.Z, P14.B // 5e46c425
+ PNOR P4.B, P2.B, P1.Z, P14.B // 4e468425
+ PNORS P4.B, P2.B, P1.Z, P14.B // 4e46c425
+ PORN P4.B, P2.B, P1.Z, P14.B // 5e448425
+ PORNS P4.B, P2.B, P1.Z, P14.B // 5e44c425
+ PORR P4.B, P2.B, P1.Z, P14.B // 4e448425
+ PORRS P4.B, P2.B, P1.Z, P14.B // 4e44c425
+ PPFALSE P13.B // 0de41825
+ PPFIRST P5.B, P9, P5.B // 25c15825
+ PPNEXT P5.D, P4, P5.D // 85c4d925
+ PPTEST P14.B, P0 // c0c15025
+ PPTRUE PN14.S // 1678a025
+ PPUNPKHI P14.B, P0.H // c0413105
+ PPUNPKLO P14.B, P0.H // c0413005
+ PRDFFR P14.Z, P0.B // c0f11825
+ PRDFFR P13.B // 0df01925
+ PRDFFRS P14.Z, P0.B // c0f15825
+ PREV P14.S, P13.S // cd41b405
+ PSEL P4.B, P2.B, P1, P14.B // 5e460425
+ PTRN1 P5.D, P4.D, P2.D // 8250e505
+ PTRN2 P5.D, P4.D, P2.D // 8254e505
+ PUZP1 P5.D, P4.D, P2.D // 8248e505
+ PUZP2 P5.D, P4.D, P2.D // 824ce505
+ PWRFFR P13.B // a0912825
+ PZIP1 P5.D, P4.D, P2.D // 8240e505
+ PZIP2 P5.D, P4.D, P2.D // 8244e505
+ SETFFR // 00902c25
+ ZABS Z7.D, P4.M, Z13.D // edb0d604
+ ZABS Z7.D, P4.Z, Z13.D // edb0c604
+ ZADCLB Z7.D, Z23.D, Z13.D // edd24745
+ ZADCLT Z7.D, Z23.D, Z13.D // edd64745
+ ZADD Z15.B, Z0.B, P3.M, Z0.B // e00d0004
+ ZADD Z7.D, Z23.D, Z13.D // ed02e704
+ ZADDHNB Z22.S, Z10.S, Z8.H // 4861b645
+ ZADDHNT Z22.S, Z10.S, Z8.H // 4865b645
+ ZADDP Z15.B, Z0.B, P3.M, Z0.B // e0ad1144
+ ZADDPT Z23.D, Z13.D, P1.M, Z13.D // ed06c404
+ ZADDPT Z7.D, Z6.D, Z23.D // d708e704
+ // TODO: ADDQP
+ ZADDQV Z25.S, P3, V5.S4 // 252f8504
+ // TODO: ADDSUBP
+ ZAESD Z7.B, Z6.B, Z6.B // e6e42245
+ ZAESE Z7.B, Z6.B, Z6.B // e6e02245
+ ZAESIMC Z11.B, Z11.B // 0be42045
+ ZAESMC Z11.B, Z11.B // 0be02045
+ ZAND Z15.B, Z0.B, P3.M, Z0.B // e00d1a04
+ ZAND Z7.D, Z6.D, Z23.D // d7302704
+ ZANDQV Z25.S, P3, V5.S4 // 252f9e04
+ ZASR Z15.B, Z0.B, P3.M, Z0.B // e08d1004
+ ZASR Z2.D, Z10.D, P3.M, Z10.D // 4a8cd004
+ ZASR Z7.D, Z6.H, Z13.H // cd806704
+ ZASRR Z15.B, Z0.B, P3.M, Z0.B // e08d1404
+ ZBCAX Z23.D, Z13.D, Z21.D, Z21.D // f53a6d04
+ ZBDEP Z7.D, Z23.D, Z13.D // edb6c745
+ ZBEXT Z7.D, Z23.D, Z13.D // edb2c745
+ ZBF1CVT Z11.B, Z6.H // 66390865
+ ZBF1CVTLT Z11.B, Z6.H // 66390965
+ ZBF2CVT Z11.B, Z6.H // 663d0865
+ ZBF2CVTLT Z11.B, Z6.H // 663d0965
+ ZBFADD Z23.H, Z13.H, P1.M, Z13.H // ed860065
+ ZBFADD Z7.H, Z6.H, Z23.H // d7000765
+ ZBFCLAMP Z7.H, Z6.H, Z23.H // d7242764
+ ZBFCVT Z13.S, P1.M, Z22.H // b6a58a65
+ ZBFCVT Z13.S, P1.Z, Z22.H // b6c59a64
+ ZBFCVTNT Z13.S, P1.M, Z22.H // b6a58a64
+ ZBFCVTNT Z13.S, P1.Z, Z22.H // b6a58264
+ ZBFDOT Z7.H, Z6.H, Z23.S // d7806764
+ ZBFMAX Z23.H, Z13.H, P1.M, Z13.H // ed860665
+ ZBFMAXNM Z23.H, Z13.H, P1.M, Z13.H // ed860465
+ ZBFMIN Z23.H, Z13.H, P1.M, Z13.H // ed860765
+ ZBFMINNM Z23.H, Z13.H, P1.M, Z13.H // ed860565
+ ZBFMLA Z23.H, Z13.H, P1.M, Z22.H // b6053765
+ ZBFMLALB Z7.H, Z6.H, Z23.S // d780e764
+ ZBFMLALT Z7.H, Z6.H, Z23.S // d784e764
+ ZBFMLS Z23.H, Z13.H, P1.M, Z22.H // b6253765
+ ZBFMLSLB Z7.H, Z6.H, Z23.S // d7a0e764
+ ZBFMLSLT Z7.H, Z6.H, Z23.S // d7a4e764
+ // TODO: BFMMLA .H, .H, .H
+ ZBFMMLA Z7.H, Z6.H, Z23.S // d7e46764
+ ZBFMUL Z23.H, Z13.H, P1.M, Z13.H // ed860265
+ ZBFMUL Z7.H, Z6.H, Z23.H // d7080765
+ ZBFSCALE Z23.H, Z13.H, P1.M, Z13.H // ed860965
+ ZBFSUB Z23.H, Z13.H, P1.M, Z13.H // ed860165
+ ZBFSUB Z7.H, Z6.H, Z23.H // d7040765
+ ZBGRP Z7.D, Z23.D, Z13.D // edbac745
+ ZBIC Z15.B, Z0.B, P3.M, Z0.B // e00d1b04
+ ZBIC Z7.D, Z6.D, Z23.D // d730e704
+ ZBSL Z23.D, Z13.D, Z21.D, Z21.D // f53e2d04
+ ZBSL1N Z23.D, Z13.D, Z21.D, Z21.D // f53e6d04
+ ZBSL2N Z23.D, Z13.D, Z21.D, Z21.D // f53ead04
+ ZCLASTA Z15.B, Z0.B, P3, Z0.B // e08d2805
+ ZCLASTB Z15.B, Z0.B, P3, Z0.B // e08d2905
+ ZCLS Z7.D, P4.M, Z13.D // edb0d804
+ ZCLS Z7.D, P4.Z, Z13.D // edb0c804
+ ZCLZ Z7.D, P4.M, Z13.D // edb0d904
+ ZCLZ Z7.D, P4.Z, Z13.D // edb0c904
+ ZCMPEQ Z0.H, Z2.H, P0.Z, P14.H // 4ea04024
+ ZCMPEQ Z0.D, Z12.S, P0.Z, P14.S // 8e218024
+ ZCMPGE Z0.H, Z2.H, P0.Z, P14.H // 4e804024
+ ZCMPGE Z0.D, Z12.S, P0.Z, P14.S // 8e418024
+ ZCMPGT Z0.H, Z2.H, P0.Z, P14.H // 5e804024
+ ZCMPGT Z0.D, Z12.S, P0.Z, P14.S // 9e418024
+ ZCMPHI Z0.H, Z2.H, P0.Z, P14.H // 5e004024
+ ZCMPHI Z0.D, Z12.S, P0.Z, P14.S // 9ec18024
+ ZCMPHS Z0.H, Z2.H, P0.Z, P14.H // 4e004024
+ ZCMPHS Z0.D, Z12.S, P0.Z, P14.S // 8ec18024
+ // TODO: CMPLE
+ // TODO: CMPLO
+ // TODO: CMPLS
+ // TODO: CMPLT
+ ZCMPNE Z0.H, Z2.H, P0.Z, P14.H // 5ea04024
+ ZCMPNE Z0.D, Z12.S, P0.Z, P14.S // 9e218024
+ ZCNOT Z7.D, P4.M, Z13.D // edb0db04
+ ZCNOT Z7.D, P4.Z, Z13.D // edb0cb04
+ ZCNT Z7.D, P4.M, Z13.D // edb0da04
+ ZCNT Z7.D, P4.Z, Z13.D // edb0ca04
+ ZCOMPACT Z7.D, P4, Z13.D // ed90e105
+ ZCOMPACT Z7.D, P4, Z13.D // ed90e105
+ ZDECP P14.S, Z26.S // da81ad25
+ ZEOR Z15.B, Z0.B, P3.M, Z0.B // e00d1904
+ ZEOR Z7.D, Z6.D, Z23.D // d730a704
+ ZEOR3 Z23.D, Z13.D, Z21.D, Z21.D // f53a2d04
+ ZEORBT Z7.D, Z23.D, Z13.D // ed92c745
+ ZEORQV Z25.S, P3, V5.S4 // 252f9d04
+ ZEORTB Z7.D, Z23.D, Z13.D // ed96c745
+ ZEXPAND Z7.D, P4, Z13.D // ed90f105
+ ZF1CVT Z11.B, Z6.H // 66310865
+ ZF1CVTLT Z11.B, Z6.H // 66310965
+ ZF2CVT Z11.B, Z6.H // 66350865
+ ZF2CVTLT Z11.B, Z6.H // 66350965
+ ZFABD Z25.S, Z2.S, P1.M, Z2.S // 22878865
+ ZFABS Z7.D, P4.M, Z13.D // edb0dc04
+ ZFABS Z7.D, P4.Z, Z13.D // edb0cc04
+ ZFACGE Z0.H, Z2.H, P0.Z, P14.H // 5ec04065
+ ZFACGT Z0.H, Z2.H, P0.Z, P14.H // 5ee04065
+ ZFADD Z25.S, Z2.S, P1.M, Z2.S // 22878065
+ ZFADD Z7.D, Z23.D, Z13.D // ed02c765
+ ZFADDP Z25.S, Z2.S, P1.M, Z2.S // 22879064
+ ZFADDQV Z25.S, P3, V5.S4 // 25af9064
+ ZFAMAX Z25.S, Z2.S, P1.M, Z2.S // 22878e65
+ ZFAMIN Z25.S, Z2.S, P1.M, Z2.S // 22878f65
+ ZFCLAMP Z7.D, Z23.D, Z13.D // ed26e764
+ ZFCMEQ Z0.H, Z2.H, P0.Z, P14.H // 4e604065
+ ZFCMGE Z0.H, Z2.H, P0.Z, P14.H // 4e404065
+ ZFCMGT Z0.H, Z2.H, P0.Z, P14.H // 5e404065
+ ZFCMNE Z0.H, Z2.H, P0.Z, P14.H // 5e604065
+ ZFCMUO Z0.H, Z2.H, P0.Z, P14.H // 4ec04065
+ ZFCVT Z13.H, P1.M, Z22.S // b6a58965
+ ZFCVT Z13.H, P1.Z, Z22.S // b6a59a64
+ ZFCVT Z13.H, P1.M, Z22.D // b6a5c965
+ ZFCVT Z13.H, P1.Z, Z22.D // b6a5da64
+ ZFCVT Z13.S, P1.M, Z22.H // b6a58865
+ ZFCVT Z13.S, P1.Z, Z22.H // b6859a64
+ ZFCVT Z13.S, P1.M, Z22.D // b6a5cb65
+ ZFCVT Z13.S, P1.Z, Z22.D // b6e5da64
+ ZFCVT Z13.D, P1.M, Z22.H // b6a5c865
+ ZFCVT Z13.D, P1.Z, Z22.H // b685da64
+ ZFCVT Z13.D, P1.M, Z22.S // b6a5ca65
+ ZFCVT Z13.D, P1.Z, Z22.S // b6c5da64
+ ZFCVTLT Z13.H, P1.M, Z22.S // b6a58964
+ ZFCVTLT Z13.H, P1.Z, Z22.S // b6a58164
+ ZFCVTLT Z13.S, P1.M, Z22.D // b6a5cb64
+ ZFCVTLT Z13.S, P1.Z, Z22.D // b6a5c364
+ ZFCVTNT Z13.S, P1.M, Z22.H // b6a58864
+ ZFCVTNT Z13.S, P1.Z, Z22.H // b6a58064
+ ZFCVTNT Z13.D, P1.M, Z22.S // b6a5ca64
+ ZFCVTNT Z13.D, P1.Z, Z22.S // b6a5c264
+ ZFCVTX Z13.D, P1.M, Z22.S // b6a50a65
+ ZFCVTX Z13.D, P1.Z, Z22.S // b6c51a64
+ ZFCVTXNT Z13.D, P1.M, Z22.S // b6a50a64
+ ZFCVTXNT Z13.D, P1.Z, Z22.S // b6a50264
+ ZFCVTZS Z13.H, P1.M, Z22.H // b6a55a65
+ ZFCVTZS Z13.H, P1.Z, Z22.H // b6c55e64
+ ZFCVTZS Z13.H, P1.M, Z22.S // b6a55c65
+ ZFCVTZS Z13.H, P1.Z, Z22.S // b6855f64
+ ZFCVTZS Z13.H, P1.M, Z22.D // b6a55e65
+ ZFCVTZS Z13.H, P1.Z, Z22.D // b6c55f64
+ ZFCVTZS Z13.S, P1.M, Z22.S // b6a59c65
+ ZFCVTZS Z13.S, P1.Z, Z22.S // b6859f64
+ ZFCVTZS Z13.S, P1.M, Z22.D // b6a5dc65
+ ZFCVTZS Z13.S, P1.Z, Z22.D // b685df64
+ ZFCVTZS Z13.D, P1.M, Z22.S // b6a5d865
+ ZFCVTZS Z13.D, P1.Z, Z22.S // b685de64
+ ZFCVTZS Z13.D, P1.M, Z22.D // b6a5de65
+ ZFCVTZS Z13.D, P1.Z, Z22.D // b6c5df64
+ ZFCVTZU Z13.H, P1.M, Z22.H // b6a55b65
+ ZFCVTZU Z13.H, P1.Z, Z22.H // b6e55e64
+ ZFCVTZU Z13.H, P1.M, Z22.S // b6a55d65
+ ZFCVTZU Z13.H, P1.Z, Z22.S // b6a55f64
+ ZFCVTZU Z13.H, P1.M, Z22.D // b6a55f65
+ ZFCVTZU Z13.H, P1.Z, Z22.D // b6e55f64
+ ZFCVTZU Z13.S, P1.M, Z22.S // b6a59d65
+ ZFCVTZU Z13.S, P1.Z, Z22.S // b6a59f64
+ ZFCVTZU Z13.S, P1.M, Z22.D // b6a5dd65
+ ZFCVTZU Z13.S, P1.Z, Z22.D // b6a5df64
+ ZFCVTZU Z13.D, P1.M, Z22.S // b6a5d965
+ ZFCVTZU Z13.D, P1.Z, Z22.S // b6a5de64
+ ZFCVTZU Z13.D, P1.M, Z22.D // b6a5df65
+ ZFCVTZU Z13.D, P1.Z, Z22.D // b6e5df64
+ ZFDIV Z25.S, Z2.S, P1.M, Z2.S // 22878d65
+ ZFDIVR Z25.S, Z2.S, P1.M, Z2.S // 22878c65
+ ZFDOT Z7.H, Z6.H, Z23.S // d7802764
+ ZFDOT Z7.B, Z6.B, Z23.H // d7842764
+ ZFDOT Z7.B, Z6.B, Z23.S // d7846764
+ ZFEXPA Z1.S, Z26.S // 3ab8a004
+ ZFLOGB Z7.D, P4.M, Z13.D // edb01e65
+ ZFLOGB Z7.D, P4.Z, Z13.D // edf01e64
+ ZFMAD Z0.H, Z2.H, P0.M, Z14.H // 4e806065
+ ZFMAX Z25.S, Z2.S, P1.M, Z2.S // 22878665
+ ZFMAXNM Z25.S, Z2.S, P1.M, Z2.S // 22878465
+ ZFMAXNMP Z25.S, Z2.S, P1.M, Z2.S // 22879464
+ ZFMAXNMQV Z25.S, P3, V5.S4 // 25af9464
+ ZFMAXP Z25.S, Z2.S, P1.M, Z2.S // 22879664
+ ZFMAXQV Z25.S, P3, V5.S4 // 25af9664
+ ZFMIN Z25.S, Z2.S, P1.M, Z2.S // 22878765
+ ZFMINNM Z25.S, Z2.S, P1.M, Z2.S // 22878565
+ ZFMINNMP Z25.S, Z2.S, P1.M, Z2.S // 22879564
+ ZFMINNMQV Z25.S, P3, V5.S4 // 25af9564
+ ZFMINP Z25.S, Z2.S, P1.M, Z2.S // 22879764
+ ZFMINQV Z25.S, P3, V5.S4 // 25af9764
+ ZFMLA Z0.H, Z2.H, P0.M, Z14.H // 4e006065
+ ZFMLALB Z7.H, Z6.H, Z23.S // d780a764
+ ZFMLALB Z7.B, Z6.B, Z23.H // d788a764
+ ZFMLALLBB Z7.B, Z6.B, Z23.S // d7882764
+ ZFMLALLBT Z7.B, Z6.B, Z23.S // d7982764
+ ZFMLALLTB Z7.B, Z6.B, Z23.S // d7a82764
+ ZFMLALLTT Z7.B, Z6.B, Z23.S // d7b82764
+ ZFMLALT Z7.H, Z6.H, Z23.S // d784a764
+ ZFMLALT Z7.B, Z6.B, Z23.H // d798a764
+ ZFMLS Z0.H, Z2.H, P0.M, Z14.H // 4e206065
+ ZFMLSLB Z7.H, Z6.H, Z23.S // d7a0a764
+ ZFMLSLT Z7.H, Z6.H, Z23.S // d7a4a764
+ // TODO: FMMLA .H, .H, .H
+ ZFMMLA Z7.S, Z6.S, Z23.S // d7e4a764
+ ZFMMLA Z7.D, Z6.D, Z23.D // d7e4e764
+ ZFMMLA Z7.H, Z6.H, Z23.S // d7e42764
+ ZFMMLA Z7.B, Z6.B, Z23.H // d7e06764
+ ZFMMLA Z7.B, Z6.B, Z23.S // d7e02764
+ ZFMSB Z0.H, Z2.H, P0.M, Z14.H // 4ea06065
+ ZFMUL Z25.S, Z2.S, P1.M, Z2.S // 22878265
+ ZFMUL Z7.D, Z23.D, Z13.D // ed0ac765
+ ZFMULX Z25.S, Z2.S, P1.M, Z2.S // 22878a65
+ ZFNEG Z7.D, P4.M, Z13.D // edb0dd04
+ ZFNEG Z7.D, P4.Z, Z13.D // edb0cd04
+ ZFNMAD Z0.H, Z2.H, P0.M, Z14.H // 4ec06065
+ ZFNMLA Z0.H, Z2.H, P0.M, Z14.H // 4e406065
+ ZFNMLS Z0.H, Z2.H, P0.M, Z14.H // 4e606065
+ ZFNMSB Z0.H, Z2.H, P0.M, Z14.H // 4ee06065
+ ZFRECPE Z1.S, Z26.S // 3a308e65
+ ZFRECPS Z7.D, Z23.D, Z13.D // ed1ac765
+ ZFRECPX Z7.D, P4.M, Z13.D // edb0cc65
+ ZFRECPX Z7.D, P4.Z, Z13.D // ed90db64
+ ZFRINT32X Z7.D, P4.M, Z13.D // edb01365
+ ZFRINT32X Z7.D, P4.Z, Z13.D // edf01c64
+ ZFRINT32Z Z7.D, P4.M, Z13.D // edb01265
+ ZFRINT32Z Z7.D, P4.Z, Z13.D // edd01c64
+ ZFRINT64X Z7.D, P4.M, Z13.D // edb01765
+ ZFRINT64X Z7.D, P4.Z, Z13.D // edf01d64
+ ZFRINT64Z Z7.D, P4.M, Z13.D // edb01665
+ ZFRINT64Z Z7.D, P4.Z, Z13.D // edd01d64
+ ZFRINTA Z7.D, P4.M, Z13.D // edb0c465
+ ZFRINTA Z7.D, P4.Z, Z13.D // ed90d964
+ ZFRINTI Z7.D, P4.M, Z13.D // edb0c765
+ ZFRINTI Z7.D, P4.Z, Z13.D // edf0d964
+ ZFRINTM Z7.D, P4.M, Z13.D // edb0c265
+ ZFRINTM Z7.D, P4.Z, Z13.D // edd0d864
+ ZFRINTN Z7.D, P4.M, Z13.D // edb0c065
+ ZFRINTN Z7.D, P4.Z, Z13.D // ed90d864
+ ZFRINTP Z7.D, P4.M, Z13.D // edb0c165
+ ZFRINTP Z7.D, P4.Z, Z13.D // edb0d864
+ ZFRINTX Z7.D, P4.M, Z13.D // edb0c665
+ ZFRINTX Z7.D, P4.Z, Z13.D // edd0d964
+ ZFRINTZ Z7.D, P4.M, Z13.D // edb0c365
+ ZFRINTZ Z7.D, P4.Z, Z13.D // edf0d864
+ ZFRSQRTE Z1.S, Z26.S // 3a308f65
+ ZFRSQRTS Z7.D, Z23.D, Z13.D // ed1ec765
+ ZFSCALE Z25.S, Z2.S, P1.M, Z2.S // 22878965
+ ZFSQRT Z7.D, P4.M, Z13.D // edb0cd65
+ ZFSQRT Z7.D, P4.Z, Z13.D // edb0db64
+ ZFSUB Z25.S, Z2.S, P1.M, Z2.S // 22878165
+ ZFSUB Z7.D, Z23.D, Z13.D // ed06c765
+ ZFSUBR Z25.S, Z2.S, P1.M, Z2.S // 22878365
+ ZFTSMUL Z7.D, Z23.D, Z13.D // ed0ec765
+ ZFTSSEL Z7.D, Z23.D, Z13.D // edb2e704
+ ZHISTCNT Z19.S, Z20.S, P1.Z, Z2.S // 82c6b345
+ ZHISTSEG Z7.B, Z6.B, Z23.B // d7a02745
+ ZINCP P14.S, Z26.S // da81ac25
+ ZLSL Z15.B, Z0.B, P3.M, Z0.B // e08d1304
+ ZLSL Z2.D, Z10.D, P3.M, Z10.D // 4a8cd304
+ ZLSL Z7.D, Z6.H, Z13.H // cd8c6704
+ ZLSLR Z15.B, Z0.B, P3.M, Z0.B // e08d1704
+ ZLSR Z15.B, Z0.B, P3.M, Z0.B // e08d1104
+ ZLSR Z2.D, Z10.D, P3.M, Z10.D // 4a8cd104
+ ZLSR Z7.D, Z6.H, Z13.H // cd846704
+ ZLSRR Z15.B, Z0.B, P3.M, Z0.B // e08d1504
+ ZMAD Z0.H, Z2.H, P0.M, Z14.H // 0ec04204
+ ZMADPT Z7.D, Z6.D, Z23.D // f7d8c644
+ ZMATCH Z0.H, Z2.H, P0.Z, P14.H // 4e806045
+ ZMLA Z0.H, Z2.H, P0.M, Z14.H // 4e404004
+ ZMLAPT Z7.D, Z6.D, Z23.D // d7d0c744
+ ZMLS Z0.H, Z2.H, P0.M, Z14.H // 4e604004
+ ZMOVPRFX Z7.D, P4.Z, Z21.D // f530d004
+ ZMOVPRFX Z11, Z6 // 66bd2004
+ ZMSB Z0.H, Z2.H, P0.M, Z14.H // 0ee04204
+ ZMUL Z15.B, Z0.B, P3.M, Z0.B // e00d1004
+ ZMUL Z7.D, Z23.D, Z13.D // ed62e704
+ ZNBSL Z23.D, Z13.D, Z21.D, Z21.D // f53eed04
+ ZNEG Z7.D, P4.M, Z13.D // edb0d704
+ ZNEG Z7.D, P4.Z, Z13.D // edb0c704
+ ZNMATCH Z0.H, Z2.H, P0.Z, P14.H // 5e806045
+ ZNOT Z7.D, P4.M, Z13.D // edb0de04
+ ZNOT Z7.D, P4.Z, Z13.D // edb0ce04
+ ZORQV Z25.S, P3, V5.S4 // 252f9c04
+ ZORR Z15.B, Z0.B, P3.M, Z0.B // e00d1804
+ ZORR Z7.D, Z6.D, Z23.D // d7306704
+ ZPMOV Z11, P0.B // 60392a05
+ ZPMOV P14.B, Z6 // c6392b05
+ ZPMUL Z7.B, Z6.B, Z23.B // d7642704
+ ZPMULLB Z8.B, Z4.B, Z30.H // 9e684845
+ ZPMULLB Z7.D, Z6.D, Z23.Q // d7680745
+ ZPMULLT Z8.B, Z4.B, Z30.H // 9e6c4845
+ ZPMULLT Z7.D, Z6.D, Z23.Q // d76c0745
+ ZRADDHNB Z22.S, Z10.S, Z8.H // 4869b645
+ ZRADDHNT Z22.S, Z10.S, Z8.H // 486db645
+ ZRAX1 Z7.D, Z6.D, Z23.D // d7f42745
+ ZRBIT Z7.D, P4.M, Z13.D // ed90e705
+ ZRBIT Z7.D, P4.Z, Z13.D // edb0e705
+ ZREV Z1.S, Z26.S // 3a38b805
+ ZREVB Z7.D, P4.M, Z13.D // ed90e405
+ ZREVB Z7.D, P4.Z, Z13.D // edb0e405
+ ZREVD Z13.Q, P1.M, Z22.Q // b6852e05
+ ZREVD Z13.Q, P1.Z, Z22.Q // b6a52e05
+ ZREVH Z7.D, P4.M, Z13.D // ed90e505
+ ZREVH Z7.D, P4.Z, Z13.D // edb0e505
+ ZREVW Z13.D, P1.M, Z22.D // b685e605
+ ZREVW Z13.D, P1.Z, Z22.D // b6a5e605
+ ZRSUBHNB Z22.S, Z10.S, Z8.H // 4879b645
+ ZRSUBHNT Z22.S, Z10.S, Z8.H // 487db645
+ ZSABA Z7.D, Z23.D, Z13.D // edfac745
+ // TODO: SABAL ., ., .
+ ZSABALB Z8.B, Z4.B, Z30.H // 9ec04845
+ ZSABALT Z8.B, Z4.B, Z30.H // 9ec44845
+ ZSABD Z15.B, Z0.B, P3.M, Z0.B // e00d0c04
+ ZSABDLB Z8.B, Z4.B, Z30.H // 9e304845
+ ZSABDLT Z8.B, Z4.B, Z30.H // 9e344845
+ ZSADALP Z14.S, P1.M, Z1.D // c1a5c444
+ ZSADDLB Z8.B, Z4.B, Z30.H // 9e004845
+ ZSADDLBT Z8.B, Z4.B, Z30.H // 9e804845
+ ZSADDLT Z8.B, Z4.B, Z30.H // 9e044845
+ ZSADDWB Z22.S, Z10.D, Z5.D // 4541d645
+ ZSADDWT Z22.S, Z10.D, Z5.D // 4545d645
+ ZSBCLB Z7.D, Z23.D, Z13.D // edd2c745
+ ZSBCLT Z7.D, Z23.D, Z13.D // edd6c745
+ ZSCLAMP Z7.D, Z23.D, Z13.D // edc2c744
+ // TODO: SCVTF ., .
+ ZSCVTF Z13.H, P1.M, Z22.H // b6a55265
+ ZSCVTF Z13.H, P1.Z, Z22.H // b6c55c64
+ ZSCVTF Z13.S, P1.M, Z22.H // b6a55465
+ ZSCVTF Z13.S, P1.Z, Z22.H // b6855d64
+ ZSCVTF Z13.S, P1.M, Z22.S // b6a59465
+ ZSCVTF Z13.S, P1.Z, Z22.S // b6859d64
+ ZSCVTF Z13.S, P1.M, Z22.D // b6a5d065
+ ZSCVTF Z13.S, P1.Z, Z22.D // b685dc64
+ ZSCVTF Z13.D, P1.M, Z22.H // b6a55665
+ ZSCVTF Z13.D, P1.Z, Z22.H // b6c55d64
+ ZSCVTF Z13.D, P1.M, Z22.S // b6a5d465
+ ZSCVTF Z13.D, P1.Z, Z22.S // b685dd64
+ ZSCVTF Z13.D, P1.M, Z22.D // b6a5d665
+ ZSCVTF Z13.D, P1.Z, Z22.D // b6c5dd64
+ // TODO: SCVTFLT
+ ZSDIV Z25.S, Z2.S, P1.M, Z2.S // 22079404
+ ZSDIVR Z25.S, Z2.S, P1.M, Z2.S // 22079604
+ // TODO: SDOT .H, .B, .B
+ ZSDOT Z7.H, Z6.H, Z23.S // d7c80744
+ ZSDOT Z15.B, Z0.B, Z12.S // 0c008f44
+ ZSEL Z23.B, Z21.B, P14, Z2.B // a2fa3705
+ ZSHADD Z15.B, Z0.B, P3.M, Z0.B // e08d1044
+ ZSHSUB Z15.B, Z0.B, P3.M, Z0.B // e08d1244
+ ZSHSUBR Z15.B, Z0.B, P3.M, Z0.B // e08d1644
+ ZSM4E Z7.S, Z6.S, Z6.S // e6e02345
+ ZSM4EKEY Z7.S, Z6.S, Z23.S // d7f02745
+ ZSMAX Z15.B, Z0.B, P3.M, Z0.B // e00d0804
+ ZSMAXP Z15.B, Z0.B, P3.M, Z0.B // e0ad1444
+ ZSMAXQV Z25.S, P3, V5.S4 // 252f8c04
+ ZSMIN Z15.B, Z0.B, P3.M, Z0.B // e00d0a04
+ ZSMINP Z15.B, Z0.B, P3.M, Z0.B // e0ad1644
+ ZSMINQV Z25.S, P3, V5.S4 // 252f8e04
+ ZSMLALB Z8.B, Z4.B, Z30.H // 9e404844
+ ZSMLALT Z8.B, Z4.B, Z30.H // 9e444844
+ ZSMLSLB Z8.B, Z4.B, Z30.H // 9e504844
+ ZSMLSLT Z8.B, Z4.B, Z30.H // 9e544844
+ ZSMMLA Z7.B, Z6.B, Z23.S // d7980745
+ ZSMULH Z15.B, Z0.B, P3.M, Z0.B // e00d1204
+ ZSMULH Z7.D, Z23.D, Z13.D // ed6ae704
+ ZSMULLB Z8.B, Z4.B, Z30.H // 9e704845
+ ZSMULLT Z8.B, Z4.B, Z30.H // 9e744845
+ ZSPLICE Z15.B, Z0.B, P3, Z0.B // e08d2c05
+ ZSQABS Z7.D, P4.M, Z13.D // edb0c844
+ ZSQABS Z7.D, P4.Z, Z13.D // edb0ca44
+ ZSQADD Z15.B, Z0.B, P3.M, Z0.B // e08d1844
+ ZSQADD Z7.D, Z23.D, Z13.D // ed12e704
+ ZSQDECP P14.S, Z26.S // da81aa25
+ ZSQDMLALB Z8.B, Z4.B, Z30.H // 9e604844
+ ZSQDMLALBT Z8.B, Z4.B, Z30.H // 9e084844
+ ZSQDMLALT Z8.B, Z4.B, Z30.H // 9e644844
+ ZSQDMLSLB Z8.B, Z4.B, Z30.H // 9e684844
+ ZSQDMLSLBT Z8.B, Z4.B, Z30.H // 9e0c4844
+ ZSQDMLSLT Z8.B, Z4.B, Z30.H // 9e6c4844
+ ZSQDMULH Z7.D, Z23.D, Z13.D // ed72e704
+ ZSQDMULLB Z8.B, Z4.B, Z30.H // 9e604845
+ ZSQDMULLT Z8.B, Z4.B, Z30.H // 9e644845
+ ZSQINCP P14.S, Z26.S // da81a825
+ ZSQNEG Z7.D, P4.M, Z13.D // edb0c944
+ ZSQNEG Z7.D, P4.Z, Z13.D // edb0cb44
+ ZSQRDMLAH Z7.D, Z23.D, Z13.D // ed72c744
+ ZSQRDMLSH Z7.D, Z23.D, Z13.D // ed76c744
+ ZSQRDMULH Z7.D, Z23.D, Z13.D // ed76e704
+ ZSQRSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0a44
+ ZSQRSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0e44
+ ZSQSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0844
+ ZSQSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0c44
+ ZSQSUB Z15.B, Z0.B, P3.M, Z0.B // e08d1a44
+ ZSQSUB Z7.D, Z23.D, Z13.D // ed1ae704
+ ZSQSUBR Z15.B, Z0.B, P3.M, Z0.B // e08d1e44
+ ZSQXTNB Z30.D, Z29.S // dd436045
+ ZSQXTNT Z30.D, Z29.S // dd476045
+ ZSQXTUNB Z30.D, Z29.S // dd536045
+ ZSQXTUNT Z30.D, Z29.S // dd576045
+ ZSRHADD Z15.B, Z0.B, P3.M, Z0.B // e08d1444
+ ZSRSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0244
+ ZSRSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0644
+ ZSSUBLB Z8.B, Z4.B, Z30.H // 9e104845
+ ZSSUBLBT Z8.B, Z4.B, Z30.H // 9e884845
+ ZSSUBLT Z8.B, Z4.B, Z30.H // 9e144845
+ ZSSUBLTB Z8.B, Z4.B, Z30.H // 9e8c4845
+ ZSSUBWB Z22.S, Z10.D, Z5.D // 4551d645
+ ZSSUBWT Z22.S, Z10.D, Z5.D // 4555d645
+ ZSUB Z15.B, Z0.B, P3.M, Z0.B // e00d0104
+ ZSUB Z7.D, Z23.D, Z13.D // ed06e704
+ ZSUBHNB Z22.S, Z10.S, Z8.H // 4871b645
+ ZSUBHNT Z22.S, Z10.S, Z8.H // 4875b645
+ // TODO: SUBP ., /M, ., .
+ ZSUBPT Z23.D, Z13.D, P1.M, Z13.D // ed06c504
+ ZSUBPT Z7.D, Z6.D, Z23.D // d70ce704
+ ZSUBR Z15.B, Z0.B, P3.M, Z0.B // e00d0304
+ ZSUNPKHI Z15.B, Z0.H // e0397105
+ ZSUNPKLO Z15.B, Z0.H // e0397005
+ ZSUQADD Z15.B, Z0.B, P3.M, Z0.B // e08d1c44
+ ZSXTB Z7.D, P4.M, Z13.D // edb0d004
+ ZSXTB Z7.D, P4.Z, Z13.D // edb0c004
+ ZSXTH Z7.D, P4.M, Z13.D // edb0d204
+ ZSXTH Z7.D, P4.Z, Z13.D // edb0c204
+ ZSXTW Z13.D, P1.M, Z22.D // b6a5d404
+ ZSXTW Z13.D, P1.Z, Z22.D // b6a5c404
+ ZTBX Z7.D, Z23.D, Z13.D // ed2ee705
+ ZTBXQ Z7.D, Z23.D, Z13.D // ed36e705
+ ZTRN1 Z7.D, Z23.D, Z13.D // ed72e705
+ ZTRN1 Z7.Q, Z6.Q, Z23.Q // d718a705
+ ZTRN2 Z7.D, Z23.D, Z13.D // ed76e705
+ ZTRN2 Z7.Q, Z6.Q, Z23.Q // d71ca705
+ ZUABA Z7.D, Z23.D, Z13.D // edfec745
+ // TODO: UABAL ., ., .
+ ZUABALB Z8.B, Z4.B, Z30.H // 9ec84845
+ ZUABALT Z8.B, Z4.B, Z30.H // 9ecc4845
+ ZUABD Z15.B, Z0.B, P3.M, Z0.B // e00d0d04
+ ZUABDLB Z8.B, Z4.B, Z30.H // 9e384845
+ ZUABDLT Z8.B, Z4.B, Z30.H // 9e3c4845
+ ZUADALP Z14.S, P1.M, Z1.D // c1a5c544
+ ZUADDLB Z8.B, Z4.B, Z30.H // 9e084845
+ ZUADDLT Z8.B, Z4.B, Z30.H // 9e0c4845
+ ZUADDWB Z22.S, Z10.D, Z5.D // 4549d645
+ ZUADDWT Z22.S, Z10.D, Z5.D // 454dd645
+ ZUCLAMP Z7.D, Z23.D, Z13.D // edc6c744
+ // TODO: UCVTF ., .
+ ZUCVTF Z13.H, P1.M, Z22.H // b6a55365
+ ZUCVTF Z13.H, P1.Z, Z22.H // b6e55c64
+ ZUCVTF Z13.S, P1.M, Z22.H // b6a55565
+ ZUCVTF Z13.S, P1.Z, Z22.H // b6a55d64
+ ZUCVTF Z13.S, P1.M, Z22.S // b6a59565
+ ZUCVTF Z13.S, P1.Z, Z22.S // b6a59d64
+ ZUCVTF Z13.S, P1.M, Z22.D // b6a5d165
+ ZUCVTF Z13.S, P1.Z, Z22.D // b6a5dc64
+ ZUCVTF Z13.D, P1.M, Z22.H // b6a55765
+ ZUCVTF Z13.D, P1.Z, Z22.H // b6e55d64
+ ZUCVTF Z13.D, P1.M, Z22.S // b6a5d565
+ ZUCVTF Z13.D, P1.Z, Z22.S // b6a5dd64
+ ZUCVTF Z13.D, P1.M, Z22.D // b6a5d765
+ ZUCVTF Z13.D, P1.Z, Z22.D // b6e5dd64
+ // TODO: UCVTFLT
+ ZUDIV Z25.S, Z2.S, P1.M, Z2.S // 22079504
+ ZUDIVR Z25.S, Z2.S, P1.M, Z2.S // 22079704
+ // TODO: UDOT .H, .B, .B
+ ZUDOT Z7.H, Z6.H, Z23.S // d7cc0744
+ ZUDOT Z15.B, Z0.B, Z12.S // 0c048f44
+ ZUHADD Z15.B, Z0.B, P3.M, Z0.B // e08d1144
+ ZUHSUB Z15.B, Z0.B, P3.M, Z0.B // e08d1344
+ ZUHSUBR Z15.B, Z0.B, P3.M, Z0.B // e08d1744
+ ZUMAX Z15.B, Z0.B, P3.M, Z0.B // e00d0904
+ ZUMAXP Z15.B, Z0.B, P3.M, Z0.B // e0ad1544
+ ZUMAXQV Z25.S, P3, V5.S4 // 252f8d04
+ ZUMIN Z15.B, Z0.B, P3.M, Z0.B // e00d0b04
+ ZUMINP Z15.B, Z0.B, P3.M, Z0.B // e0ad1744
+ ZUMINQV Z25.S, P3, V5.S4 // 252f8f04
+ ZUMLALB Z8.B, Z4.B, Z30.H // 9e484844
+ ZUMLALT Z8.B, Z4.B, Z30.H // 9e4c4844
+ ZUMLSLB Z8.B, Z4.B, Z30.H // 9e584844
+ ZUMLSLT Z8.B, Z4.B, Z30.H // 9e5c4844
+ ZUMMLA Z7.B, Z6.B, Z23.S // d798c745
+ ZUMULH Z15.B, Z0.B, P3.M, Z0.B // e00d1304
+ ZUMULH Z7.D, Z23.D, Z13.D // ed6ee704
+ ZUMULLB Z8.B, Z4.B, Z30.H // 9e784845
+ ZUMULLT Z8.B, Z4.B, Z30.H // 9e7c4845
+ ZUQADD Z15.B, Z0.B, P3.M, Z0.B // e08d1944
+ ZUQADD Z7.D, Z23.D, Z13.D // ed16e704
+ ZUQDECP P14.S, Z26.S // da81ab25
+ ZUQINCP P14.S, Z26.S // da81a925
+ ZUQRSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0b44
+ ZUQRSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0f44
+ ZUQSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0944
+ ZUQSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0d44
+ ZUQSUB Z15.B, Z0.B, P3.M, Z0.B // e08d1b44
+ ZUQSUB Z7.D, Z23.D, Z13.D // ed1ee704
+ ZUQSUBR Z15.B, Z0.B, P3.M, Z0.B // e08d1f44
+ ZUQXTNB Z30.D, Z29.S // dd4b6045
+ ZUQXTNT Z30.D, Z29.S // dd4f6045
+ ZURECPE Z13.S, P1.M, Z22.S // b6a58044
+ ZURECPE Z13.S, P1.Z, Z22.S // b6a58244
+ ZURHADD Z15.B, Z0.B, P3.M, Z0.B // e08d1544
+ ZURSHL Z15.B, Z0.B, P3.M, Z0.B // e08d0344
+ ZURSHLR Z15.B, Z0.B, P3.M, Z0.B // e08d0744
+ ZURSQRTE Z13.S, P1.M, Z22.S // b6a58144
+ ZURSQRTE Z13.S, P1.Z, Z22.S // b6a58344
+ ZUSDOT Z7.B, Z6.B, Z23.S // d7788744
+ ZUSMMLA Z7.B, Z6.B, Z23.S // d7988745
+ ZUSQADD Z15.B, Z0.B, P3.M, Z0.B // e08d1d44
+ ZUSUBLB Z8.B, Z4.B, Z30.H // 9e184845
+ ZUSUBLT Z8.B, Z4.B, Z30.H // 9e1c4845
+ ZUSUBWB Z22.S, Z10.D, Z5.D // 4559d645
+ ZUSUBWT Z22.S, Z10.D, Z5.D // 455dd645
+ ZUUNPKHI Z15.B, Z0.H // e0397305
+ ZUUNPKLO Z15.B, Z0.H // e0397205
+ ZUXTB Z7.D, P4.M, Z13.D // edb0d104
+ ZUXTB Z7.D, P4.Z, Z13.D // edb0c104
+ ZUXTH Z7.D, P4.M, Z13.D // edb0d304
+ ZUXTH Z7.D, P4.Z, Z13.D // edb0c304
+ ZUXTW Z13.D, P1.M, Z22.D // b6a5d504
+ ZUXTW Z13.D, P1.Z, Z22.D // b6a5c504
+ ZUZP1 Z7.D, Z23.D, Z13.D // ed6ae705
+ ZUZP1 Z7.Q, Z6.Q, Z23.Q // d708a705
+ ZUZP2 Z7.D, Z23.D, Z13.D // ed6ee705
+ ZUZP2 Z7.Q, Z6.Q, Z23.Q // d70ca705
+ ZUZPQ1 Z7.D, Z23.D, Z13.D // edeac744
+ ZUZPQ2 Z7.D, Z23.D, Z13.D // edeec744
+ ZZIP1 Z7.D, Z23.D, Z13.D // ed62e705
+ ZZIP1 Z7.Q, Z6.Q, Z23.Q // d700a705
+ ZZIP2 Z7.D, Z23.D, Z13.D // ed66e705
+ ZZIP2 Z7.Q, Z6.Q, Z23.Q // d704a705
+ ZZIPQ1 Z7.D, Z23.D, Z13.D // ede2c744
+ ZZIPQ2 Z7.D, Z23.D, Z13.D // ede6c744
+ CTERMEQ ZR, R25 // 2023ff25
+ CTERMEQW ZR, R25 // 2023bf25
+ CTERMNE ZR, R25 // 3023ff25
+ CTERMNEW ZR, R25 // 3023bf25
+ PCNTP P2.B, P14, R2 // 42b82025
+ PDECP P14.S, ZR // df89ad25
+ PFIRSTP P2.B, P14, R2 // 42b82125
+ PINCP P14.S, ZR // df89ac25
+ PLASTP P2.B, P14, R2 // 42b82225
+ PSQDECP P14.S, ZR // df8daa25
+ PSQDECPW R8, P10.D, R8 // 4889ea25
+ PSQINCP P14.S, ZR // df8da825
+ PSQINCPW R8, P10.D, R8 // 4889e825
+ PUQDECP P14.S, ZR // df8dab25
+ PUQDECPW P14.S, ZR // df89ab25
+ PUQINCP P14.S, ZR // df8da925
+ PUQINCPW P14.S, ZR // df89a925
+ PWHILEGE R2, R10, P10.H // 4a116225
+ PWHILEGEW R2, R10, P10.H // 4a016225
+ PWHILEGT R2, R10, P10.H // 5a116225
+ PWHILEGTW R2, R10, P10.H // 5a016225
+ PWHILEHI R2, R10, P10.H // 5a196225
+ PWHILEHIW R2, R10, P10.H // 5a096225
+ PWHILEHS R2, R10, P10.H // 4a196225
+ PWHILEHSW R2, R10, P10.H // 4a096225
+ PWHILELE R2, R10, P10.H // 5a156225
+ PWHILELEW R2, R10, P10.H // 5a056225
+ PWHILELO R2, R10, P10.H // 4a1d6225
+ PWHILELOW R2, R10, P10.H // 4a0d6225
+ PWHILELS R2, R10, P10.H // 5a1d6225
+ PWHILELSW R2, R10, P10.H // 5a0d6225
+ PWHILELT R2, R10, P10.H // 4a156225
+ PWHILELTW R2, R10, P10.H // 4a056225
+ PWHILERW R2, R10, P10.H // 5a316225
+ PWHILEWR R2, R10, P10.H // 4a316225
+ ZANDVB Z6.B, P3, V2 // c22c1a04
+ ZANDVD Z10.D, P3, V15 // 4f2dda04
+ ZANDVH Z3.H, P1, V29 // 7d245a04
+ ZANDVS Z17.S, P1, V27 // 3b269a04
+ ZCLASTA Z9.D, R10, P2, R10 // 2aa9f005
+ ZCLASTAB Z2.B, V29, P1, V29 // 5d842a05
+ ZCLASTAD Z9.D, V10, P2, V10 // 2a89ea05
+ ZCLASTAH Z8.H, V15, P2, V15 // 0f896a05
+ ZCLASTAS Z26.S, V30, P7, V30 // 5e9faa05
+ ZCLASTAW Z8.H, R15, P2, R15 // 0fa97005
+ ZCLASTB Z9.D, R10, P2, R10 // 2aa9f105
+ ZCLASTBB Z2.B, V29, P1, V29 // 5d842b05
+ ZCLASTBD Z9.D, V10, P2, V10 // 2a89eb05
+ ZCLASTBH Z8.H, V15, P2, V15 // 0f896b05
+ ZCLASTBS Z26.S, V30, P7, V30 // 5e9fab05
+ ZCLASTBW Z8.H, R15, P2, R15 // 0fa97105
+ ZCPY R20, P1.M, Z2.D // 82a6e805
+ ZCPYB V4, P1.M, Z16.B // 90842005
+ ZCPYD V20, P1.M, Z2.D // 8286e005
+ ZCPYH V7, P6.M, Z23.H // f7986005
+ ZCPYS V13, P1.M, Z22.S // b685a005
+ ZCPYW RSP, P5.M, Z6.H // e6b76805
+ ZDUP R2, Z10.D // 4a38e005
+ ZDUPW R25, Z11.B // 2b3b2005
+ ZEORVB Z6.B, P3, V2 // c22c1904
+ ZEORVD Z10.D, P3, V15 // 4f2dd904
+ ZEORVH Z3.H, P1, V29 // 7d245904
+ ZEORVS Z17.S, P1, V27 // 3b269904
+ ZFADDAD Z9.D, V10, P2, V10 // 2a29d865
+ ZFADDAH Z8.H, V15, P2, V15 // 0f295865
+ ZFADDAS Z26.S, V30, P7, V30 // 5e3f9865
+ ZFADDVD Z10.D, P3, V15 // 4f2dc065
+ ZFADDVH Z3.H, P1, V29 // 7d244065
+ ZFADDVS Z17.S, P1, V27 // 3b268065
+ ZFMAXNMVD Z10.D, P3, V15 // 4f2dc465
+ ZFMAXNMVH Z3.H, P1, V29 // 7d244465
+ ZFMAXNMVS Z17.S, P1, V27 // 3b268465
+ ZFMAXVD Z10.D, P3, V15 // 4f2dc665
+ ZFMAXVH Z3.H, P1, V29 // 7d244665
+ ZFMAXVS Z17.S, P1, V27 // 3b268665
+ ZFMINNMVD Z10.D, P3, V15 // 4f2dc565
+ ZFMINNMVH Z3.H, P1, V29 // 7d244565
+ ZFMINNMVS Z17.S, P1, V27 // 3b268565
+ ZFMINVD Z10.D, P3, V15 // 4f2dc765
+ ZFMINVH Z3.H, P1, V29 // 7d244765
+ ZFMINVS Z17.S, P1, V27 // 3b268765
+ ZINDEX R13, R20, Z9.D // 894eed04
+ ZINDEXW R2, R10, Z8.H // 484d6204
+ ZINSR R2, Z10.D // 4a38e405
+ ZINSRB V25, Z11.B // 2b3b3405
+ ZINSRD V2, Z10.D // 4a38f405
+ ZINSRH V7, Z6.H // e6387405
+ ZINSRS V14, Z8.S // c839b405
+ ZINSRW R25, Z11.B // 2b3b2405
+ ZLASTA Z10.D, P3, R15 // 4fade005
+ ZLASTAB Z6.B, P3, V2 // c28c2205
+ ZLASTAD Z10.D, P3, V15 // 4f8de205
+ ZLASTAH Z3.H, P1, V29 // 7d846205
+ ZLASTAS Z17.S, P1, V27 // 3b86a205
+ ZLASTAW Z6.B, P3, R2 // c2ac2005
+ ZLASTB Z10.D, P3, R15 // 4fade105
+ ZLASTBB Z6.B, P3, V2 // c28c2305
+ ZLASTBD Z10.D, P3, V15 // 4f8de305
+ ZLASTBH Z3.H, P1, V29 // 7d846305
+ ZLASTBS Z17.S, P1, V27 // 3b86a305
+ ZLASTBW Z6.B, P3, R2 // c2ac2105
+ ZORVB Z6.B, P3, V2 // c22c1804
+ ZORVD Z10.D, P3, V15 // 4f2dd804
+ ZORVH Z3.H, P1, V29 // 7d245804
+ ZORVS Z17.S, P1, V27 // 3b269804
+ ZSADDVD Z6.B, P3, V2 // c22c0004
+ ZSMAXVB Z6.B, P3, V2 // c22c0804
+ ZSMAXVD Z10.D, P3, V15 // 4f2dc804
+ ZSMAXVH Z3.H, P1, V29 // 7d244804
+ ZSMAXVS Z17.S, P1, V27 // 3b268804
+ ZSMINVB Z6.B, P3, V2 // c22c0a04
+ ZSMINVD Z10.D, P3, V15 // 4f2dca04
+ ZSMINVH Z3.H, P1, V29 // 7d244a04
+ ZSMINVS Z17.S, P1, V27 // 3b268a04
+ ZUADDVD Z10.D, P3, V15 // 4f2dc104
+ ZUMAXVB Z6.B, P3, V2 // c22c0904
+ ZUMAXVD Z10.D, P3, V15 // 4f2dc904
+ ZUMAXVH Z3.H, P1, V29 // 7d244904
+ ZUMAXVS Z17.S, P1, V27 // 3b268904
+ ZUMINVB Z6.B, P3, V2 // c22c0b04
+ ZUMINVD Z10.D, P3, V15 // 4f2dcb04
+ ZUMINVH Z3.H, P1, V29 // 7d244b04
+ ZUMINVS Z17.S, P1, V27 // 3b268b04
+ PPEXT PN11[1], P13.S // 7d71a025
+ ZBFDOT Z6.H[3], Z23.H, Z13.S // ed427e64
+ ZBFMLA Z6.H[3], Z23.H, Z13.H // ed0a3e64
+ ZBFMLALB Z6.H[3], Z23.H, Z13.S // ed4aee64
+ ZBFMLALT Z6.H[3], Z23.H, Z13.S // ed4eee64
+ ZBFMLS Z6.H[3], Z23.H, Z13.H // ed0e3e64
+ ZBFMLSLB Z6.H[3], Z23.H, Z13.S // ed6aee64
+ ZBFMLSLT Z6.H[3], Z23.H, Z13.S // ed6eee64
+ ZBFMUL Z6.H[3], Z23.H, Z13.H // ed2a3e64
+ ZDUP Z27.Q[1], Z25.Q // 79237005
+ ZDUPQ Z8.H[2], Z15.H // 0f252a05
+ ZFDOT Z6.H[3], Z23.H, Z13.S // ed423e64
+ ZFDOT Z6.B[3], Z23.B, Z13.H // ed4e2e64
+ ZFDOT Z6.B[3], Z23.B, Z13.S // ed467e64
+ ZFMLA Z6.H[3], Z23.H, Z13.H // ed023e64
+ ZFMLA Z6.S[3], Z23.S, Z13.S // ed02be64
+ ZFMLA Z5.D[0], Z15.D, Z6.D // e601e564
+ ZFMLALB Z6.H[3], Z23.H, Z13.S // ed4aae64
+ ZFMLALB Z6.B[3], Z23.B, Z13.H // ed5e2664
+ ZFMLALLBB Z6.B[3], Z23.B, Z13.S // edce2664
+ ZFMLALLBT Z6.B[3], Z23.B, Z13.S // edce6664
+ ZFMLALLTB Z6.B[3], Z23.B, Z13.S // edcea664
+ ZFMLALLTT Z6.B[3], Z23.B, Z13.S // edcee664
+ ZFMLALT Z6.H[3], Z23.H, Z13.S // ed4eae64
+ ZFMLALT Z6.B[3], Z23.B, Z13.H // ed5ea664
+ ZFMLS Z6.H[3], Z23.H, Z13.H // ed063e64
+ ZFMLS Z6.S[3], Z23.S, Z13.S // ed06be64
+ ZFMLS Z5.D[0], Z15.D, Z6.D // e605e564
+ ZFMLSLB Z6.H[3], Z23.H, Z13.S // ed6aae64
+ ZFMLSLT Z6.H[3], Z23.H, Z13.S // ed6eae64
+ ZFMUL Z6.H[3], Z23.H, Z13.H // ed223e64
+ ZFMUL Z6.S[3], Z23.S, Z13.S // ed22be64
+ ZFMUL Z5.D[0], Z15.D, Z6.D // e621e564
+ ZMLA Z6.H[3], Z23.H, Z13.H // ed0a3e44
+ ZMLA Z6.S[3], Z23.S, Z13.S // ed0abe44
+ ZMLA Z5.D[0], Z15.D, Z6.D // e609e544
+ ZMLS Z6.H[3], Z23.H, Z13.H // ed0e3e44
+ ZMLS Z6.S[3], Z23.S, Z13.S // ed0ebe44
+ ZMLS Z5.D[0], Z15.D, Z6.D // e60de544
+ ZMUL Z6.H[3], Z23.H, Z13.H // edfa3e44
+ ZMUL Z6.S[3], Z23.S, Z13.S // edfabe44
+ ZMUL Z5.D[0], Z15.D, Z6.D // e6f9e544
+ ZPMOV Z27[1], P13.S // 6d3b6a05
+ ZPMOV Z27[1], P13.S // 6d3b6a05
+ ZPMOV Z27[1], P13.S // 6d3b6a05
+ ZPMOV P14.S, Z25[2] // d9396d05
+ ZPMOV P14.S, Z25[2] // d9396d05
+ ZPMOV P14.S, Z25[2] // d9396d05
+ // TODO: SDOT .H, .B, .B[]
+ ZSDOT Z6.H[3], Z23.H, Z13.S // edca9e44
+ ZSDOT Z6.B[3], Z23.B, Z13.S // ed02be44
+ ZSDOT Z5.H[0], Z15.H, Z6.D // e601e544
+ ZSMLALB Z6.H[3], Z23.H, Z13.S // ed8aae44
+ ZSMLALB Z6.S[3], Z23.S, Z13.D // ed8af644
+ ZSMLALT Z6.H[3], Z23.H, Z13.S // ed8eae44
+ ZSMLALT Z6.S[3], Z23.S, Z13.D // ed8ef644
+ ZSMLSLB Z6.H[3], Z23.H, Z13.S // edaaae44
+ ZSMLSLB Z6.S[3], Z23.S, Z13.D // edaaf644
+ ZSMLSLT Z6.H[3], Z23.H, Z13.S // edaeae44
+ ZSMLSLT Z6.S[3], Z23.S, Z13.D // edaef644
+ ZSMULLB Z6.H[3], Z23.H, Z13.S // edcaae44
+ ZSMULLB Z6.S[3], Z23.S, Z13.D // edcaf644
+ ZSMULLT Z6.H[3], Z23.H, Z13.S // edceae44
+ ZSMULLT Z6.S[3], Z23.S, Z13.D // edcef644
+ ZSQDMLALB Z6.H[3], Z23.H, Z13.S // ed2aae44
+ ZSQDMLALB Z6.S[3], Z23.S, Z13.D // ed2af644
+ ZSQDMLALT Z6.H[3], Z23.H, Z13.S // ed2eae44
+ ZSQDMLALT Z6.S[3], Z23.S, Z13.D // ed2ef644
+ ZSQDMLSLB Z6.H[3], Z23.H, Z13.S // ed3aae44
+ ZSQDMLSLB Z6.S[3], Z23.S, Z13.D // ed3af644
+ ZSQDMLSLT Z6.H[3], Z23.H, Z13.S // ed3eae44
+ ZSQDMLSLT Z6.S[3], Z23.S, Z13.D // ed3ef644
+ ZSQDMULH Z6.H[3], Z23.H, Z13.H // edf23e44
+ ZSQDMULH Z6.S[3], Z23.S, Z13.S // edf2be44
+ ZSQDMULH Z5.D[0], Z15.D, Z6.D // e6f1e544
+ ZSQDMULLB Z6.H[3], Z23.H, Z13.S // edeaae44
+ ZSQDMULLB Z6.S[3], Z23.S, Z13.D // edeaf644
+ ZSQDMULLT Z6.H[3], Z23.H, Z13.S // edeeae44
+ ZSQDMULLT Z6.S[3], Z23.S, Z13.D // edeef644
+ ZSQRDMLAH Z6.H[3], Z23.H, Z13.H // ed123e44
+ ZSQRDMLAH Z6.S[3], Z23.S, Z13.S // ed12be44
+ ZSQRDMLAH Z5.D[0], Z15.D, Z6.D // e611e544
+ ZSQRDMLSH Z6.H[3], Z23.H, Z13.H // ed163e44
+ ZSQRDMLSH Z6.S[3], Z23.S, Z13.S // ed16be44
+ ZSQRDMLSH Z5.D[0], Z15.D, Z6.D // e615e544
+ ZSQRDMULH Z6.H[3], Z23.H, Z13.H // edf63e44
+ ZSQRDMULH Z6.S[3], Z23.S, Z13.S // edf6be44
+ ZSQRDMULH Z5.D[0], Z15.D, Z6.D // e6f5e544
+ ZSUDOT Z6.B[3], Z23.B, Z13.S // ed1ebe44
+ // TODO: UDOT .H, .B, .B[]
+ ZUDOT Z6.H[3], Z23.H, Z13.S // edce9e44
+ ZUDOT Z6.B[3], Z23.B, Z13.S // ed06be44
+ ZUDOT Z5.H[0], Z15.H, Z6.D // e605e544
+ ZUMLALB Z6.H[3], Z23.H, Z13.S // ed9aae44
+ ZUMLALB Z6.S[3], Z23.S, Z13.D // ed9af644
+ ZUMLALT Z6.H[3], Z23.H, Z13.S // ed9eae44
+ ZUMLALT Z6.S[3], Z23.S, Z13.D // ed9ef644
+ ZUMLSLB Z6.H[3], Z23.H, Z13.S // edbaae44
+ ZUMLSLB Z6.S[3], Z23.S, Z13.D // edbaf644
+ ZUMLSLT Z6.H[3], Z23.H, Z13.S // edbeae44
+ ZUMLSLT Z6.S[3], Z23.S, Z13.D // edbef644
+ ZUMULLB Z6.H[3], Z23.H, Z13.S // eddaae44
+ ZUMULLB Z6.S[3], Z23.S, Z13.D // eddaf644
+ ZUMULLT Z6.H[3], Z23.H, Z13.S // eddeae44
+ ZUMULLT Z6.S[3], Z23.S, Z13.D // eddef644
+ ZUSDOT Z6.B[3], Z23.B, Z13.S // ed1abe44
+ ADDPL $9, R11, RSP // 3f516b04
+ ADDVL $9, R11, RSP // 3f512b04
+ RDVL $1, R27 // 3b50bf04
+ ZADD $6, Z7.D, Z7.D // c7c0e025
+ ZAND $6, Z7.D, Z7.D // 27f88305
+ ZASR $6, Z23.B, P1.M, Z23.B // 57850004
+ ZASR $6, Z7.D, Z23.D // f790fa04
+ ZASRD $6, Z23.B, P1.M, Z23.B // 57850404
+ ZCADD $270, Z25.S, Z22.S, Z22.S // 36df8045
+ ZCDOT $0, Z5.B[3], Z16.B, Z26.S // 1a42bd44
+ ZCDOT $270, Z3.H[0], Z2.H, Z23.D // 574ce344
+ ZCDOT $270, Z28.B, Z1.B, Z9.S // 291c9c44
+ ZCMLA $0, Z5.H[3], Z16.H, Z26.H // 1a62bd44
+ ZCMLA $270, Z3.S[0], Z2.S, Z23.S // 576ce344
+ ZCMLA $180, Z19.S, Z20.S, Z9.S // 892a9344
+ ZCMPEQ $6, Z23.B, P1.Z, P14.B // ee860625
+ ZCMPGE $6, Z23.B, P1.Z, P14.B // ee060625
+ ZCMPGT $6, Z23.B, P1.Z, P14.B // fe060625
+ ZCMPHI $6, Z23.B, P1.Z, P14.B // fe862124
+ ZCMPHS $6, Z23.B, P1.Z, P14.B // ee862124
+ // TODO: CMPLE
+ // TODO: CMPLO
+ // TODO: CMPLS
+ // TODO: CMPLT
+ ZCMPNE $6, Z23.B, P1.Z, P14.B // fe860625
+ ZCPY $6, P5.M, Z6.H // c6405505
+ ZCPY $6, P5.Z, Z6.H // c6005505
+ ZDUP $9, Z11.B // 2bc13825
+ ZDUPM $7, Z6.H // 4604c005
+ ZEOR $6, Z7.D, Z7.D // 27f84305
+ ZEXT $6, Z23.B, Z13.B, Z13.B // ed1a2005
+ ZEXTQ $6, Z23.B, Z13.B, Z13.B // ed266605
+ ZFADD $(1.0), Z5.D, P5.M, Z5.D // 2594d865
+ ZFCADD $90, Z13.S, Z4.S, P4.M, Z4.S // a4918064
+ ZFCMEQ $(0.0), Z7.D, P4.Z, P2.D // e230d265
+ ZFCMGE $(0.0), Z7.D, P4.Z, P2.D // e230d065
+ ZFCMGT $(0.0), Z7.D, P4.Z, P2.D // f230d065
+ ZFCMLA $0, Z5.H[3], Z16.H, Z26.H // 1a12bd64
+ ZFCMLA $270, Z3.S[0], Z2.S, Z23.S // 571ce364
+ ZFCMLA $180, Z30.D, Z29.D, P7.M, Z25.D // b95fde64
+ ZFCMLE $(0.0), Z7.D, P4.Z, P2.D // f230d165
+ ZFCMLT $(0.0), Z7.D, P4.Z, P2.D // e230d165
+ ZFCMNE $(0.0), Z7.D, P4.Z, P2.D // e230d365
+ ZFCPY $(2.0), P6.M, Z23.H // 17c05605
+ ZFDUP $(2.0), Z4.S // 04c0b925
+ ZFMAX $(0.0), Z3.D, P1.M, Z3.D // 0384de65
+ ZFMAXNM $(0.0), Z3.D, P1.M, Z3.D // 0384dc65
+ ZFMIN $(0.0), Z3.D, P1.M, Z3.D // 0384df65
+ ZFMINNM $(0.0), Z3.D, P1.M, Z3.D // 0384dd65
+ ZFMUL $(2.0), Z16.S, P1.M, Z16.S // 30849a65
+ ZFSUB $(1.0), Z5.D, P5.M, Z5.D // 2594d965
+ ZFSUBR $(1.0), Z5.D, P5.M, Z5.D // 2594db65
+ ZFTMAD $3, Z13.S, Z9.S, Z9.S // a9819365
+ ZINDEX R11, $6, Z7.D // c748eb04
+ ZINDEX $1, $11, Z26.S // 7a41a104
+ ZINDEX $8, R24, Z14.D // 0e47e804
+ ZINDEXW R2, $10, Z8.H // 48496204
+ ZINDEXW $1, R27, Z26.S // 7a47a104
+ ZLSL $6, Z23.B, P1.M, Z23.B // d7850304
+ ZLSL $6, Z7.D, Z23.D // f79ca604
+ ZLSR $6, Z23.B, P1.M, Z23.B // 57850104
+ ZLSR $6, Z7.D, Z23.D // f794fa04
+ ZMUL $6, Z7.D, Z7.D // c7c0f025
+ ZORR $6, Z7.D, Z7.D // 27f80305
+ ZRSHRNB $9, Z7.S, Z23.H // f7183745
+ ZRSHRNT $9, Z7.S, Z23.H // f71c3745
+ ZSHRNB $9, Z7.S, Z23.H // f7103745
+ ZSHRNT $9, Z7.S, Z23.H // f7143745
+ ZSLI $6, Z7.D, Z23.D // f7f48645
+ ZSMAX $6, Z7.D, Z7.D // c7c0e825
+ ZSMIN $6, Z7.D, Z7.D // c7c0ea25
+ ZSQADD $6, Z7.D, Z7.D // c7c0e425
+ ZSQCADD $270, Z25.S, Z22.S, Z22.S // 36df8145
+ ZSQRDCMLAH $0, Z5.H[3], Z16.H, Z26.H // 1a72bd44
+ ZSQRDCMLAH $270, Z3.S[0], Z2.S, Z23.S // 577ce344
+ ZSQRDCMLAH $180, Z19.S, Z20.S, Z9.S // 893a9344
+ ZSQRSHRNB $9, Z7.S, Z23.H // f7283745
+ ZSQRSHRNT $9, Z7.S, Z23.H // f72c3745
+ ZSQRSHRUNB $9, Z7.S, Z23.H // f7083745
+ ZSQRSHRUNT $9, Z7.S, Z23.H // f70c3745
+ ZSQSHL $6, Z23.B, P1.M, Z23.B // d7850604
+ ZSQSHLU $6, Z23.B, P1.M, Z23.B // d7850f04
+ ZSQSHRNB $9, Z7.S, Z23.H // f7203745
+ ZSQSHRNT $9, Z7.S, Z23.H // f7243745
+ ZSQSHRUNB $9, Z7.S, Z23.H // f7003745
+ ZSQSHRUNT $9, Z7.S, Z23.H // f7043745
+ ZSQSUB $6, Z7.D, Z7.D // c7c0e625
+ ZSRI $6, Z7.D, Z23.D // f7f0da45
+ ZSRSHR $6, Z23.B, P1.M, Z23.B // 57850c04
+ ZSRSRA $6, Z7.D, Z23.D // f7e8da45
+ ZSSHLLB $5, Z22.S, Z10.D // caa24545
+ ZSSHLLT $5, Z22.S, Z10.D // caa64545
+ ZSSRA $6, Z7.D, Z23.D // f7e0da45
+ ZSUB $6, Z7.D, Z7.D // c7c0e125
+ ZSUBR $6, Z7.D, Z7.D // c7c0e325
+ ZUMAX $6, Z7.D, Z7.D // c7c0e925
+ ZUMIN $6, Z7.D, Z7.D // c7c0eb25
+ ZUQADD $6, Z7.D, Z7.D // c7c0e525
+ ZUQRSHRNB $9, Z7.S, Z23.H // f7383745
+ ZUQRSHRNT $9, Z7.S, Z23.H // f73c3745
+ ZUQSHL $6, Z23.B, P1.M, Z23.B // d7850704
+ ZUQSHRNB $9, Z7.S, Z23.H // f7303745
+ ZUQSHRNT $9, Z7.S, Z23.H // f7343745
+ ZUQSUB $6, Z7.D, Z7.D // c7c0e725
+ ZURSHR $6, Z23.B, P1.M, Z23.B // 57850d04
+ ZURSRA $6, Z7.D, Z23.D // f7ecda45
+ ZUSHLLB $5, Z22.S, Z10.D // caaa4545
+ ZUSHLLT $5, Z22.S, Z10.D // caae4545
+ ZUSRA $6, Z7.D, Z23.D // f7e4da45
+ ZXAR $6, Z23.B, Z21.B, Z21.B // f5362a04
+ PPEXT PN11[1], [P13.S, P14.S] // 7d75a025
+ PWHILEGE R2, R10, [P10.H, P11.H] // 5a516225
+ PWHILEGT R2, R10, [P10.H, P11.H] // 5b516225
+ PWHILEHI R2, R10, [P10.H, P11.H] // 5b596225
+ PWHILEHS R2, R10, [P10.H, P11.H] // 5a596225
+ PWHILELE R2, R10, [P10.H, P11.H] // 5b556225
+ PWHILELO R2, R10, [P10.H, P11.H] // 5a5d6225
+ PWHILELS R2, R10, [P10.H, P11.H] // 5b5d6225
+ PWHILELT R2, R10, [P10.H, P11.H] // 5a556225
+ ZEXT $6, [Z7.B, Z8.B], Z6.B // e6186005
+ ZLUTI2 Z6[3], [Z23.B], Z13.B // edb2e645
+ ZLUTI2 Z6[3], [Z23.H], Z13.H // edba6645
+ ZLUTI4 Z22[1], [Z2.B], Z10.B // 4aa4f645
+ ZLUTI4 Z6[3], [Z23.H, Z24.H], Z13.H // edb6e645
+ ZLUTI4 Z6[3], [Z23.H], Z13.H // edbee645
+ // TODO: LUTI6
+ // TODO: LUTI6
+ ZSPLICE [Z7.D, Z8.D], P4, Z13.D // ed90ed05
+ ZTBL Z7.D, [Z23.D], Z13.D // ed32e705
+ ZTBL Z7.D, [Z23.D, Z24.D], Z13.D // ed2ae705
+ ZTBLQ Z7.D, [Z23.D], Z13.D // edfac744
+ ZADR (Z16.S)(Z12.S), Z13.S // 8da1b004
+ ZADR (Z7.D.SXTW<<2)(Z6.D), Z23.D // d7a82704
+ ZADR (Z7.D.UXTW<<2)(Z6.D), Z23.D // d7a86704
+ ZLD1B (R6)(R14), P4.Z, [Z13.B] // cd5106a4
+ ZLD1B (R6)(R14), P4.Z, [Z13.H] // cd5126a4
+ ZLD1B (R6)(R14), P4.Z, [Z13.S] // cd5146a4
+ ZLD1B (R6)(R14), P4.Z, [Z13.D] // cd5166a4
+ ZLD1B (Z10.D)(R19), P3.Z, [Z15.D] // 6fce4ac4
+ ZLD1B (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 644c0484
+ ZLD1B (Z6.D)(R14), P4.Z, [Z13.D] // cdd146c4
+ ZLD1D (R6<<3)(R14), P4.Z, [Z13.D] // cd51e6a5
+ ZLD1D (R6<<3)(R14), P4.Z, [Z13.Q] // cd9186a5
+ ZLD1D (Z23.D<<3)(R24), P1.Z, [Z22.D] // 16c7f7c5
+ ZLD1D (Z10.D)(R19), P3.Z, [Z15.D] // 6fcecac5
+ ZLD1D (Z6.D<<3)(R14), P4.Z, [Z13.D] // cdd1e6c5
+ ZLD1D (Z6.D)(R14), P4.Z, [Z13.D] // cdd1c6c5
+ ZLD1H (R6<<1)(R14), P4.Z, [Z13.H] // cd51a6a4
+ ZLD1H (R6<<1)(R14), P4.Z, [Z13.S] // cd51c6a4
+ ZLD1H (R6<<1)(R14), P4.Z, [Z13.D] // cd51e6a4
+ ZLD1H (Z4.S.UXTW<<1)(R3), P3.Z, [Z4.S] // 644ca484
+ ZLD1H (Z23.D<<1)(R24), P1.Z, [Z22.D] // 16c7f7c4
+ ZLD1H (Z10.D)(R19), P3.Z, [Z15.D] // 6fcecac4
+ ZLD1H (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 644c8484
+ ZLD1H (Z6.D<<1)(R14), P4.Z, [Z13.D] // cdd1e6c4
+ ZLD1H (Z6.D)(R14), P4.Z, [Z13.D] // cdd1c6c4
+ ZLD1Q (R6)(Z7.D), P4.Z, [Z13.Q] // edb006c4
+ ZLD1ROB (R6)(R14), P4.Z, [Z13.B] // cd1126a4
+ ZLD1ROD (R6<<3)(R14), P4.Z, [Z13.D] // cd11a6a5
+ ZLD1ROH (R6<<1)(R14), P4.Z, [Z13.H] // cd11a6a4
+ ZLD1ROW (R6<<2)(R14), P4.Z, [Z13.S] // cd1126a5
+ ZLD1RQB (R6)(R14), P4.Z, [Z13.B] // cd1106a4
+ ZLD1RQD (R6<<3)(R14), P4.Z, [Z13.D] // cd1186a5
+ ZLD1RQH (R6<<1)(R14), P4.Z, [Z13.H] // cd1186a4
+ ZLD1RQW (R6<<2)(R14), P4.Z, [Z13.S] // cd1106a5
+ ZLD1SB (R6)(R14), P4.Z, [Z13.H] // cd51c6a5
+ ZLD1SB (R6)(R14), P4.Z, [Z13.S] // cd51a6a5
+ ZLD1SB (R6)(R14), P4.Z, [Z13.D] // cd5186a5
+ ZLD1SB (Z10.D)(R19), P3.Z, [Z15.D] // 6f8e4ac4
+ ZLD1SB (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 640c0484
+ ZLD1SB (Z6.D)(R14), P4.Z, [Z13.D] // cd9146c4
+ ZLD1SH (R6<<1)(R14), P4.Z, [Z13.S] // cd5126a5
+ ZLD1SH (R6<<1)(R14), P4.Z, [Z13.D] // cd5106a5
+ ZLD1SH (Z4.S.UXTW<<1)(R3), P3.Z, [Z4.S] // 640ca484
+ ZLD1SH (Z23.D<<1)(R24), P1.Z, [Z22.D] // 1687f7c4
+ ZLD1SH (Z10.D)(R19), P3.Z, [Z15.D] // 6f8ecac4
+ ZLD1SH (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 640c8484
+ ZLD1SH (Z6.D<<1)(R14), P4.Z, [Z13.D] // cd91e6c4
+ ZLD1SH (Z6.D)(R14), P4.Z, [Z13.D] // cd91c6c4
+ ZLD1SW (R6<<2)(R14), P4.Z, [Z13.D] // cd5186a4
+ ZLD1SW (Z23.D<<2)(R24), P1.Z, [Z22.D] // 168777c5
+ ZLD1SW (Z10.D)(R19), P3.Z, [Z15.D] // 6f8e4ac5
+ ZLD1SW (Z6.D<<2)(R14), P4.Z, [Z13.D] // cd9166c5
+ ZLD1SW (Z6.D)(R14), P4.Z, [Z13.D] // cd9146c5
+ ZLD1W (R6<<2)(R14), P4.Z, [Z13.S] // cd5146a5
+ ZLD1W (R6<<2)(R14), P4.Z, [Z13.D] // cd5166a5
+ ZLD1W (R6<<2)(R14), P4.Z, [Z13.Q] // cd9106a5
+ ZLD1W (Z4.S.UXTW<<2)(R3), P3.Z, [Z4.S] // 644c2485
+ ZLD1W (Z23.D<<2)(R24), P1.Z, [Z22.D] // 16c777c5
+ ZLD1W (Z10.D)(R19), P3.Z, [Z15.D] // 6fce4ac5
+ ZLD1W (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 644c0485
+ ZLD1W (Z6.D<<2)(R14), P4.Z, [Z13.D] // cdd166c5
+ ZLD1W (Z6.D)(R14), P4.Z, [Z13.D] // cdd146c5
+ ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B] // cdd126a4
+ ZLD2D (R6<<3)(R14), P4.Z, [Z13.D, Z14.D] // cdd1a6a5
+ ZLD2H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H] // cdd1a6a4
+ ZLD2Q (R6<<4)(R14), P4.Z, [Z13.Q, Z14.Q] // cd91a6a4
+ ZLD2W (R6<<2)(R14), P4.Z, [Z13.S, Z14.S] // cdd126a5
+ ZLD3B (R6)(R14), P4.Z, [Z13.B, Z14.B, Z15.B] // cdd146a4
+ ZLD3D (R6<<3)(R14), P4.Z, [Z13.D, Z14.D, Z15.D] // cdd1c6a5
+ ZLD3H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H, Z15.H] // cdd1c6a4
+ ZLD3Q (R6<<4)(R14), P4.Z, [Z13.Q, Z14.Q, Z15.Q] // cd9126a5
+ ZLD3W (R6<<2)(R14), P4.Z, [Z13.S, Z14.S, Z15.S] // cdd146a5
+ ZLD4B (R6)(R14), P4.Z, [Z13.B, Z14.B, Z15.B, Z16.B]// cdd166a4
+ ZLD4D (R6<<3)(R14), P4.Z, [Z13.D, Z14.D, Z15.D, Z16.D]// cdd1e6a5
+ ZLD4H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H, Z15.H, Z16.H]// cdd1e6a4
+ ZLD4Q (R6<<4)(R14), P4.Z, [Z13.Q, Z14.Q, Z15.Q, Z16.Q]// cd91a6a5
+ ZLD4W (R6<<2)(R14), P4.Z, [Z13.S, Z14.S, Z15.S, Z16.S]// cdd166a5
+ ZLDFF1B (R6)(R14), P4.Z, [Z13.B] // cd7106a4
+ ZLDFF1B (R6)(R14), P4.Z, [Z13.H] // cd7126a4
+ ZLDFF1B (R6)(R14), P4.Z, [Z13.S] // cd7146a4
+ ZLDFF1B (R6)(R14), P4.Z, [Z13.D] // cd7166a4
+ ZLDFF1B (Z10.D)(R19), P3.Z, [Z15.D] // 6fee4ac4
+ ZLDFF1B (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 646c0484
+ ZLDFF1B (Z6.D)(R14), P4.Z, [Z13.D] // cdf146c4
+ ZLDFF1D (R6<<3)(R14), P4.Z, [Z13.D] // cd71e6a5
+ ZLDFF1D (Z23.D<<3)(R24), P1.Z, [Z22.D] // 16e7f7c5
+ ZLDFF1D (Z10.D)(R19), P3.Z, [Z15.D] // 6feecac5
+ ZLDFF1D (Z6.D<<3)(R14), P4.Z, [Z13.D] // cdf1e6c5
+ ZLDFF1D (Z6.D)(R14), P4.Z, [Z13.D] // cdf1c6c5
+ ZLDFF1H (R6<<1)(R14), P4.Z, [Z13.H] // cd71a6a4
+ ZLDFF1H (R6<<1)(R14), P4.Z, [Z13.S] // cd71c6a4
+ ZLDFF1H (R6<<1)(R14), P4.Z, [Z13.D] // cd71e6a4
+ ZLDFF1H (Z4.S.UXTW<<1)(R3), P3.Z, [Z4.S] // 646ca484
+ ZLDFF1H (Z23.D<<1)(R24), P1.Z, [Z22.D] // 16e7f7c4
+ ZLDFF1H (Z10.D)(R19), P3.Z, [Z15.D] // 6feecac4
+ ZLDFF1H (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 646c8484
+ ZLDFF1H (Z6.D<<1)(R14), P4.Z, [Z13.D] // cdf1e6c4
+ ZLDFF1H (Z6.D)(R14), P4.Z, [Z13.D] // cdf1c6c4
+ ZLDFF1SB (R6)(R14), P4.Z, [Z13.H] // cd71c6a5
+ ZLDFF1SB (R6)(R14), P4.Z, [Z13.S] // cd71a6a5
+ ZLDFF1SB (R6)(R14), P4.Z, [Z13.D] // cd7186a5
+ ZLDFF1SB (Z10.D)(R19), P3.Z, [Z15.D] // 6fae4ac4
+ ZLDFF1SB (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 642c0484
+ ZLDFF1SB (Z6.D)(R14), P4.Z, [Z13.D] // cdb146c4
+ ZLDFF1SH (R6<<1)(R14), P4.Z, [Z13.S] // cd7126a5
+ ZLDFF1SH (R6<<1)(R14), P4.Z, [Z13.D] // cd7106a5
+ ZLDFF1SH (Z4.S.UXTW<<1)(R3), P3.Z, [Z4.S] // 642ca484
+ ZLDFF1SH (Z23.D<<1)(R24), P1.Z, [Z22.D] // 16a7f7c4
+ ZLDFF1SH (Z10.D)(R19), P3.Z, [Z15.D] // 6faecac4
+ ZLDFF1SH (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 642c8484
+ ZLDFF1SH (Z6.D<<1)(R14), P4.Z, [Z13.D] // cdb1e6c4
+ ZLDFF1SH (Z6.D)(R14), P4.Z, [Z13.D] // cdb1c6c4
+ ZLDFF1SW (R6<<2)(R14), P4.Z, [Z13.D] // cd7186a4
+ ZLDFF1SW (Z23.D<<2)(R24), P1.Z, [Z22.D] // 16a777c5
+ ZLDFF1SW (Z10.D)(R19), P3.Z, [Z15.D] // 6fae4ac5
+ ZLDFF1SW (Z6.D<<2)(R14), P4.Z, [Z13.D] // cdb166c5
+ ZLDFF1SW (Z6.D)(R14), P4.Z, [Z13.D] // cdb146c5
+ ZLDFF1W (R6<<2)(R14), P4.Z, [Z13.S] // cd7146a5
+ ZLDFF1W (R6<<2)(R14), P4.Z, [Z13.D] // cd7166a5
+ ZLDFF1W (Z4.S.UXTW<<2)(R3), P3.Z, [Z4.S] // 646c2485
+ ZLDFF1W (Z23.D<<2)(R24), P1.Z, [Z22.D] // 16e777c5
+ ZLDFF1W (Z10.D)(R19), P3.Z, [Z15.D] // 6fee4ac5
+ ZLDFF1W (Z4.S.UXTW)(R3), P3.Z, [Z4.S] // 646c0485
+ ZLDFF1W (Z6.D<<2)(R14), P4.Z, [Z13.D] // cdf166c5
+ ZLDFF1W (Z6.D)(R14), P4.Z, [Z13.D] // cdf146c5
+ ZLDNT1B (R6)(R14), P4.Z, [Z13.B] // cdd106a4
+ ZLDNT1B (R6)(Z7.S), P4.Z, [Z13.S] // edb00684
+ ZLDNT1B (R6)(Z7.D), P4.Z, [Z13.D] // edd006c4
+ ZLDNT1D (R6<<3)(R14), P4.Z, [Z13.D] // cdd186a5
+ ZLDNT1D (R6)(Z7.D), P4.Z, [Z13.D] // edd086c5
+ ZLDNT1H (R6<<1)(R14), P4.Z, [Z13.H] // cdd186a4
+ ZLDNT1H (R6)(Z7.S), P4.Z, [Z13.S] // edb08684
+ ZLDNT1H (R6)(Z7.D), P4.Z, [Z13.D] // edd086c4
+ ZLDNT1SB (R6)(Z7.S), P4.Z, [Z13.S] // ed900684
+ ZLDNT1SB (R6)(Z7.D), P4.Z, [Z13.D] // ed9006c4
+ ZLDNT1SH (R6)(Z7.S), P4.Z, [Z13.S] // ed908684
+ ZLDNT1SH (R6)(Z7.D), P4.Z, [Z13.D] // ed9086c4
+ ZLDNT1SW (R6)(Z7.D), P4.Z, [Z13.D] // ed9006c5
+ ZLDNT1W (R6<<2)(R14), P4.Z, [Z13.S] // cdd106a5
+ ZLDNT1W (R6)(Z7.S), P4.Z, [Z13.S] // edb00685
+ ZLDNT1W (R6)(Z7.D), P4.Z, [Z13.D] // edd006c5
+ ZST1B [Z7.D], P4, (R21)(R7) // e75075e4
+ ZST1B [Z6.D], P4, (Z21.D.UXTW)(R7) // e69015e4
+ ZST1B [Z6.S], P4, (Z21.S.UXTW)(R7) // e69055e4
+ ZST1B [Z8.D], P3, (Z6.D)(RSP) // e8af06e4
+ ZST1D [Z8.D], P3, (R6<<3)(RSP) // e84fe6e5
+ ZST1D [Z8.Q], P3, (R6<<3)(RSP) // e84fc6e5
+ ZST1D [Z6.D], P4, (Z21.D.UXTW<<3)(R7) // e690b5e5
+ ZST1D [Z6.D], P4, (Z21.D.UXTW)(R7) // e69095e5
+ ZST1D [Z8.D], P3, (Z6.D<<3)(RSP) // e8afa6e5
+ ZST1D [Z8.D], P3, (Z6.D)(RSP) // e8af86e5
+ ZST1H [Z7.D], P4, (R21<<1)(R7) // e750f5e4
+ ZST1H [Z6.S], P4, (Z21.S.UXTW<<1)(R7) // e690f5e4
+ ZST1H [Z6.D], P4, (Z21.D.UXTW<<1)(R7) // e690b5e4
+ ZST1H [Z6.D], P4, (Z21.D.UXTW)(R7) // e69095e4
+ ZST1H [Z6.S], P4, (Z21.S.UXTW)(R7) // e690d5e4
+ ZST1H [Z8.D], P3, (Z6.D<<1)(RSP) // e8afa6e4
+ ZST1H [Z8.D], P3, (Z6.D)(RSP) // e8af86e4
+ ZST1Q [Z8.Q], P3, (R6)(Z15.D) // e82d26e4
+ ZST1W [Z7.D], P4, (R21<<2)(R7) // e75075e5
+ ZST1W [Z8.Q], P3, (R6<<2)(RSP) // e84f06e5
+ ZST1W [Z6.S], P4, (Z21.S.UXTW<<2)(R7) // e69075e5
+ ZST1W [Z6.D], P4, (Z21.D.UXTW<<2)(R7) // e69035e5
+ ZST1W [Z6.D], P4, (Z21.D.UXTW)(R7) // e69015e5
+ ZST1W [Z6.S], P4, (Z21.S.UXTW)(R7) // e69055e5
+ ZST1W [Z8.D], P3, (Z6.D<<2)(RSP) // e8af26e5
+ ZST1W [Z8.D], P3, (Z6.D)(RSP) // e8af06e5
+ ZST2B [Z8.B, Z9.B], P3, (R6)(RSP) // e86f26e4
+ ZST2D [Z8.D, Z9.D], P3, (R6<<3)(RSP) // e86fa6e5
+ ZST2H [Z8.H, Z9.H], P3, (R6<<1)(RSP) // e86fa6e4
+ ZST2Q [Z8.Q, Z9.Q], P3, (R6<<4)(RSP) // e80f66e4
+ ZST2W [Z8.S, Z9.S], P3, (R6<<2)(RSP) // e86f26e5
+ ZST3B [Z8.B, Z9.B, Z10.B], P3, (R6)(RSP) // e86f46e4
+ ZST3D [Z8.D, Z9.D, Z10.D], P3, (R6<<3)(RSP) // e86fc6e5
+ ZST3H [Z8.H, Z9.H, Z10.H], P3, (R6<<1)(RSP) // e86fc6e4
+ ZST3Q [Z8.Q, Z9.Q, Z10.Q], P3, (R6<<4)(RSP) // e80fa6e4
+ ZST3W [Z8.S, Z9.S, Z10.S], P3, (R6<<2)(RSP) // e86f46e5
+ ZST4B [Z8.B, Z9.B, Z10.B, Z11.B], P3, (R6)(RSP) // e86f66e4
+ ZST4D [Z8.D, Z9.D, Z10.D, Z11.D], P3, (R6<<3)(RSP)// e86fe6e5
+ ZST4H [Z8.H, Z9.H, Z10.H, Z11.H], P3, (R6<<1)(RSP)// e86fe6e4
+ ZST4Q [Z8.Q, Z9.Q, Z10.Q, Z11.Q], P3, (R6<<4)(RSP)// e80fe6e4
+ ZST4W [Z8.S, Z9.S, Z10.S, Z11.S], P3, (R6<<2)(RSP)// e86f66e5
+ ZSTNT1B [Z8.B], P3, (R6)(RSP) // e86f06e4
+ ZSTNT1B [Z8.S], P3, (R6)(Z15.S) // e82d46e4
+ ZSTNT1B [Z8.D], P3, (R6)(Z15.D) // e82d06e4
+ ZSTNT1D [Z8.D], P3, (R6<<3)(RSP) // e86f86e5
+ ZSTNT1D [Z8.D], P3, (R6)(Z15.D) // e82d86e5
+ ZSTNT1H [Z8.H], P3, (R6<<1)(RSP) // e86f86e4
+ ZSTNT1H [Z8.S], P3, (R6)(Z15.S) // e82dc6e4
+ ZSTNT1H [Z8.D], P3, (R6)(Z15.D) // e82d86e4
+ ZSTNT1W [Z8.S], P3, (R6<<2)(RSP) // e86f06e5
+ ZSTNT1W [Z8.S], P3, (R6)(Z15.S) // e82d46e5
+ ZSTNT1W [Z8.D], P3, (R6)(Z15.D) // e82d06e5
+ PCNTP VLx2, PN13.D, R5 // a583e025
+ PPRFB (R8)(RSP), P3, PSTL3KEEP // eccf0884
+ PPRFD (R8<<3)(RSP), P3, PSTL3KEEP // eccf8885
+ PPRFH (R8<<1)(RSP), P3, PSTL3KEEP // eccf8884
+ PPRFW (R8<<2)(RSP), P3, PSTL3KEEP // eccf0885
+ PWHILEGE VLx4, R21, R9, PN12.H // 34617525
+ PWHILEGT VLx4, R21, R9, PN12.H // 3c617525
+ PWHILEHI VLx4, R21, R9, PN12.H // 3c697525
+ PWHILEHS VLx4, R21, R9, PN12.H // 34697525
+ PWHILELE VLx4, R21, R9, PN12.H // 3c657525
+ PWHILELO VLx4, R21, R9, PN12.H // 346d7525
+ PWHILELS VLx4, R21, R9, PN12.H // 3c6d7525
+ PWHILELT VLx4, R21, R9, PN12.H // 34657525
+ ZPRFB (Z6.S.SXTW)(R14), P2, PLDL1STRM // c1096684
+ ZPRFB (Z6.D.SXTW)(R14), P2, PLDL1STRM // c10966c4
+ ZPRFB (Z8.D)(RSP), P3, PSTL3KEEP // ec8f68c4
+ ZPRFD (Z6.S.SXTW<<3)(R14), P2, PLDL1STRM // c1696684
+ ZPRFD (Z6.D.SXTW<<3)(R14), P2, PLDL1STRM // c16966c4
+ ZPRFD (Z8.D<<3)(RSP), P3, PSTL3KEEP // ecef68c4
+ ZPRFH (Z6.S.SXTW<<1)(R14), P2, PLDL1STRM // c1296684
+ ZPRFH (Z6.D.SXTW<<1)(R14), P2, PLDL1STRM // c12966c4
+ ZPRFH (Z8.D<<1)(RSP), P3, PSTL3KEEP // ecaf68c4
+ ZPRFW (Z6.S.SXTW<<2)(R14), P2, PLDL1STRM // c1496684
+ ZPRFW (Z6.D.SXTW<<2)(R14), P2, PLDL1STRM // c14966c4
+ ZPRFW (Z8.D<<2)(RSP), P3, PSTL3KEEP // eccf68c4
+ ZLD1B 6(Z7.S), P4.Z, [Z13.S] // edd02684
+ ZLD1B 6(Z7.D), P4.Z, [Z13.D] // edd026c4
+ ZLD1D 8(Z14.D), P6.Z, [Z6.D] // c6d9a1c5
+ ZLD1H 6(Z7.S), P4.Z, [Z13.S] // edd0a384
+ ZLD1H 6(Z7.D), P4.Z, [Z13.D] // edd0a3c4
+ ZLD1RB 6(R14), P4.Z, [Z13.B] // cd914684
+ ZLD1RB 6(R14), P4.Z, [Z13.H] // cdb14684
+ ZLD1RB 6(R14), P4.Z, [Z13.S] // cdd14684
+ ZLD1RB 6(R14), P4.Z, [Z13.D] // cdf14684
+ ZLD1RD 8(R29), P6.Z, [Z6.D] // a6fbc185
+ ZLD1RH 6(R14), P4.Z, [Z13.H] // cdb1c384
+ ZLD1RH 6(R14), P4.Z, [Z13.S] // cdd1c384
+ ZLD1RH 6(R14), P4.Z, [Z13.D] // cdf1c384
+ ZLD1ROB (R5), P2.Z, [Z23.B] // b72820a4
+ ZLD1ROD (R5), P2.Z, [Z23.D] // b728a0a5
+ ZLD1ROH (R5), P2.Z, [Z23.H] // b728a0a4
+ ZLD1ROW (R5), P2.Z, [Z23.S] // b72820a5
+ ZLD1RQB (R5), P2.Z, [Z23.B] // b72800a4
+ ZLD1RQD (R5), P2.Z, [Z23.D] // b72880a5
+ ZLD1RQH (R5), P2.Z, [Z23.H] // b72880a4
+ ZLD1RQW (R5), P2.Z, [Z23.S] // b72800a5
+ ZLD1RSB 6(R14), P4.Z, [Z13.H] // cdd1c685
+ ZLD1RSB 6(R14), P4.Z, [Z13.S] // cdb1c685
+ ZLD1RSB 6(R14), P4.Z, [Z13.D] // cd91c685
+ ZLD1RSH 6(R14), P4.Z, [Z13.S] // cdb14385
+ ZLD1RSH 6(R14), P4.Z, [Z13.D] // cd914385
+ ZLD1RSW 12(R4), P6.Z, [Z25.D] // 9998c384
+ ZLD1RW 12(R4), P6.Z, [Z25.S] // 99d84385
+ ZLD1RW 12(R4), P6.Z, [Z25.D] // 99f84385
+ ZLD1SB 6(Z7.S), P4.Z, [Z13.S] // ed902684
+ ZLD1SB 6(Z7.D), P4.Z, [Z13.D] // ed9026c4
+ ZLD1SH 6(Z7.S), P4.Z, [Z13.S] // ed90a384
+ ZLD1SH 6(Z7.D), P4.Z, [Z13.D] // ed90a3c4
+ ZLD1SW 12(Z0.D), P6.Z, [Z25.D] // 199823c5
+ ZLD1W 12(Z0.S), P6.Z, [Z25.S] // 19d82385
+ ZLD1W 12(Z0.D), P6.Z, [Z25.D] // 19d823c5
+ ZLDFF1B 6(Z7.S), P4.Z, [Z13.S] // edf02684
+ ZLDFF1B 6(Z7.D), P4.Z, [Z13.D] // edf026c4
+ ZLDFF1D 8(Z14.D), P6.Z, [Z6.D] // c6f9a1c5
+ ZLDFF1H 6(Z7.S), P4.Z, [Z13.S] // edf0a384
+ ZLDFF1H 6(Z7.D), P4.Z, [Z13.D] // edf0a3c4
+ ZLDFF1SB 6(Z7.S), P4.Z, [Z13.S] // edb02684
+ ZLDFF1SB 6(Z7.D), P4.Z, [Z13.D] // edb026c4
+ ZLDFF1SH 6(Z7.S), P4.Z, [Z13.S] // edb0a384
+ ZLDFF1SH 6(Z7.D), P4.Z, [Z13.D] // edb0a3c4
+ ZLDFF1SW 12(Z0.D), P6.Z, [Z25.D] // 19b823c5
+ ZLDFF1W 12(Z0.S), P6.Z, [Z25.S] // 19f82385
+ ZLDFF1W 12(Z0.D), P6.Z, [Z25.D] // 19f823c5
+ ZPRFB 8(Z10.S), P3, PSTL3KEEP // 4ced0884
+ ZPRFB 8(Z10.D), P3, PSTL3KEEP // 4ced08c4
+ ZPRFD 8(Z10.S), P3, PSTL3KEEP // 4ced8185
+ ZPRFD 8(Z10.D), P3, PSTL3KEEP // 4ced81c5
+ ZPRFH 8(Z10.S), P3, PSTL3KEEP // 4ced8484
+ ZPRFH 8(Z10.D), P3, PSTL3KEEP // 4ced84c4
+ ZPRFW 8(Z10.S), P3, PSTL3KEEP // 4ced0285
+ ZPRFW 8(Z10.D), P3, PSTL3KEEP // 4ced02c5
+ ZST1B [Z8.S], P3, 6(Z15.S) // e8ad66e4
+ ZST1B [Z8.D], P3, 6(Z15.D) // e8ad46e4
+ ZST1D [Z16.D], P2, (Z5.D) // b0a8c0e5
+ ZST1H [Z8.S], P3, 6(Z15.S) // e8ade3e4
+ ZST1H [Z8.D], P3, 6(Z15.D) // e8adc3e4
+ ZST1W [Z16.S], P2, (Z5.S) // b0a860e5
+ ZST1W [Z16.D], P2, (Z5.D) // b0a840e5
+ PLDR (-VL*2)(RSP), P5 // e51bbf85
+ PPRFB (VL*1)(RSP), P3, PSTL3KEEP // ec0fc185
+ PPRFD (VL*1)(RSP), P3, PSTL3KEEP // ec6fc185
+ PPRFH (VL*1)(RSP), P3, PSTL3KEEP // ec2fc185
+ PPRFW (VL*1)(RSP), P3, PSTL3KEEP // ec4fc185
+ PSTR P14, (-VL*1)(R0) // 0e1cbfe5
+ ZLD1B (-VL*2)(R14), P4.Z, [Z13.B] // cdb10ea4
+ ZLD1B (-VL*2)(R14), P4.Z, [Z13.H] // cdb12ea4
+ ZLD1B (-VL*2)(R14), P4.Z, [Z13.S] // cdb14ea4
+ ZLD1B (-VL*2)(R14), P4.Z, [Z13.D] // cdb16ea4
+ ZLD1D (-VL*2)(R14), P4.Z, [Z13.D] // cdb1eea5
+ ZLD1D (-VL*2)(R14), P4.Z, [Z13.Q] // cd319ea5
+ ZLD1H (-VL*2)(R14), P4.Z, [Z13.H] // cdb1aea4
+ ZLD1H (-VL*2)(R14), P4.Z, [Z13.S] // cdb1cea4
+ ZLD1H (-VL*2)(R14), P4.Z, [Z13.D] // cdb1eea4
+ ZLD1SB (-VL*2)(R14), P4.Z, [Z13.H] // cdb1cea5
+ ZLD1SB (-VL*2)(R14), P4.Z, [Z13.S] // cdb1aea5
+ ZLD1SB (-VL*2)(R14), P4.Z, [Z13.D] // cdb18ea5
+ ZLD1SH (-VL*2)(R14), P4.Z, [Z13.S] // cdb12ea5
+ ZLD1SH (-VL*2)(R14), P4.Z, [Z13.D] // cdb10ea5
+ ZLD1SW (-VL*2)(R14), P4.Z, [Z13.D] // cdb18ea4
+ ZLD1W (-VL*2)(R14), P4.Z, [Z13.S] // cdb14ea5
+ ZLD1W (-VL*2)(R14), P4.Z, [Z13.D] // cdb16ea5
+ ZLD1W (-VL*2)(R14), P4.Z, [Z13.Q] // cd311ea5
+ ZLD2B (-VL*2)(R14), P4.Z, [Z13.B, Z14.B] // cdf12fa4
+ ZLD2D (-VL*2)(R14), P4.Z, [Z13.D, Z14.D] // cdf1afa5
+ ZLD2H (-VL*2)(R14), P4.Z, [Z13.H, Z14.H] // cdf1afa4
+ ZLD2Q (-VL*2)(R14), P4.Z, [Z13.Q, Z14.Q] // cdf19fa4
+ ZLD2W (-VL*2)(R14), P4.Z, [Z13.S, Z14.S] // cdf12fa5
+ ZLD3B (-VL*3)(R10), P4.Z, [Z21.B, Z22.B, Z23.B] // 55f14fa4
+ ZLD3D (-VL*3)(R10), P4.Z, [Z21.D, Z22.D, Z23.D] // 55f1cfa5
+ ZLD3H (-VL*3)(R10), P4.Z, [Z21.H, Z22.H, Z23.H] // 55f1cfa4
+ ZLD3Q (-VL*3)(R10), P4.Z, [Z21.Q, Z22.Q, Z23.Q] // 55f11fa5
+ ZLD3W (-VL*3)(R10), P4.Z, [Z21.S, Z22.S, Z23.S] // 55f14fa5
+ ZLD4B (VL*4)(R4), P6.Z, [Z25.B, Z26.B, Z27.B, Z28.B]// 99f861a4
+ ZLD4D (VL*4)(R4), P6.Z, [Z25.D, Z26.D, Z27.D, Z28.D]// 99f8e1a5
+ ZLD4H (VL*4)(R4), P6.Z, [Z25.H, Z26.H, Z27.H, Z28.H]// 99f8e1a4
+ ZLD4Q (VL*4)(R4), P6.Z, [Z25.Q, Z26.Q, Z27.Q, Z28.Q]// 99f891a5
+ ZLD4W (VL*4)(R4), P6.Z, [Z25.S, Z26.S, Z27.S, Z28.S]// 99f861a5
+ ZLDNF1B (-VL*2)(R14), P4.Z, [Z13.B] // cdb11ea4
+ ZLDNF1B (-VL*2)(R14), P4.Z, [Z13.H] // cdb13ea4
+ ZLDNF1B (-VL*2)(R14), P4.Z, [Z13.S] // cdb15ea4
+ ZLDNF1B (-VL*2)(R14), P4.Z, [Z13.D] // cdb17ea4
+ ZLDNF1D (-VL*2)(R14), P4.Z, [Z13.D] // cdb1fea5
+ ZLDNF1H (-VL*2)(R14), P4.Z, [Z13.H] // cdb1bea4
+ ZLDNF1H (-VL*2)(R14), P4.Z, [Z13.S] // cdb1dea4
+ ZLDNF1H (-VL*2)(R14), P4.Z, [Z13.D] // cdb1fea4
+ ZLDNF1SB (-VL*2)(R14), P4.Z, [Z13.H] // cdb1dea5
+ ZLDNF1SB (-VL*2)(R14), P4.Z, [Z13.S] // cdb1bea5
+ ZLDNF1SB (-VL*2)(R14), P4.Z, [Z13.D] // cdb19ea5
+ ZLDNF1SH (-VL*2)(R14), P4.Z, [Z13.S] // cdb13ea5
+ ZLDNF1SH (-VL*2)(R14), P4.Z, [Z13.D] // cdb11ea5
+ ZLDNF1SW (-VL*2)(R14), P4.Z, [Z13.D] // cdb19ea4
+ ZLDNF1W (-VL*2)(R14), P4.Z, [Z13.S] // cdb15ea5
+ ZLDNF1W (-VL*2)(R14), P4.Z, [Z13.D] // cdb17ea5
+ ZLDNT1B (-VL*2)(R14), P4.Z, [Z13.B] // cdf10ea4
+ ZLDNT1D (-VL*2)(R14), P4.Z, [Z13.D] // cdf18ea5
+ ZLDNT1H (-VL*2)(R14), P4.Z, [Z13.H] // cdf18ea4
+ ZLDNT1W (-VL*2)(R14), P4.Z, [Z13.S] // cdf10ea5
+ ZLDR (-VL*2)(RSP), Z7 // e75bbf85
+ ZST1B [Z7.D], P4, (-VL*3)(R7) // e7f06de4
+ ZST1D [Z8.D], P3, (-VL*2)(RSP) // e8efeee5
+ ZST1D [Z8.Q], P3, (-VL*2)(RSP) // e8efcee5
+ ZST1H [Z7.D], P4, (-VL*3)(R7) // e7f0ede4
+ ZST1W [Z7.D], P4, (-VL*3)(R7) // e7f06de5
+ ZST1W [Z8.Q], P3, (-VL*2)(RSP) // e8ef0ee5
+ ZST2B [Z8.B, Z9.B], P3, (-VL*2)(RSP) // e8ef3fe4
+ ZST2D [Z8.D, Z9.D], P3, (-VL*2)(RSP) // e8efbfe5
+ ZST2H [Z8.H, Z9.H], P3, (-VL*2)(RSP) // e8efbfe4
+ ZST2Q [Z8.Q, Z9.Q], P3, (-VL*2)(RSP) // e80f4fe4
+ ZST2W [Z8.S, Z9.S], P3, (-VL*2)(RSP) // e8ef3fe5
+ ZST3B [Z2.B, Z3.B, Z4.B], P1, (-VL*3)(R12) // 82e55fe4
+ ZST3D [Z2.D, Z3.D, Z4.D], P1, (-VL*3)(R12) // 82e5dfe5
+ ZST3H [Z2.H, Z3.H, Z4.H], P1, (-VL*3)(R12) // 82e5dfe4
+ ZST3Q [Z2.Q, Z3.Q, Z4.Q], P1, (-VL*3)(R12) // 82058fe4
+ ZST3W [Z2.S, Z3.S, Z4.S], P1, (-VL*3)(R12) // 82e55fe5
+ ZST4B [Z16.B, Z17.B, Z18.B, Z19.B], P2, (-VL*8)(R9)// 30e97ee4
+ ZST4D [Z16.D, Z17.D, Z18.D, Z19.D], P2, (-VL*8)(R9)// 30e9fee5
+ ZST4H [Z16.H, Z17.H, Z18.H, Z19.H], P2, (-VL*8)(R9)// 30e9fee4
+ ZST4Q [Z16.Q, Z17.Q, Z18.Q, Z19.Q], P2, (-VL*8)(R9)// 3009cee4
+ ZST4W [Z16.S, Z17.S, Z18.S, Z19.S], P2, (-VL*8)(R9)// 30e97ee5
+ ZSTNT1B [Z8.B], P3, (-VL*2)(RSP) // e8ef1ee4
+ ZSTNT1D [Z8.D], P3, (-VL*2)(RSP) // e8ef9ee5
+ ZSTNT1H [Z8.H], P3, (-VL*2)(RSP) // e8ef9ee4
+ ZSTNT1W [Z8.S], P3, (-VL*2)(RSP) // e8ef1ee5
+ ZSTR Z11, (-VL*1)(R0) // 0b5cbfe5
+ ZAESD Z21.Q[1], [Z22.B-Z23.B], [Z22.B-Z23.B] // b6ee2a45
+ ZAESD Z10.Q[2], [Z8.B-Z11.B], [Z8.B-Z11.B] // 48ed3645
+ ZAESDIMC Z21.Q[1], [Z22.B-Z23.B], [Z22.B-Z23.B] // b6ee2b45
+ ZAESDIMC Z10.Q[2], [Z8.B-Z11.B], [Z8.B-Z11.B] // 48ed3745
+ ZAESE Z21.Q[1], [Z22.B-Z23.B], [Z22.B-Z23.B] // b6ea2a45
+ ZAESE Z10.Q[2], [Z8.B-Z11.B], [Z8.B-Z11.B] // 48e93645
+ ZAESEMC Z21.Q[1], [Z22.B-Z23.B], [Z22.B-Z23.B] // b6ea2b45
+ ZAESEMC Z10.Q[2], [Z8.B-Z11.B], [Z8.B-Z11.B] // 48e93745
+ ZBFCVTN [Z2.H-Z3.H], Z10.B // 4a380a65
+ ZFCVTN [Z2.H-Z3.H], Z10.B // 4a300a65
+ ZFCVTNB [Z2.S-Z3.S], Z10.B // 4a340a65
+ ZFCVTNT [Z2.S-Z3.S], Z10.B // 4a3c0a65
+ // TODO: FCVTZSN
+ // TODO: FCVTZUN
+ ZLD1B (-VL*6)(R3), PN10.Z, [Z24.B-Z25.B] // 78084da0
+ ZLD1B (VL*4)(RSP), PN10.Z, [Z16.B-Z19.B] // f08b41a0
+ ZLD1B (R5)(R27), PN12.Z, [Z6.B-Z7.B] // 661305a0
+ ZLD1B (R2)(R3), PN10.Z, [Z24.B-Z27.B] // 788802a0
+ ZLD1D (-VL*6)(R3), PN10.Z, [Z24.D-Z25.D] // 78684da0
+ ZLD1D (VL*4)(RSP), PN10.Z, [Z16.D-Z19.D] // f0eb41a0
+ ZLD1D (R5<<3)(R27), PN12.Z, [Z6.D-Z7.D] // 667305a0
+ ZLD1D (R2<<3)(R3), PN10.Z, [Z24.D-Z27.D] // 78e802a0
+ ZLD1H (-VL*6)(R3), PN10.Z, [Z24.H-Z25.H] // 78284da0
+ ZLD1H (VL*4)(RSP), PN10.Z, [Z16.H-Z19.H] // f0ab41a0
+ ZLD1H (R5<<1)(R27), PN12.Z, [Z6.H-Z7.H] // 663305a0
+ ZLD1H (R2<<1)(R3), PN10.Z, [Z24.H-Z27.H] // 78a802a0
+ ZLD1W (-VL*6)(R3), PN10.Z, [Z24.S-Z25.S] // 78484da0
+ ZLD1W (VL*4)(RSP), PN10.Z, [Z16.S-Z19.S] // f0cb41a0
+ ZLD1W (R5<<2)(R27), PN12.Z, [Z6.S-Z7.S] // 665305a0
+ ZLD1W (R2<<2)(R3), PN10.Z, [Z24.S-Z27.S] // 78c802a0
+ ZLDNT1B (-VL*6)(R3), PN10.Z, [Z24.B-Z25.B] // 79084da0
+ ZLDNT1B (VL*4)(RSP), PN10.Z, [Z16.B-Z19.B] // f18b41a0
+ ZLDNT1B (R5)(R27), PN12.Z, [Z6.B-Z7.B] // 671305a0
+ ZLDNT1B (R2)(R3), PN10.Z, [Z24.B-Z27.B] // 798802a0
+ ZLDNT1D (-VL*6)(R3), PN10.Z, [Z24.D-Z25.D] // 79684da0
+ ZLDNT1D (VL*4)(RSP), PN10.Z, [Z16.D-Z19.D] // f1eb41a0
+ ZLDNT1D (R5<<3)(R27), PN12.Z, [Z6.D-Z7.D] // 677305a0
+ ZLDNT1D (R2<<3)(R3), PN10.Z, [Z24.D-Z27.D] // 79e802a0
+ ZLDNT1H (-VL*6)(R3), PN10.Z, [Z24.H-Z25.H] // 79284da0
+ ZLDNT1H (VL*4)(RSP), PN10.Z, [Z16.H-Z19.H] // f1ab41a0
+ ZLDNT1H (R5<<1)(R27), PN12.Z, [Z6.H-Z7.H] // 673305a0
+ ZLDNT1H (R2<<1)(R3), PN10.Z, [Z24.H-Z27.H] // 79a802a0
+ ZLDNT1W (-VL*6)(R3), PN10.Z, [Z24.S-Z25.S] // 79484da0
+ ZLDNT1W (VL*4)(RSP), PN10.Z, [Z16.S-Z19.S] // f1cb41a0
+ ZLDNT1W (R5<<2)(R27), PN12.Z, [Z6.S-Z7.S] // 675305a0
+ ZLDNT1W (R2<<2)(R3), PN10.Z, [Z24.S-Z27.S] // 79c802a0
+ ZPMLAL Z13.D, Z21.D, [Z22.Q-Z23.Q] // b6fe2d45
+ ZPMULL Z13.D, Z21.D, [Z22.Q-Z23.Q] // b6fa2d45
+ ZSQCVTN [Z2.S-Z3.S], Z10.H // 4a403145
+ ZSQCVTUN [Z2.S-Z3.S], Z10.H // 4a503145
+ // TODO: SQRSHRN .B, { .H-.H }, #
+ ZSQRSHRN $6, [Z2.S-Z3.S], Z10.H // 4a28ba45
+ // TODO: SQRSHRUN .B, { .H-.H }, #
+ ZSQRSHRUN $6, [Z2.S-Z3.S], Z10.H // 4a08ba45
+ // TODO: SQSHRN ., { .-. }, #
+ // TODO: SQSHRUN ., { .-. }, #
+ ZST1B [Z14.B-Z15.B], PN12, (-VL*4)(R17) // 2e126ea0
+ ZST1B [Z4.B-Z7.B], PN12, (VL*4)(RSP) // e49361a0
+ ZST1B [Z14.B-Z15.B], PN12, (R20)(R17) // 2e1234a0
+ ZST1B [Z4.B-Z7.B], PN12, (R12)(RSP) // e4932ca0
+ ZST1D [Z14.D-Z15.D], PN12, (-VL*4)(R17) // 2e726ea0
+ ZST1D [Z4.D-Z7.D], PN12, (VL*4)(RSP) // e4f361a0
+ ZST1D [Z14.D-Z15.D], PN12, (R20<<3)(R17) // 2e7234a0
+ ZST1D [Z4.D-Z7.D], PN12, (R12<<3)(RSP) // e4f32ca0
+ ZST1H [Z14.H-Z15.H], PN12, (-VL*4)(R17) // 2e326ea0
+ ZST1H [Z4.H-Z7.H], PN12, (VL*4)(RSP) // e4b361a0
+ ZST1H [Z14.H-Z15.H], PN12, (R20<<1)(R17) // 2e3234a0
+ ZST1H [Z4.H-Z7.H], PN12, (R12<<1)(RSP) // e4b32ca0
+ ZST1W [Z14.S-Z15.S], PN12, (-VL*4)(R17) // 2e526ea0
+ ZST1W [Z4.S-Z7.S], PN12, (VL*4)(RSP) // e4d361a0
+ ZST1W [Z14.S-Z15.S], PN12, (R20<<2)(R17) // 2e5234a0
+ ZST1W [Z4.S-Z7.S], PN12, (R12<<2)(RSP) // e4d32ca0
+ ZSTNT1B [Z14.B-Z15.B], PN12, (-VL*4)(R17) // 2f126ea0
+ ZSTNT1B [Z4.B-Z7.B], PN12, (VL*4)(RSP) // e59361a0
+ ZSTNT1B [Z14.B-Z15.B], PN12, (R20)(R17) // 2f1234a0
+ ZSTNT1B [Z4.B-Z7.B], PN12, (R12)(RSP) // e5932ca0
+ ZSTNT1D [Z14.D-Z15.D], PN12, (-VL*4)(R17) // 2f726ea0
+ ZSTNT1D [Z4.D-Z7.D], PN12, (VL*4)(RSP) // e5f361a0
+ ZSTNT1D [Z14.D-Z15.D], PN12, (R20<<3)(R17) // 2f7234a0
+ ZSTNT1D [Z4.D-Z7.D], PN12, (R12<<3)(RSP) // e5f32ca0
+ ZSTNT1H [Z14.H-Z15.H], PN12, (-VL*4)(R17) // 2f326ea0
+ ZSTNT1H [Z4.H-Z7.H], PN12, (VL*4)(RSP) // e5b361a0
+ ZSTNT1H [Z14.H-Z15.H], PN12, (R20<<1)(R17) // 2f3234a0
+ ZSTNT1H [Z4.H-Z7.H], PN12, (R12<<1)(RSP) // e5b32ca0
+ ZSTNT1W [Z14.S-Z15.S], PN12, (-VL*4)(R17) // 2f526ea0
+ ZSTNT1W [Z4.S-Z7.S], PN12, (VL*4)(RSP) // e5d361a0
+ ZSTNT1W [Z14.S-Z15.S], PN12, (R20<<2)(R17) // 2f5234a0
+ ZSTNT1W [Z4.S-Z7.S], PN12, (R12<<2)(RSP) // e5d32ca0
+ ZUQCVTN [Z2.S-Z3.S], Z10.H // 4a483145
+ // TODO: UQRSHRN .B, { .H-.H }, #
+ ZUQRSHRN $6, [Z2.S-Z3.S], Z10.H // 4a38ba45
+ // TODO: UQSHRN ., { .-. }, #
+ RET
diff --git a/src/cmd/asm/internal/asm/testdata/arm64sveerror.s b/src/cmd/asm/internal/asm/testdata/arm64sveerror.s
new file mode 100644
index 00000000000000..f381ff534b94b6
--- /dev/null
+++ b/src/cmd/asm/internal/asm/testdata/arm64sveerror.s
@@ -0,0 +1,1400 @@
+// Code generated by 'instgen -o=$GOROOT # from go install golang.org/x/arch/arm64/instgen@latest'. DO NOT EDIT.
+
+#include "../../../../../runtime/textflag.h"
+
+TEXT asmtest(SB),DUPOK|NOSPLIT,$-8
+ PAND P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PANDS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBIC P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBICS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKA P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PBRKAS P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PBRKB P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PBRKBS P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PBRKN P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKNS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKPA P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKPAS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKPB P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PBRKPBS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PEOR P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PEORS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PNAND P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PNANDS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PNOR P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PNORS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PORN P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PORNS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PORR P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PORRS P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PPFALSE P14.S // ERROR "illegal combination from SVE"
+ PPFIRST P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PPNEXT P14.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ PPTEST P14.S, P13.Z // ERROR "illegal combination from SVE"
+ PPTRUE PN5.D // ERROR "illegal combination from SVE"
+ PPUNPKHI P14.S, P13.S // ERROR "illegal combination from SVE"
+ PPUNPKLO P14.S, P13.S // ERROR "illegal combination from SVE"
+ PRDFFR P14.Z, P13.S // ERROR "illegal combination from SVE"
+ PRDFFR P14.S // ERROR "illegal combination from SVE"
+ PRDFFRS P14.Z, P13.S // ERROR "illegal combination from SVE"
+ PREV P14.B, P5.D // ERROR "illegal combination from SVE"
+ PSEL P14.S, P13.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ PTRN1 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ PTRN2 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ PUZP1 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ PUZP2 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ PWRFFR P14.S // ERROR "illegal combination from SVE"
+ PZIP1 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ PZIP2 P14.S, P13.S, P14.B // ERROR "illegal combination from SVE"
+ ZABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZADCLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZADCLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZADD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZADDHNB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZADDHNT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZADDP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZADDPT Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZADDPT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: ADDQP
+ ZADDQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ // TODO: ADDSUBP
+ ZAESD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZAESE Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZAESIMC Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZAESMC Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZAND Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZAND Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZANDQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZASR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZASR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZASR Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZASRR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBCAX Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZBDEP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBEXT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBF1CVT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZBF1CVTLT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZBF2CVT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZBF2CVTLT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZBFADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFADD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFCLAMP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZBFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZBFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZBFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZBFDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMAX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMAXNM Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMIN Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMINNM Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMLA Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLS Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMLSLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLSLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: BFMMLA .H, .H, .H
+ ZBFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMUL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFMUL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFSCALE Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBFSUB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBGRP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBIC Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZBIC Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBSL Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZBSL1N Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZBSL2N Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZCLASTA Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZCLASTB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZCLS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCLS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCLZ Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCLZ Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCMPEQ Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPEQ Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPGE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPGE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPGT Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPGT Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPHI Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPHI Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPHS Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPHS Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPLE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPLO Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPLS Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPLT Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPNE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCMPNE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZCNOT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCNOT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCOMPACT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZCOMPACT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZDECP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZEOR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZEOR Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZEOR3 Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZEORBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZEORQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZEORTB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZEXPAND Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZF1CVT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZF1CVTLT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZF2CVT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZF2CVTLT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZFABD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFACGE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFACGT Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFADD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFADDP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFADDQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZFAMAX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFAMIN Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFCLAMP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFCMEQ Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFCMGE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFCMGT Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFCMNE Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFCMUO Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTLT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTLT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTLT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTLT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTXNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTXNT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFCVTZU Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFDIV Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFDIVR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFEXPA Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZFLOGB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFLOGB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFMAD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMAX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMAXNM Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMAXNMP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMAXNMQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZFMAXP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMAXQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZFMIN Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMINNM Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMINNMP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMINNMQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZFMINP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMINQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZFMLA Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLBB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLTB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLTT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLS Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMLSLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLSLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: FMMLA .H, .H, .H
+ ZFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMSB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMUL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFMUL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMULX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFNMAD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFNMLA Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFNMLS Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFNMSB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFRECPE Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZFRECPS Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFRECPX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRECPX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT32X Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT32X Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT32Z Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT32Z Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT64X Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT64X Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT64Z Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINT64Z Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTA Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTA Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTI Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTI Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTM Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTM Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTN Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTN Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTP Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTP Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTZ Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRINTZ Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFRSQRTE Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZFRSQRTS Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFSCALE Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFSQRT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFSQRT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZFSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFSUB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZFTSMUL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFTSSEL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZHISTCNT Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZHISTSEG Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZINCP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZLSL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZLSL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZLSL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZLSLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZLSR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZLSR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZLSR Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZLSRR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMAD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMADPT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMATCH Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZMLA Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMLAPT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLS Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMOVPRFX Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZMOVPRFX Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZMSB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMUL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZMUL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZNBSL Z1.S, Z26.S, Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZNMATCH Z1.S, Z26.S, P14.M, P5.D // ERROR "illegal combination from SVE"
+ ZNOT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZNOT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZORQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZORR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZORR Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMOV Z1.S, P13.S // ERROR "illegal combination from SVE"
+ ZPMOV P14.S, Z26.S // ERROR "illegal combination from SVE"
+ ZPMUL Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMULLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMULLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMULLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMULLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZRADDHNB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZRADDHNT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZRAX1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZRBIT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZRBIT Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREV Z11.B, Z7.D // ERROR "illegal combination from SVE"
+ ZREVB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVD Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVD Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZREVW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZRSUBHNB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZRSUBHNT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSABA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: SABAL ., ., .
+ ZSABALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSABALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSABD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSABDLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSABDLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSADALP Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSADDLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSADDLBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSADDLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSADDWB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSADDWT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSBCLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSBCLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSCLAMP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: SCVTF ., .
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: SCVTFLT
+ ZSDIV Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSDIVR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ // TODO: SDOT .H, .B, .B
+ ZSDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSEL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSHADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSHSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSHSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSM4E Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSM4EKEY Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMAX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSMAXP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSMAXQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZSMIN Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSMINP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSMINQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZSMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULH Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSMULH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSPLICE Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSQABS Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSQADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQADD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDECP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZSQDMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQINCP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZSQNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSQNEG Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLAH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLSH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMULH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQRSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQSUB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSQXTNB Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSQXTNT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSQXTUNB Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSQXTUNT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSRHADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSRSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSRSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSSUBLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSSUBLBT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSSUBLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSSUBLTB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSSUBWB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSSUBWT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSUB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSUBHNB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSUBHNT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: SUBP ., /M, ., .
+ ZSUBPT Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSUBPT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSUNPKHI Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSUNPKLO Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZSUQADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZSXTB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSXTB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSXTH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSXTH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSXTW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZSXTW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZTBX Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZTBXQ Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZTRN1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZTRN1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZTRN2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZTRN2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUABA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: UABAL ., ., .
+ ZUABALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUABALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUABD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUABDLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUABDLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUADALP Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUADDLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUADDLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUADDWB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUADDWT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUCLAMP Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: UCVTF ., .
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUCVTF Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: UCVTFLT
+ ZUDIV Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUDIVR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ // TODO: UDOT .H, .B, .B
+ ZUDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUHADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUHSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUHSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMAX Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMAXP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMAXQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZUMIN Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMINP Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMINQV Z1.S, P13.Z, V11.D2 // ERROR "illegal combination from SVE"
+ ZUMLALB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLALT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULH Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUMULH Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUQADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQADD Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUQDECP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZUQINCP P14.B, Z7.D // ERROR "illegal combination from SVE"
+ ZUQRSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQRSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQSUB Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQSUB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUQSUBR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUQXTNB Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZUQXTNT Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZURECPE Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZURECPE Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZURHADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZURSHL Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZURSHLR Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZURSQRTE Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZURSQRTE Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUSDOT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSMMLA Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSQADD Z1.S, Z26.S, P14.M, Z7.D // ERROR "illegal combination from SVE"
+ ZUSUBLB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSUBLT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSUBWB Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSUBWT Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUUNPKHI Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZUUNPKLO Z1.S, Z26.S // ERROR "illegal combination from SVE"
+ ZUXTB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUXTB Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUXTH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUXTH Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUXTW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUXTW Z1.S, P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZUZP1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUZP1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUZP2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUZP2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUZPQ1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUZPQ2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIP1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIP1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIP2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIP2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIPQ1 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZZIPQ2 Z1.S, Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ CTERMEQ RSP, R27 // ERROR "illegal combination from SVE"
+ CTERMEQW RSP, R27 // ERROR "illegal combination from SVE"
+ CTERMNE RSP, R27 // ERROR "illegal combination from SVE"
+ CTERMNEW RSP, R27 // ERROR "illegal combination from SVE"
+ PCNTP P14.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ PDECP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PFIRSTP P14.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ PINCP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PLASTP P14.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ PSQDECP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PSQDECPW RSP, P7.Q, R25 // ERROR "illegal combination from SVE"
+ PSQINCP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PSQINCPW RSP, P7.Q, R25 // ERROR "illegal combination from SVE"
+ PUQDECP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PUQDECPW P7.Q, RSP // ERROR "illegal combination from SVE"
+ PUQINCP P7.Q, RSP // ERROR "illegal combination from SVE"
+ PUQINCPW P7.Q, RSP // ERROR "illegal combination from SVE"
+ PWHILEGE RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEGEW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEGT RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEGTW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEHI RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEHIW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEHS RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEHSW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELE RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELEW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELO RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELOW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELS RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELSW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELT RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILELTW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILERW RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ PWHILEWR RSP, R27, P13.S // ERROR "illegal combination from SVE"
+ ZANDVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZANDVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZANDVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZANDVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZCLASTA Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTAB Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTAD Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTAH Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTAS Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTAW Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTB Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTBB Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTBD Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTBH Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTBS Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCLASTBW Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZCPY RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPYB RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPYD RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPYH RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPYS RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPYW RSP, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZDUP RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZDUPW RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZEORVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZEORVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZEORVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZEORVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFADDAD Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZFADDAH Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZFADDAS Z1.S, ZR, P7.Z, RSP // ERROR "illegal combination from SVE"
+ ZFADDVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFADDVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFADDVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXNMVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXNMVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXNMVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMAXVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINNMVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINNMVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINNMVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZFMINVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZINDEX RSP, R27, Z26.S // ERROR "illegal combination from SVE"
+ ZINDEXW RSP, R27, Z26.S // ERROR "illegal combination from SVE"
+ ZINSR RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZINSRB RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZINSRD RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZINSRH RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZINSRS RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZINSRW RSP, Z27.Q // ERROR "illegal combination from SVE"
+ ZLASTA Z1.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ ZLASTAB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTAD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTAH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTAS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTAW Z1.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ ZLASTB Z1.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ ZLASTBB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTBD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTBH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTBS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZLASTBW Z1.S, P13.Z, R11 // ERROR "illegal combination from SVE"
+ ZORVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZORVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZORVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZORVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSADDVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMAXVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMAXVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMAXVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMAXVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMINVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMINVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMINVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZSMINVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUADDVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMAXVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMAXVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMAXVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMAXVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMINVB Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMINVD Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMINVH Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ ZUMINVS Z1.S, P13.Z, V11 // ERROR "illegal combination from SVE"
+ PPEXT PN6[3], P5.D // ERROR "illegal combination from SVE"
+ ZBFDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLA Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLALB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLALT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLS Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLSLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMLSLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZBFMUL Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZDUP Z7.D[2], Z23.B // ERROR "illegal combination from SVE"
+ ZDUPQ Z27.Q[1], Z25.Q // ERROR "illegal combination from SVE"
+ ZFDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLA Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLA Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLA Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALB Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLBB Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLBT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLTB Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALLTT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLALT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLS Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLS Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLS Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLSLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMLSLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMUL Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMUL Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZFMUL Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLA Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLA Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLA Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLS Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLS Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMLS Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMUL Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMUL Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZMUL Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZPMOV Z13[3], P1.Q // ERROR "illegal combination from SVE"
+ ZPMOV Z13[3], P1.Q // ERROR "illegal combination from SVE"
+ ZPMOV Z13[3], P1.Q // ERROR "illegal combination from SVE"
+ ZPMOV P14.B, Z6[3] // ERROR "illegal combination from SVE"
+ ZPMOV P14.B, Z6[3] // ERROR "illegal combination from SVE"
+ ZPMOV P14.B, Z6[3] // ERROR "illegal combination from SVE"
+ // TODO: SDOT .H, .B, .B[]
+ ZSDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLALB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLALB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLALT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLALT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMLSLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSMULLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLALT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMLSLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULH Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULH Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULH Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQDMULLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLAH Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLAH Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLAH Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLSH Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLSH Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMLSH Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMULH Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMULH Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSQRDMULH Z27.D[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZSUDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ // TODO: UDOT .H, .B, .B[]
+ ZUDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUDOT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLALB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLALB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLALT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLALT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMLSLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLB Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLB Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLT Z27.H[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUMULLT Z27.S[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ZUSDOT Z27.B[1], Z26.S, Z11.B // ERROR "illegal combination from SVE"
+ ADDPL $1, R27, ZR // ERROR "illegal combination from SVE"
+ ADDVL $1, R27, ZR // ERROR "illegal combination from SVE"
+ RDVL $11, RSP // ERROR "illegal combination from SVE"
+ ZADD $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZAND $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZASR $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZASR $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZASRD $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZCADD $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCDOT $1, Z26.B[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCDOT $1, Z26.H[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCDOT $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCMLA $1, Z26.H[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCMLA $1, Z26.S[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCMLA $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZCMPEQ $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPGE $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPGT $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPHI $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPHS $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPLE $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPLO $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPLS $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPLT $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCMPNE $1, Z27.Q, P7.Z, P0.H // ERROR "illegal combination from SVE"
+ ZCPY $1, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZCPY $1, P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZDUP $1, Z27.Q // ERROR "illegal combination from SVE"
+ ZDUPM $1, Z27.Q // ERROR "illegal combination from SVE"
+ ZEOR $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZEXT $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZEXTQ $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZFADD $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFCADD $1, Z27.Q, Z25.Q, P0.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFCMEQ $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCMGE $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCMGT $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCMLA $1, Z26.H[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZFCMLA $1, Z26.S[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZFCMLA $1, Z27.Q, Z25.Q, P0.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFCMLE $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCMLT $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCMNE $(0.0), Z1.S, P13.Z, P14.B // ERROR "illegal combination from SVE"
+ ZFCPY $(3.6), P7.M, Z25.Q // ERROR "illegal combination from SVE"
+ ZFDUP $(3.6), Z27.Q // ERROR "illegal combination from SVE"
+ ZFMAX $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFMAXNM $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFMIN $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFMINNM $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFMUL $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFSUB $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFSUBR $(3.6), Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZFTMAD $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZINDEX RSP, $11, Z26.S // ERROR "illegal combination from SVE"
+ ZINDEX $7, $90, Z21.Q // ERROR "illegal combination from SVE"
+ // TODO: INDEX
+ ZINDEXW RSP, $11, Z26.S // ERROR "illegal combination from SVE"
+ // TODO: INDEX
+ ZLSL $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZLSL $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZLSR $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZLSR $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZMUL $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZORR $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZRSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZRSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSLI $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSMAX $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSMIN $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQADD $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQCADD $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZSQRDCMLAH $1, Z26.H[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZSQRDCMLAH $1, Z26.S[3], Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZSQRDCMLAH $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ ZSQRSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQRSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQRSHRUNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQRSHRUNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQSHL $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZSQSHLU $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZSQSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQSHRUNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQSHRUNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSQSUB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSRI $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSRSHR $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZSRSRA $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSSHLLB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSSHLLT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSSRA $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSUB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZSUBR $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUMAX $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUMIN $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQADD $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQRSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQRSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQSHL $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZUQSHRNB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQSHRNT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUQSUB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZURSHR $1, Z27.Q, P7.Z, Z6.H // ERROR "illegal combination from SVE"
+ ZURSRA $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUSHLLB $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUSHLLT $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZUSRA $1, Z27.Q, Z25.Q // ERROR "illegal combination from SVE"
+ ZXAR $1, Z27.Q, Z25.Q, Z6.H // ERROR "illegal combination from SVE"
+ PPEXT PN6[3], [P5.D, P6.D] // ERROR "illegal combination from SVE"
+ PWHILEGE RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILEGT RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILEHI RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILEHS RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILELE RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILELO RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILELS RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ PWHILELT RSP, R27, [P13.S, P14.S] // ERROR "illegal combination from SVE"
+ ZEXT $1, [Z27.Q, Z28.Q], Z25.Q // ERROR "illegal combination from SVE"
+ ZLUTI2 Z27[1], [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ ZLUTI2 Z27[1], [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ ZLUTI4 Z27[1], [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ ZLUTI4 Z27[1], [Z26.S, Z27.S], Z11.B // ERROR "illegal combination from SVE"
+ ZLUTI4 Z27[1], [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ // TODO: LUTI6
+ // TODO: LUTI6
+ ZSPLICE [Z1.S, Z2.S], P13.Z, Z11.B // ERROR "illegal combination from SVE"
+ ZTBL Z1.S, [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ ZTBL Z1.S, [Z26.S, Z27.S], Z11.B // ERROR "illegal combination from SVE"
+ ZTBLQ Z1.S, [Z26.S], Z11.B // ERROR "illegal combination from SVE"
+ ZADR (Z26.S.SXTW<<1)(Z1.S), Z7.D // ERROR "illegal combination from SVE"
+ ZADR (Z27.D.SXTW<<3)(Z1.D), Z25.Q // ERROR "illegal combination from SVE"
+ ZADR (Z27.D.UXTW<<3)(Z1.D), Z25.Q // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1B (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1B (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (Z27.D.UXTW<<3)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1D (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1D (Z27.D<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.S.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.D.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.D<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1Q (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROD (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROW (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQD (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQW (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SB (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SB (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.S.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.D.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.D<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SW (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SW (Z27.D.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SW (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1SW (Z27.D<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SW (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.S.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.D.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.D<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD2B (R27)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2D (R27<<3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2H (R27<<1)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2Q (R27<<4)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2W (R27<<2)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD3B (R27)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3D (R27<<3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3H (R27<<1)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3Q (R27<<4)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3W (R27<<2)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD4B (R27)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4D (R27<<3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4H (R27<<1)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4Q (R27<<4)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4W (R27<<2)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLDFF1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1B (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1B (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1D (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1D (Z27.D.UXTW<<3)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1D (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1D (Z27.D<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1D (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.S.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.D.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.D<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SB (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.S.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.D.UXTW<<1)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.D<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SW (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SW (Z27.D.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SW (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1SW (Z27.D<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SW (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.S.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.D.UXTW<<2)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.D.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.S.UXTW)(RSP), P7.Z, [Z6.H] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.D<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W (Z27.D)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (R27)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (R27)(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (R27<<3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (R27<<1)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (R27)(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1SB (R27)(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1SB (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1SH (R27)(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1SH (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1SW (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (R27<<2)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (R27)(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (R27)(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, (Z6.D.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, (Z6.S.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, (Z6.D)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (Z6.D.SXTW<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (Z6.D.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (Z6.D<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (Z6.D)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.S.SXTW<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.D.SXTW<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.D.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.S.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.D<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (Z6.D)(RSP) // ERROR "illegal combination from SVE"
+ ZST1Q [Z1.S], P13.Z, (R6)(Z11.D) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.S.SXTW<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.D.SXTW<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.D.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.S.SXTW)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.D<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (Z6.D)(RSP) // ERROR "illegal combination from SVE"
+ ZST2B [Z1.S, Z2.S], P13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST2D [Z1.S, Z2.S], P13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST2H [Z1.S, Z2.S], P13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST2Q [Z1.S, Z2.S], P13.Z, (R6<<4)(RSP) // ERROR "illegal combination from SVE"
+ ZST2W [Z1.S, Z2.S], P13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3B [Z1.S, Z2.S, Z3.S], P13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST3D [Z1.S, Z2.S, Z3.S], P13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST3H [Z1.S, Z2.S, Z3.S], P13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST3Q [Z1.S, Z2.S, Z3.S], P13.Z, (R6<<4)(RSP) // ERROR "illegal combination from SVE"
+ ZST3W [Z1.S, Z2.S, Z3.S], P13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST4B [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST4D [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (R6<<3)(RSP)// ERROR "illegal combination from SVE"
+ ZST4H [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (R6<<1)(RSP)// ERROR "illegal combination from SVE"
+ ZST4Q [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (R6<<4)(RSP)// ERROR "illegal combination from SVE"
+ ZST4W [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (R6<<2)(RSP)// ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S], P13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S], P13.Z, (R6)(Z11.S) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S], P13.Z, (R6)(Z11.D) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S], P13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S], P13.Z, (R6)(Z11.D) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S], P13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S], P13.Z, (R6)(Z11.S) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S], P13.Z, (R6)(Z11.D) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S], P13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S], P13.Z, (R6)(Z11.S) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S], P13.Z, (R6)(Z11.D) // ERROR "illegal combination from SVE"
+ PCNTP VLx4, PN7.Q, R25 // ERROR "illegal combination from SVE"
+ PPRFB (R27)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFD (R27<<3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFH (R27<<1)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFW (R27<<2)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PWHILEGE VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILEGT VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILEHI VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILEHS VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILELE VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILELO VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILELS VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ PWHILELT VLx4, R27, ZR, PN7.Q // ERROR "illegal combination from SVE"
+ ZPRFB (Z27.S.UXTW)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFB (Z27.D.UXTW)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFB (Z27.D)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFD (Z27.S.UXTW<<3)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFD (Z27.D.UXTW<<3)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFD (Z27.D<<3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFH (Z27.S.UXTW<<1)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFH (Z27.D.UXTW<<1)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFH (Z27.D<<1)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFW (Z27.S.UXTW<<2)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFW (Z27.D.UXTW<<2)(RSP), P7.Z, PLDL1KEEP // ERROR "illegal combination from SVE"
+ ZPRFW (Z27.D<<2)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZLD1B 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RD 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROD 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1ROW 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQD 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RQW 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSB 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSH 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RSW 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RW 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1RW 11(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SW 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1B 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1D 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1H 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SB 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SH 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1SW 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W 11(Z1.S), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDFF1W 11(Z1.D), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZPRFB 11(Z1.S), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFB 11(Z1.D), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFD 11(Z1.S), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFD 11(Z1.D), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFH 11(Z1.S), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFH 11(Z1.D), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFW 11(Z1.S), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZPRFW 11(Z1.D), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, 6(Z11.S) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, 6(Z11.D) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, 6(Z11.D) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, 6(Z11.S) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, 6(Z11.D) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, 6(Z11.S) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, 6(Z11.D) // ERROR "illegal combination from SVE"
+ PLDR (VL*3)(RSP), P13.Z // ERROR "illegal combination from SVE"
+ PPRFB (VL*3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFD (VL*3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFH (VL*3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PPRFW (VL*3)(RSP), P13.Z, PSTL3STRM // ERROR "illegal combination from SVE"
+ PSTR P14.Z, (VL*1)(RSP) // ERROR "illegal combination from SVE"
+ ZLD1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1D (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SH (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1SW (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLD2B (VL*3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2D (VL*3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2H (VL*3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2Q (VL*3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD2W (VL*3)(RSP), P13.Z, [Z11.B, Z12.B] // ERROR "illegal combination from SVE"
+ ZLD3B (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3D (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3H (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3Q (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD3W (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B] // ERROR "illegal combination from SVE"
+ ZLD4B (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4D (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4H (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4Q (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLD4W (VL*3)(RSP), P13.Z, [Z11.B, Z12.B, Z13.B, Z14.B]// ERROR "illegal combination from SVE"
+ ZLDNF1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1D (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SB (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SH (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SH (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1SW (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNF1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (VL*3)(RSP), P13.Z, [Z11.B] // ERROR "illegal combination from SVE"
+ ZLDR (VL*3)(RSP), Z26.S // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST2B [Z1.S, Z2.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST2D [Z1.S, Z2.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST2H [Z1.S, Z2.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST2Q [Z1.S, Z2.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST2W [Z1.S, Z2.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3B [Z1.S, Z2.S, Z3.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3D [Z1.S, Z2.S, Z3.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3H [Z1.S, Z2.S, Z3.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3Q [Z1.S, Z2.S, Z3.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST3W [Z1.S, Z2.S, Z3.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST4B [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (-VL*2)(RSP)// ERROR "illegal combination from SVE"
+ ZST4D [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (-VL*2)(RSP)// ERROR "illegal combination from SVE"
+ ZST4H [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (-VL*2)(RSP)// ERROR "illegal combination from SVE"
+ ZST4Q [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (-VL*2)(RSP)// ERROR "illegal combination from SVE"
+ ZST4W [Z1.S, Z2.S, Z3.S, Z4.S], P13.Z, (-VL*2)(RSP)// ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S], P13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTR Z1.S, (VL*1)(RSP) // ERROR "illegal combination from SVE"
+ ZAESD Z27.Q[1], [Z26.S-Z27.S], [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZAESD Z27.Q[1], [Z26.S-Z29.S], [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZAESDIMC Z27.Q[1], [Z26.S-Z27.S], [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZAESDIMC Z27.Q[1], [Z26.S-Z29.S], [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZAESE Z27.Q[1], [Z26.S-Z27.S], [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZAESE Z27.Q[1], [Z26.S-Z29.S], [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZAESEMC Z27.Q[1], [Z26.S-Z27.S], [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZAESEMC Z27.Q[1], [Z26.S-Z29.S], [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZBFCVTN [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ ZFCVTN [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ ZFCVTNB [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ ZFCVTNT [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ // TODO: FCVTZSN
+ // TODO: FCVTZUN
+ ZLD1B (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1B (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1B (R27)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1D (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1D (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1D (R27<<3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1D (R27<<3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1H (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1H (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1H (R27<<1)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1H (R27<<1)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1W (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1W (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLD1W (R27<<2)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLD1W (R27<<2)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (R27)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1B (R27)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (R27<<3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1D (R27<<3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (R27<<1)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1H (R27<<1)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (VL*3)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (VL*3)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (R27<<2)(RSP), PN13.Z, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZLDNT1W (R27<<2)(RSP), PN13.Z, [Z11.B-Z14.B] // ERROR "illegal combination from SVE"
+ ZPMLAL Z1.S, Z26.S, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZPMULL Z1.S, Z26.S, [Z11.B-Z12.B] // ERROR "illegal combination from SVE"
+ ZSQCVTN [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ ZSQCVTUN [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ // TODO: SQRSHRN .B, { .H-.H }, #
+ ZSQRSHRN $1, [Z27.Q-Z28.Q], Z25.Q // ERROR "illegal combination from SVE"
+ // TODO: SQRSHRUN .B, { .H-.H }, #
+ ZSQRSHRUN $1, [Z27.Q-Z28.Q], Z25.Q // ERROR "illegal combination from SVE"
+ // TODO: SQSHRN ., { .-. }, #
+ // TODO: SQSHRUN ., { .-. }, #
+ ZST1B [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S-Z2.S], PN13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST1B [Z1.S-Z4.S], PN13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S-Z2.S], PN13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1D [Z1.S-Z4.S], PN13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S-Z2.S], PN13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1H [Z1.S-Z4.S], PN13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S-Z2.S], PN13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZST1W [Z1.S-Z4.S], PN13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S-Z2.S], PN13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1B [Z1.S-Z4.S], PN13.Z, (R6)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S-Z2.S], PN13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1D [Z1.S-Z4.S], PN13.Z, (R6<<3)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S-Z2.S], PN13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1H [Z1.S-Z4.S], PN13.Z, (R6<<1)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S-Z2.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S-Z4.S], PN13.Z, (-VL*2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S-Z2.S], PN13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZSTNT1W [Z1.S-Z4.S], PN13.Z, (R6<<2)(RSP) // ERROR "illegal combination from SVE"
+ ZUQCVTN [Z1.S-Z2.S], Z26.S // ERROR "illegal combination from SVE"
+ // TODO: UQRSHRN .B, { .H-.H }, #
+ ZUQRSHRN $1, [Z27.Q-Z28.Q], Z25.Q // ERROR "illegal combination from SVE"
+ // TODO: UQSHRN ., { .-. }, #
+ RET
diff --git a/src/cmd/asm/internal/asm/testdata/loong64enc1.s b/src/cmd/asm/internal/asm/testdata/loong64enc1.s
index fd86db7a4fc184..fa433d53e12134 100644
--- a/src/cmd/asm/internal/asm/testdata/loong64enc1.s
+++ b/src/cmd/asm/internal/asm/testdata/loong64enc1.s
@@ -33,13 +33,17 @@ lable2:
MOVV R4, R5 // 85001500
MOVBU R4, R5 // 85fc4303
SUB R4, R5, R6 // a6101100
+ SUBW R4, R5, R6 // a6101100
SUBV R4, R5, R6 // a6901100
ADD R4, R5, R6 // a6101000
+ ADDW R4, R5, R6 // a6101000
ADDV R4, R5, R6 // a6901000
AND R4, R5, R6 // a6901400
SUB R4, R5 // a5101100
+ SUBW R4, R5 // a5101100
SUBV R4, R5 // a5901100
ADD R4, R5 // a5101000
+ ADDW R4, R5 // a5101000
ADDV R4, R5 // a5901000
AND R4, R5 // a5901400
NEGW R4, R5 // 05101100
@@ -93,8 +97,6 @@ lable2:
MOVV R4, 1(R5) // a404c029
MOVB R4, 1(R5) // a4040029
MOVBU R4, 1(R5) // a4040029
- SC R4, 1(R5) // a4040021
- SCV R4, 1(R5) // a4040023
MOVW y+8(FP), R4 // 64408028
MOVWU y+8(FP), R4 // 6440802a
MOVV y+8(FP), R4 // 6440c028
@@ -105,8 +107,6 @@ lable2:
MOVV 1(R5), R4 // a404c028
MOVB 1(R5), R4 // a4040028
MOVBU 1(R5), R4 // a404002a
- LL 1(R5), R4 // a4040020
- LLV 1(R5), R4 // a4040022
MOVW $4(R4), R5 // 8510c002
MOVV $4(R4), R5 // 8510c002
MOVW $-1, R4 // 04fcff02
@@ -115,6 +115,8 @@ lable2:
MOVV $1, R4 // 04048003
ADD $-1, R4, R5 // 85fcbf02
ADD $-1, R4 // 84fcbf02
+ ADDW $-1, R4, R5 // 85fcbf02
+ ADDW $-1, R4 // 84fcbf02
ADDV $-1, R4, R5 // 85fcff02
ADDV $-1, R4 // 84fcff02
AND $1, R4, R5 // 85044003
@@ -165,6 +167,8 @@ lable2:
// mul
MUL R4, R5 // a5101c00
MUL R4, R5, R6 // a6101c00
+ MULW R4, R5 // a5101c00
+ MULW R4, R5, R6 // a6101c00
MULV R4, R5 // a5901d00
MULV R4, R5, R6 // a6901d00
MULVU R4, R5 // a5901d00
@@ -191,20 +195,26 @@ lable2:
MOVHU R4, 1(R5) // a4044029
MOVHU y+8(FP), R4 // 6440402a
MOVHU 1(R5), R4 // a404402a
- MULU R4, R5 // a5101c00
- MULU R4, R5, R6 // a6101c00
MULH R4, R5 // a5901c00
MULH R4, R5, R6 // a6901c00
MULHU R4, R5 // a5101d00
MULHU R4, R5, R6 // a6101d00
REM R4, R5 // a5902000
REM R4, R5, R6 // a6902000
+ REMW R4, R5 // a5902000
+ REMW R4, R5, R6 // a6902000
REMU R4, R5 // a5902100
REMU R4, R5, R6 // a6902100
+ REMWU R4, R5 // a5902100
+ REMWU R4, R5, R6 // a6902100
DIV R4, R5 // a5102000
DIV R4, R5, R6 // a6102000
+ DIVW R4, R5 // a5102000
+ DIVW R4, R5, R6 // a6102000
DIVU R4, R5 // a5102100
DIVU R4, R5, R6 // a6102100
+ DIVWU R4, R5 // a5102100
+ DIVWU R4, R5, R6 // a6102100
SRLV R4, R5 // a5101900
SRLV R4, R5, R6 // a6101900
SRLV $4, R4, R5 // 85104500
@@ -212,6 +222,12 @@ lable2:
SRLV $32, R4, R5 // 85804500
SRLV $32, R4 // 84804500
+ // MULW.D.W[U] instructions
+ MULWVW R4, R5 // a5101f00
+ MULWVW R4, R5, R6 // a6101f00
+ MULWVWU R4, R5 // a5901f00
+ MULWVWU R4, R5, R6 // a6901f00
+
MASKEQZ R4, R5, R6 // a6101300
MASKNEZ R4, R5, R6 // a6901300
@@ -261,22 +277,18 @@ lable2:
MOVV R4, FCC0 // 80d81401
// LDPTR.{W/D} and STPTR.{W/D} instructions
- MOVWP R5, -32768(R4) // 85008025
MOVWP R5, 32764(R4) // 85fc7f25
MOVWP R5, 32(R4) // 85200025
MOVWP R5, 4(R4) // 85040025
MOVWP R5, (R4) // 85000025
- MOVVP R5, -32768(R4) // 85008027
MOVVP R5, 32764(R4) // 85fc7f27
MOVVP R5, 32(R4) // 85200027
MOVVP R5, 4(R4) // 85040027
MOVVP R5, (R4) // 85000027
- MOVWP -32768(R5), R4 // a4008024
MOVWP 32764(R5), R4 // a4fc7f24
MOVWP 32(R5), R4 // a4200024
MOVWP 4(R5), R4 // a4040024
MOVWP (R5), R4 // a4000024
- MOVVP -32768(R5), R4 // a4008026
MOVVP 32764(R5), R4 // a4fc7f26
MOVVP 32(R5), R4 // a4200026
MOVVP 4(R5), R4 // a4040026
@@ -338,6 +350,18 @@ lable2:
AMMAXDBVU R14, (R13), R12 // acb97038
AMMINDBWU R14, (R13), R12 // ac397138
AMMINDBVU R14, (R13), R12 // acb97138
+ LL 4096(R5), R4 // a4001020
+ LLW 4096(R5), R4 // a4001020
+ LLV 4096(R5), R4 // a4001022
+ LLACQW (R5), R4 // a4805738
+ LLACQV (R5), R4 // a4885738
+ SC R4, 4096(R5) // a4001021
+ SCW R4, 4096(R5) // a4001021
+ SCV R4, 4096(R5) // a4001023
+ SCQ R4, R5, (R6) // c4145738
+ SCRELW R4, (R6) // c4845738
+ SCRELV R4, (R6) // c48c5738
+
FMADDF F2, F14, F9, F16 // 30391108
FMADDD F11, F20, F23, F12 // ecd22508
@@ -379,6 +403,10 @@ lable2:
FTINTVF F0, F1 // 01241b01
FTINTVD F0, F1 // 01281b01
+ // FRINT{F,D} instructions
+ FRINTF F0, F1 // 01441e01
+ FRINTD F0, F1 // 01481e01
+
FMAXAF F4, F5, F6 // a6900c01
FMAXAF F4, F5 // a5900c01
FMAXAD F4, F5, F6 // a6100d01
@@ -537,12 +565,18 @@ lable2:
XVMOVQ X28.V[3], X8 // 88ef0377
XVMOVQ X27.V[0], X9 // 69e30377
- //Move vector element to vector.
+ // Move vector element to vector.
VMOVQ V1.B[3], V9.B16 // 298cf772
VMOVQ V2.H[2], V8.H8 // 48c8f772
VMOVQ V3.W[1], V7.W4 // 67e4f772
VMOVQ V4.V[0], V6.V2 // 86f0f772
+ // Move vector register to vector register.
+ VMOVQ V1, V9 // 29002d73
+ VMOVQ V2, V8 // 48002d73
+ XVMOVQ X3, X7 // 67002d77
+ XVMOVQ X4, X6 // 86002d77
+
// Load data from memory and broadcast to each element of a vector register: VMOVQ offset(Rj), .
VMOVQ (R4), V0.B16 // 80008030
VMOVQ 1(R4), V0.B16 // 80048030
@@ -589,6 +623,42 @@ lable2:
XVSEQV $15, X2, X4 // 44bc8176
XVSEQV $-15, X2, X4 // 44c48176
+ // VSLTB{B,H,W,V}, XVSLTB{B,H,W,V} instruction
+ VSLTB V1, V2, V3 // 43040670
+ VSLTH V1, V2, V3 // 43840670
+ VSLTW V1, V2, V3 // 43040770
+ VSLTV V1, V2, V3 // 43840770
+ XVSLTB X1, X2, X3 // 43040674
+ XVSLTH X1, X2, X3 // 43840674
+ XVSLTW X1, X2, X3 // 43040774
+ XVSLTV X1, X2, X3 // 43840774
+ VSLTB $1, V2, V3 // 43048672
+ VSLTH $-16, V2, V3 // 43c08672
+ VSLTW $-16, V2, V3 // 43408772
+ VSLTV $-15, V2, V3 // 43c48772
+ XVSLTB $1, X2, X3 // 43048676
+ XVSLTH $-16, X2, X3 // 43c08676
+ XVSLTW $-16, X2, X3 // 43408776
+ XVSLTV $-16, X2, X3 // 43c08776
+
+ // VSLTB{B,H,W,V}U, XVSLTB{B,H,W,V}U instruction
+ VSLTBU V1, V2, V3 // 43040870
+ VSLTHU V1, V2, V3 // 43840870
+ VSLTWU V1, V2, V3 // 43040970
+ VSLTVU V1, V2, V3 // 43840970
+ XVSLTBU X1, X2, X3 // 43040874
+ XVSLTHU X1, X2, X3 // 43840874
+ XVSLTWU X1, X2, X3 // 43040974
+ XVSLTVU X1, X2, X3 // 43840974
+ VSLTBU $0, V2, V3 // 43008872
+ VSLTHU $31, V2, V3 // 43fc8872
+ VSLTWU $16, V2, V3 // 43408972
+ VSLTVU $1, V2, V3 // 43848972
+ XVSLTBU $0, X2, X3 // 43008876
+ XVSLTHU $31, X2, X3 // 43fc8876
+ XVSLTWU $8, X2, X3 // 43208976
+ XVSLTVU $0, X2, X3 // 43808976
+
// VPCNT{B,H,W,V}, XVPCNT{B,H,W,V} instruction
VPCNTB V1, V2 // 22209c72
VPCNTH V1, V2 // 22249c72
@@ -824,6 +894,7 @@ lable2:
XVSUBQ X3, X2, X1 // 418c2d75
// [X]VADD{B,H,W,V}U, [X]VSUB{B,H,W,V}U instructions
+ VADDBU $1, V2 // 42048a72
VADDBU $1, V2, V1 // 41048a72
VADDHU $2, V2, V1 // 41888a72
VADDWU $3, V2, V1 // 410c8b72
@@ -841,6 +912,42 @@ lable2:
XVSUBWU $15, X1, X2 // 223c8d76
XVSUBVU $16, X1, X2 // 22c08d76
+ // [X]VSADD{B,H,W,V}, [X]VSSUB{B,H,W,V} instructions
+ VSADDB V1, V2, V3 // 43044670
+ VSADDH V1, V2, V3 // 43844670
+ VSADDW V1, V2, V3 // 43044770
+ VSADDV V1, V2, V3 // 43844770
+ VSSUBB V1, V2, V3 // 43044870
+ VSSUBH V1, V2, V3 // 43844870
+ VSSUBW V1, V2, V3 // 43044970
+ VSSUBV V1, V2, V3 // 43844970
+ XVSADDB X3, X2, X1 // 410c4674
+ XVSADDH X3, X2, X1 // 418c4674
+ XVSADDW X3, X2, X1 // 410c4774
+ XVSADDV X3, X2, X1 // 418c4774
+ XVSSUBB X3, X2, X1 // 410c4874
+ XVSSUBH X3, X2, X1 // 418c4874
+ XVSSUBW X3, X2, X1 // 410c4974
+ XVSSUBV X3, X2, X1 // 418c4974
+
+ // [X]VSADD{B,H,W,V}U, [X]VSSUB{B,H,W,V}U instructions
+ VSADDBU V1, V2, V3 // 43044a70
+ VSADDHU V1, V2, V3 // 43844a70
+ VSADDWU V1, V2, V3 // 43044b70
+ VSADDVU V1, V2, V3 // 43844b70
+ VSSUBBU V1, V2, V3 // 43044c70
+ VSSUBHU V1, V2, V3 // 43844c70
+ VSSUBWU V1, V2, V3 // 43044d70
+ VSSUBVU V1, V2, V3 // 43844d70
+ XVSADDBU X1, X2, X3 // 43044a74
+ XVSADDHU X1, X2, X3 // 43844a74
+ XVSADDWU X1, X2, X3 // 43044b74
+ XVSADDVU X1, X2, X3 // 43844b74
+ XVSSUBBU X1, X2, X3 // 43044c74
+ XVSSUBHU X1, X2, X3 // 43844c74
+ XVSSUBWU X1, X2, X3 // 43044d74
+ XVSSUBVU X1, X2, X3 // 43844d74
+
// [X]VILV{L/H}{B,H,W,V} instructions
VILVLB V1, V2, V3 // 43041a71
VILVLH V1, V2, V3 // 43841a71
@@ -995,6 +1102,150 @@ lable2:
XVMULWODVWUW X1, X2, X3 // 4304a374
XVMULWODQVUV X1, X2, X3 // 4384a374
+ // [X]VADDW{EV/OD}.{H.B/W.H/D.W/Q.D} instructions
+ VADDWEVHB V1, V2, V3 // 43041e70
+ VADDWEVWH V1, V2, V3 // 43841e70
+ VADDWEVVW V1, V2, V3 // 43041f70
+ VADDWEVQV V1, V2, V3 // 43841f70
+ VADDWODHB V1, V2, V3 // 43042270
+ VADDWODWH V1, V2, V3 // 43842270
+ VADDWODVW V1, V2, V3 // 43042370
+ VADDWODQV V1, V2, V3 // 43842370
+ XVADDWEVHB X1, X2, X3 // 43041e74
+ XVADDWEVWH X1, X2, X3 // 43841e74
+ XVADDWEVVW X1, X2, X3 // 43041f74
+ XVADDWEVQV X1, X2, X3 // 43841f74
+ XVADDWODHB X1, X2, X3 // 43042274
+ XVADDWODWH X1, X2, X3 // 43842274
+ XVADDWODVW X1, X2, X3 // 43042374
+ XVADDWODQV X1, X2, X3 // 43842374
+
+ // [X]VSUBW{EV/OD}.{H.B/W.H/D.W/Q.D} instructions
+ VSUBWEVHB V1, V2, V3 // 43042070
+ VSUBWEVWH V1, V2, V3 // 43842070
+ VSUBWEVVW V1, V2, V3 // 43042170
+ VSUBWEVQV V1, V2, V3 // 43842170
+ VSUBWODHB V1, V2, V3 // 43042470
+ VSUBWODWH V1, V2, V3 // 43842470
+ VSUBWODVW V1, V2, V3 // 43042570
+ VSUBWODQV V1, V2, V3 // 43842570
+ XVSUBWEVHB X1, X2, X3 // 43042074
+ XVSUBWEVWH X1, X2, X3 // 43842074
+ XVSUBWEVVW X1, X2, X3 // 43042174
+ XVSUBWEVQV X1, X2, X3 // 43842174
+ XVSUBWODHB X1, X2, X3 // 43042474
+ XVSUBWODWH X1, X2, X3 // 43842474
+ XVSUBWODVW X1, X2, X3 // 43042574
+ XVSUBWODQV X1, X2, X3 // 43842574
+
+ // [X]VADDW{EV/OD}.{H.B/W.H/D.W/Q.D}U instructions
+ VADDWEVHBU V1, V2, V3 // 43042e70
+ VADDWEVWHU V1, V2, V3 // 43842e70
+ VADDWEVVWU V1, V2, V3 // 43042f70
+ VADDWEVQVU V1, V2, V3 // 43842f70
+ VADDWODHBU V1, V2, V3 // 43043270
+ VADDWODWHU V1, V2, V3 // 43843270
+ VADDWODVWU V1, V2, V3 // 43043370
+ VADDWODQVU V1, V2, V3 // 43843370
+ XVADDWEVHBU X1, X2, X3 // 43042e74
+ XVADDWEVWHU X1, X2, X3 // 43842e74
+ XVADDWEVVWU X1, X2, X3 // 43042f74
+ XVADDWEVQVU X1, X2, X3 // 43842f74
+ XVADDWODHBU X1, X2, X3 // 43043274
+ XVADDWODWHU X1, X2, X3 // 43843274
+ XVADDWODVWU X1, X2, X3 // 43043374
+ XVADDWODQVU X1, X2, X3 // 43843374
+
+ // [X]VSUBW{EV/OD}.{H.B/W.H/D.W/Q.D}U instructions
+ VSUBWEVHBU V1, V2, V3 // 43043070
+ VSUBWEVWHU V1, V2, V3 // 43843070
+ VSUBWEVVWU V1, V2, V3 // 43043170
+ VSUBWEVQVU V1, V2, V3 // 43843170
+ VSUBWODHBU V1, V2, V3 // 43043470
+ VSUBWODWHU V1, V2, V3 // 43843470
+ VSUBWODVWU V1, V2, V3 // 43043570
+ VSUBWODQVU V1, V2, V3 // 43843570
+ XVSUBWEVHBU X1, X2, X3 // 43043074
+ XVSUBWEVWHU X1, X2, X3 // 43843074
+ XVSUBWEVVWU X1, X2, X3 // 43043174
+ XVSUBWEVQVU X1, X2, X3 // 43843174
+ XVSUBWODHBU X1, X2, X3 // 43043474
+ XVSUBWODWHU X1, X2, X3 // 43843474
+ XVSUBWODVWU X1, X2, X3 // 43043574
+ XVSUBWODQVU X1, X2, X3 // 43843574
+
+ // [X]VMADD.{B/H/W/D}, [X]VMSUB.{B/H/W/D} instructions
+ VMADDB V1, V2, V3 // 4304a870
+ VMADDH V1, V2, V3 // 4384a870
+ VMADDW V1, V2, V3 // 4304a970
+ VMADDV V1, V2, V3 // 4384a970
+ VMSUBB V1, V2, V3 // 4304aa70
+ VMSUBH V1, V2, V3 // 4384aa70
+ VMSUBW V1, V2, V3 // 4304ab70
+ VMSUBV V1, V2, V3 // 4384ab70
+ XVMADDB X1, X2, X3 // 4304a874
+ XVMADDH X1, X2, X3 // 4384a874
+ XVMADDW X1, X2, X3 // 4304a974
+ XVMADDV X1, X2, X3 // 4384a974
+ XVMSUBB X1, X2, X3 // 4304aa74
+ XVMSUBH X1, X2, X3 // 4384aa74
+ XVMSUBW X1, X2, X3 // 4304ab74
+ XVMSUBV X1, X2, X3 // 4384ab74
+
+ // [X]VMADDW{EV/OD}.{H.B/W.H/D.W/Q.D} instructions
+ VMADDWEVHB V1, V2, V3 // 4304ac70
+ VMADDWEVWH V1, V2, V3 // 4384ac70
+ VMADDWEVVW V1, V2, V3 // 4304ad70
+ VMADDWEVQV V1, V2, V3 // 4384ad70
+ VMADDWODHB V1, V2, V3 // 4304ae70
+ VMADDWODWH V1, V2, V3 // 4384ae70
+ VMADDWODVW V1, V2, V3 // 4304af70
+ VMADDWODQV V1, V2, V3 // 4384af70
+ XVMADDWEVHB X1, X2, X3 // 4304ac74
+ XVMADDWEVWH X1, X2, X3 // 4384ac74
+ XVMADDWEVVW X1, X2, X3 // 4304ad74
+ XVMADDWEVQV X1, X2, X3 // 4384ad74
+ XVMADDWODHB X1, X2, X3 // 4304ae74
+ XVMADDWODWH X1, X2, X3 // 4384ae74
+ XVMADDWODVW X1, X2, X3 // 4304af74
+ XVMADDWODQV X1, X2, X3 // 4384af74
+
+ // [X]VMADDW{EV/OD}.{H.B/W.H/D.W/Q.D}U instructions
+ VMADDWEVHBU V1, V2, V3 // 4304b470
+ VMADDWEVWHU V1, V2, V3 // 4384b470
+ VMADDWEVVWU V1, V2, V3 // 4304b570
+ VMADDWEVQVU V1, V2, V3 // 4384b570
+ VMADDWODHBU V1, V2, V3 // 4304b670
+ VMADDWODWHU V1, V2, V3 // 4384b670
+ VMADDWODVWU V1, V2, V3 // 4304b770
+ VMADDWODQVU V1, V2, V3 // 4384b770
+ XVMADDWEVHBU X1, X2, X3 // 4304b474
+ XVMADDWEVWHU X1, X2, X3 // 4384b474
+ XVMADDWEVVWU X1, X2, X3 // 4304b574
+ XVMADDWEVQVU X1, X2, X3 // 4384b574
+ XVMADDWODHBU X1, X2, X3 // 4304b674
+ XVMADDWODWHU X1, X2, X3 // 4384b674
+ XVMADDWODVWU X1, X2, X3 // 4304b774
+ XVMADDWODQVU X1, X2, X3 // 4384b774
+
+ // [X]VMADDW{EV/OD}.{H.BU.B/W.HU.H/D.WU.W/Q.DU.D} instructions
+ VMADDWEVHBUB V1, V2, V3 // 4304bc70
+ VMADDWEVWHUH V1, V2, V3 // 4384bc70
+ VMADDWEVVWUW V1, V2, V3 // 4304bd70
+ VMADDWEVQVUV V1, V2, V3 // 4384bd70
+ VMADDWODHBUB V1, V2, V3 // 4304be70
+ VMADDWODWHUH V1, V2, V3 // 4384be70
+ VMADDWODVWUW V1, V2, V3 // 4304bf70
+ VMADDWODQVUV V1, V2, V3 // 4384bf70
+ XVMADDWEVHBUB X1, X2, X3 // 4304bc74
+ XVMADDWEVWHUH X1, X2, X3 // 4384bc74
+ XVMADDWEVVWUW X1, X2, X3 // 4304bd74
+ XVMADDWEVQVUV X1, X2, X3 // 4384bd74
+ XVMADDWODHBUB X1, X2, X3 // 4304be74
+ XVMADDWODWHUH X1, X2, X3 // 4384be74
+ XVMADDWODVWUW X1, X2, X3 // 4304bf74
+ XVMADDWODQVUV X1, X2, X3 // 4384bf74
+
// [X]VSHUF4I.{B/H/W/D} instructions
VSHUF4IB $0, V2, V1 // 41009073
VSHUF4IB $16, V2, V1 // 41409073
@@ -1021,6 +1272,32 @@ lable2:
XVSHUF4IV $8, X1, X2 // 22209c77
XVSHUF4IV $15, X1, X2 // 223c9c77
+ // [X]VSHUF.{B/H/W/V} instructions
+ VSHUFH V1, V2, V3 // 43847a71
+ VSHUFW V1, V2, V3 // 43047b71
+ VSHUFV V1, V2, V3 // 43847b71
+ XVSHUFH X1, X2, X3 // 43847a75
+ XVSHUFW X1, X2, X3 // 43047b75
+ XVSHUFV X1, X2, X3 // 43847b75
+ VSHUFB V1, V2, V3, V4 // 6488500d
+ XVSHUFB X1, X2, X3, X4 // 6488600d
+
+ // VPERMIW, XVPERMI{W,V,Q} instructions
+ VPERMIW $0x1B, V1, V2 // VPERMIW $27, V1, V2 // 226ce473
+ XVPERMIW $0x2B, X1, X2 // XVPERMIW $43, X1, X2 // 22ace477
+ XVPERMIV $0x3B, X1, X2 // XVPERMIV $59, X1, X2 // 22ece877
+ XVPERMIQ $0x4B, X1, X2 // XVPERMIQ $75, X1, X2 // 222ced77
+
+ // A{,X}VEXTRINS.{B,H,W,V} instructions
+ VEXTRINSB $0x18, V1, V2 // VEXTRINSB $24, V1, V2 // 22608c73
+ VEXTRINSH $0x27, V1, V2 // VEXTRINSH $39, V1, V2 // 229c8873
+ VEXTRINSW $0x36, V1, V2 // VEXTRINSW $54, V1, V2 // 22d88473
+ VEXTRINSV $0x45, V1, V2 // VEXTRINSV $69, V1, V2 // 22148173
+ XVEXTRINSB $0x54, X1, X2 // XVEXTRINSB $84, X1, X2 // 22508d77
+ XVEXTRINSH $0x63, X1, X2 // XVEXTRINSH $99, X1, X2 // 228c8977
+ XVEXTRINSW $0x72, X1, X2 // XVEXTRINSW $114, X1, X2 // 22c88577
+ XVEXTRINSV $0x81, X1, X2 // XVEXTRINSV $129, X1, X2 // 22048277
+
// [X]VSETEQZ.V, [X]VSETNEZ.V
VSETEQV V1, FCC0 // 20989c72
VSETNEV V1, FCC0 // 209c9c72
diff --git a/src/cmd/asm/internal/asm/testdata/loong64enc2.s b/src/cmd/asm/internal/asm/testdata/loong64enc2.s
index 91aed4e2c7007d..38f50d2bfc235f 100644
--- a/src/cmd/asm/internal/asm/testdata/loong64enc2.s
+++ b/src/cmd/asm/internal/asm/testdata/loong64enc2.s
@@ -21,6 +21,10 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
ADD $4096, R4, R5 // 3e00001485781000
ADD $65536, R4 // 1e02001484781000
ADD $4096, R4 // 3e00001484781000
+ ADDW $65536, R4, R5 // 1e02001485781000
+ ADDW $4096, R4, R5 // 3e00001485781000
+ ADDW $65536, R4 // 1e02001484781000
+ ADDW $4096, R4 // 3e00001484781000
ADDV $65536, R4, R5 // 1e02001485f81000
ADDV $4096, R4, R5 // 3e00001485f81000
ADDV $65536, R4 // 1e02001484f81000
@@ -37,10 +41,6 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
SGTU $4096, R4, R5 // 3e00001485f81200
SGTU $65536, R4 // 1e02001484f81200
SGTU $4096, R4 // 3e00001484f81200
- ADDU $65536, R4, R5 // 1e02001485781000
- ADDU $4096, R4, R5 // 3e00001485781000
- ADDU $65536, R4 // 1e02001484781000
- ADDU $4096, R4 // 3e00001484781000
ADDVU $65536, R4, R5 // 1e02001485f81000
ADDVU $4096, R4, R5 // 3e00001485f81000
ADDVU $65536, R4 // 1e02001484f81000
diff --git a/src/cmd/asm/internal/asm/testdata/loong64enc3.s b/src/cmd/asm/internal/asm/testdata/loong64enc3.s
index 2d83bd719a584e..8b5f96bf4a224a 100644
--- a/src/cmd/asm/internal/asm/testdata/loong64enc3.s
+++ b/src/cmd/asm/internal/asm/testdata/loong64enc3.s
@@ -11,12 +11,16 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
MOVV $4096(R4), R5 // 3e000014de03800385f81000
ADD $74565, R4 // 5e020014de178d0384781000
ADD $4097, R4 // 3e000014de07800384781000
+ ADDW $74565, R4 // 5e020014de178d0384781000
+ ADDW $4097, R4 // 3e000014de07800384781000
ADDV $74565, R4 // 5e020014de178d0384f81000
ADDV $4097, R4 // 3e000014de07800384f81000
AND $74565, R4 // 5e020014de178d0384f81400
AND $4097, R4 // 3e000014de07800384f81400
ADD $74565, R4, R5 // 5e020014de178d0385781000
ADD $4097, R4, R5 // 3e000014de07800385781000
+ ADDW $74565, R4, R5 // 5e020014de178d0385781000
+ ADDW $4097, R4, R5 // 3e000014de07800385781000
ADDV $74565, R4, R5 // 5e020014de178d0385f81000
ADDV $4097, R4, R5 // 3e000014de07800385f81000
AND $74565, R4, R5 // 5e020014de178d0385f81400
@@ -42,8 +46,10 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
MOVB R4, 4096(R5) // 3e000014de971000c4030029
MOVBU R4, 65536(R5) // 1e020014de971000c4030029
MOVBU R4, 4096(R5) // 3e000014de971000c4030029
- SC R4, 65536(R5) // 1e020014de971000c4030021
- SC R4, 4096(R5) // 3e000014de971000c4030021
+ SC R4, 65536(R5) // 1e040010de971000c4030021
+ SCV R4, 65536(R5) // 1e040010de971000c4030023
+ LL 65536(R5), R4 // 1e040010de971000c4030020
+ LLV 65536(R5), R4 // 1e040010de971000c4030022
MOVW y+65540(FP), R4 // 1e020014de8f1000c4338028
MOVWU y+65540(FP), R4 // 1e020014de8f1000c433802a
MOVV y+65540(FP), R4 // 1e020014de8f1000c433c028
@@ -105,10 +111,6 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
SGTU $74565, R4, R5 // 5e020014de178d0385f81200
SGTU $4097, R4 // 3e000014de07800384f81200
SGTU $4097, R4, R5 // 3e000014de07800385f81200
- ADDU $74565, R4 // 5e020014de178d0384781000
- ADDU $74565, R4, R5 // 5e020014de178d0385781000
- ADDU $4097, R4 // 3e000014de07800384781000
- ADDU $4097, R4, R5 // 3e000014de07800385781000
ADDVU $4097, R4 // 3e000014de07800384f81000
ADDVU $4097, R4, R5 // 3e000014de07800385f81000
ADDVU $74565, R4 // 5e020014de178d0384f81000
@@ -122,6 +124,21 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0
XOR $4097, R4 // 3e000014de07800384f81500
XOR $4097, R4, R5 // 3e000014de07800385f81500
+ MOVWP R5, -32768(R4) // 1efcff13de931000c5038025
+ MOVWP R5, 32768(R4) // 1e000010de931000c5038025
+ MOVWP R5, 65536(R4) // 1e040010de931000c5030025
+ MOVWP R5, 1048576(R4) // 1e400010de931000c5030025
+ MOVVP R5, -32768(R4) // 1efcff13de931000c5038027
+ MOVVP R5, 65536(R4) // 1e040010de931000c5030027
+ MOVVP R5, 1048576(R4) // 1e400010de931000c5030027
+ MOVWP -32768(R5), R4 // 1efcff13de971000c4038024
+ MOVWP 2229248(R5), R4 // 1e880010de971000c4030424
+ MOVWP -2145518592(R5), R4 // 1e740012de971000c403fc24
+ MOVVP -32768(R5), R4 // 1efcff13de971000c4038026
+ MOVVP 2229248(R5), R4 // 1e880010de971000c4030426
+ MOVVP -2145518592(R5), R4 // 1e740012de971000c403fc26
+
+
// MOVV C_DCON32_12S, r
MOVV $0x27312345fffff800, R4 // MOVV $2824077224892692480, R4 // 0400a002a468241684cc0903
MOVV $0xf7312345fffff800, R4 // MOVV $-634687288927848448, R4 // 0400a002a468241684cc3d03
diff --git a/src/cmd/asm/internal/asm/testdata/loong64enc6.s b/src/cmd/asm/internal/asm/testdata/loong64enc6.s
new file mode 100644
index 00000000000000..bd19ea76012a5a
--- /dev/null
+++ b/src/cmd/asm/internal/asm/testdata/loong64enc6.s
@@ -0,0 +1,12 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "../../../../../runtime/textflag.h"
+
+TEXT asmtest(SB),DUPOK|NOSPLIT,$0
+ // MOVWP LOREG_64(Rx), Ry
+ MOVWP 81985529216486896(R4), R5 // 9e571315dec3b703feac6816de4b000384f8100085000025
+ MOVWP -81985529216486896(R4), R5 // 7ea8ec14de4388031e539717deb73f0384f8100085000025
+ MOVWP R4, 81985529216486896(R5) // 9e571315dec3b703feac6816de4b0003a5f81000a4000025
+ MOVWP R4, -81985529216486896(R5) // 7ea8ec14de4388031e539717deb73f03a5f81000a4000025
diff --git a/src/cmd/asm/internal/asm/testdata/loong64error.s b/src/cmd/asm/internal/asm/testdata/loong64error.s
index 2dcd34bf61c676..25ef3a85a46738 100644
--- a/src/cmd/asm/internal/asm/testdata/loong64error.s
+++ b/src/cmd/asm/internal/asm/testdata/loong64error.s
@@ -7,3 +7,9 @@ TEXT errors(SB),$0
XVSHUF4IV $16, X1, X2 // ERROR "operand out of range 0 to 15"
ADDV16 $1, R4, R5 // ERROR "the constant must be a multiple of 65536."
ADDV16 $65535, R4, R5 // ERROR "the constant must be a multiple of 65536."
+ SC R4, 1(R5) // ERROR "offset must be a multiple of 4."
+ SCW R4, 1(R5) // ERROR "offset must be a multiple of 4."
+ SCV R4, 1(R5) // ERROR "offset must be a multiple of 4."
+ LL 1(R5), R4 // ERROR "offset must be a multiple of 4."
+ LLW 1(R5), R4 // ERROR "offset must be a multiple of 4."
+ LLV 1(R5), R4 // ERROR "offset must be a multiple of 4."
diff --git a/src/cmd/asm/internal/asm/testdata/riscv64.s b/src/cmd/asm/internal/asm/testdata/riscv64.s
index 702b82223b36af..752743892f6eaa 100644
--- a/src/cmd/asm/internal/asm/testdata/riscv64.s
+++ b/src/cmd/asm/internal/asm/testdata/riscv64.s
@@ -11,7 +11,6 @@ start:
//
// 2.4: Integer Computational Instructions
-
ADDI $2047, X5 // 9382f27f
ADDI $-2048, X5 // 93820280
ADDI $2048, X5 // 9382024093820240
@@ -140,6 +139,10 @@ start:
// 2.7: Memory Ordering Instructions
FENCE // 0f00f00f
+ FENCE W, W // 0f001001
+ FENCE I, O // 0f004008
+ FENCE IORW, IORW // 0f00f00f
+ FENCE.TSO // 0f003083
// 4.2: Integer Computational Instructions (RV64I)
ADDIW $1, X5, X6 // 1b831200
@@ -172,7 +175,17 @@ start:
SD X5, (X6) // 23305300
SD X5, 4(X6) // 23325300
- // 7.1: CSR Instructions
+ // 6.1: CSR Instructions
+ CSRC X5, VSTART // 73b08200
+ CSRC $2, VSTART // 73708100
+ CSRCI $2, VSTART // 73708100
+ CSRR CYCLE, X5 // f32200c0
+ CSRR VCSR, X5 // f322f000
+ CSRR VL, X5 // f32200c2
+ CSRR VLENB, X5 // f32220c2
+ CSRR VSTART, X5 // f3228000
+ CSRR VXSAT, X5 // f3229000
+ CSRR VXRM, X5 // f322a000
CSRRC X0, CYCLE, X5 // f33200c0
CSRRC X0, CYCLE, X0 // 733000c0
CSRRC X10, CYCLE, X5 // f33205c0
@@ -189,26 +202,35 @@ start:
CSRRW X10, CYCLE, X5 // f31205c0
CSRRW $2, TIME, X5 // f35211c0
CSRRWI $2, TIME, X5 // f35211c0
+ CSRS X10, VSTART // 73208500
+ CSRS $2, VSTART // 73608100
+ CSRSI $2, VSTART // 73608100
+ CSRW X10, VSTART // 73108500
+ CSRW $2, VSTART // 73508100
+ CSRWI $2, VSTART // 73508100
// 8.1: Base Counters and Timers (Zicntr)
RDCYCLE X5 // f32200c0
RDTIME X5 // f32210c0
RDINSTRET X5 // f32220c0
- // 12.3: Integer Conditional Operations (Zicond)
+ // 9: Zihintpause Extension for Pause Hint
+ PAUSE // 0f000001
+
+ // 11.1: Integer Conditional Operations (Zicond)
CZEROEQZ X5, X6, X7 // b353530e
CZEROEQZ X5, X7 // b3d3530e
CZERONEZ X5, X6, X7 // b373530e
CZERONEZ X5, X7 // b3f3530e
- // 13.1: Multiplication Operations
+ // 12.1: Multiplication Operations
MUL X5, X6, X7 // b3035302
MULH X5, X6, X7 // b3135302
MULHU X5, X6, X7 // b3335302
MULHSU X5, X6, X7 // b3235302
MULW X5, X6, X7 // bb035302
- // 13.2: Division Operations
+ // 12.2: Division Operations
DIV X5, X6, X7 // b3435302
DIVU X5, X6, X7 // b3535302
REM X5, X6, X7 // b3635302
@@ -218,13 +240,13 @@ start:
REMW X5, X6, X7 // bb635302
REMUW X5, X6, X7 // bb735302
- // 14.2: Load-Reserved/Store-Conditional (Zalrsc)
+ // 13.2: Load-Reserved/Store-Conditional (Zalrsc)
LRW (X5), X6 // 2fa30214
LRD (X5), X6 // 2fb30214
SCW X5, (X6), X7 // af23531a
SCD X5, (X6), X7 // af33531a
- // 14.4: Atomic Memory Operations (Zaamo)
+ // 13.4: Atomic Memory Operations (Zaamo)
AMOSWAPW X5, (X6), X7 // af23530e
AMOSWAPD X5, (X6), X7 // af33530e
AMOADDW X5, (X6), X7 // af235306
@@ -244,13 +266,13 @@ start:
AMOMINUW X5, (X6), X7 // af2353c6
AMOMINUD X5, (X6), X7 // af3353c6
- // 20.5: Single-Precision Load and Store Instructions
+ // 21.5: Single-Precision Load and Store Instructions
FLW (X5), F0 // 07a00200
FLW 4(X5), F0 // 07a04200
FSW F0, (X5) // 27a00200
FSW F0, 4(X5) // 27a20200
- // 20.6: Single-Precision Floating-Point Computational Instructions
+ // 21.6: Single-Precision Floating-Point Computational Instructions
FADDS F1, F0, F2 // 53011000
FSUBS F1, F0, F2 // 53011008
FMULS F1, F0, F2 // 53011010
@@ -259,7 +281,7 @@ start:
FMAXS F1, F0, F2 // 53111028
FSQRTS F0, F1 // d3000058
- // 20.7: Single-Precision Floating-Point Conversion and Move Instructions
+ // 21.7: Single-Precision Floating-Point Conversion and Move Instructions
FCVTWS F0, X5 // d31200c0
FCVTWS.RNE F0, X5 // d30200c0
FCVTWS.RTZ F0, X5 // d31200c0
@@ -300,21 +322,21 @@ start:
FNMSUBS F1, F2, F3, F4 // 4b822018
FNMADDS F1, F2, F3, F4 // 4f822018
- // 20.8: Single-Precision Floating-Point Compare Instructions
+ // 21.8: Single-Precision Floating-Point Compare Instructions
FEQS F0, F1, X7 // d3a300a0
FLTS F0, F1, X7 // d39300a0
FLES F0, F1, X7 // d38300a0
- // 20.9: Single-Precision Floating-Point Classify Instruction
+ // 21.9: Single-Precision Floating-Point Classify Instruction
FCLASSS F0, X5 // d31200e0
- // 21.3: Double-Precision Load and Store Instructions
+ // 22.3: Double-Precision Load and Store Instructions
FLD (X5), F0 // 07b00200
FLD 4(X5), F0 // 07b04200
FSD F0, (X5) // 27b00200
FSD F0, 4(X5) // 27b20200
- // 21.4: Double-Precision Floating-Point Computational Instructions
+ // 22.4: Double-Precision Floating-Point Computational Instructions
FADDD F1, F0, F2 // 53011002
FSUBD F1, F0, F2 // 5301100a
FMULD F1, F0, F2 // 53011012
@@ -323,7 +345,7 @@ start:
FMAXD F1, F0, F2 // 5311102a
FSQRTD F0, F1 // d300005a
- // 21.5: Double-Precision Floating-Point Conversion and Move Instructions
+ // 22.5: Double-Precision Floating-Point Conversion and Move Instructions
FCVTWD F0, X5 // d31200c2
FCVTWD.RNE F0, X5 // d30200c2
FCVTWD.RTZ F0, X5 // d31200c2
@@ -364,15 +386,85 @@ start:
FNMSUBD F1, F2, F3, F4 // 4b82201a
FNMADDD F1, F2, F3, F4 // 4f82201a
- // 21.6: Double-Precision Floating-Point Compare Instructions
+ // 22.6: Double-Precision Floating-Point Compare Instructions
FEQD F0, F1, X7 // d3a300a2
FLTD F0, F1, X7 // d39300a2
FLED F0, F1, X7 // d38300a2
- // 21.7: Double-Precision Floating-Point Classify Instruction
+ // 22.7: Double-Precision Floating-Point Classify Instruction
FCLASSD F0, X5 // d31200e2
- // 28.4.1: Address Generation Instructions (Zba)
+ //
+ // "C" Extension for Compressed Instructions, Version 2.0
+ //
+
+ // 28.3.1: Compressed Stack-Pointer-Based Loads and Stores
+ CLWSP 20(SP), X10 // 5245
+ CLDSP 24(SP), X10 // 6265
+ CFLDSP 32(SP), F10 // 0235
+ CSWSP X10, 20(SP) // 2aca
+ CSDSP X10, 24(SP) // 2aec
+ CFSDSP F10, 32(SP) // 2ab0
+
+ // 28.3.2: Compressed Register-Based Loads and Stores
+ CLW 20(X10), X11 // 4c49
+ CLD 24(X10), X11 // 0c6d
+ CFLD 32(X10), F11 // 0c31
+ CSW X11, 20(X10) // 4cc9
+ CSD X11, 24(X10) // 0ced
+ CFSD F11, 32(X10) // 0cb1
+
+ // 28.4: Compressed Control Transfer Instructions
+ CJ 1(PC) // 09a0
+ CJR X5 // 8282
+ CJALR X5 // 8292
+ CBEQZ X10, 1(PC) // 09c1
+ CBNEZ X10, 1(PC) // 09e1
+
+ // 28.5.1: Compressed Integer Constant-Generation Instructions
+ CLI $-32, X5 // 8152
+ CLI $31, X5 // fd42
+ CLUI $-32, X5 // 8172
+ CLUI $31, X5 // fd62
+
+ // 28.5.2: Compressed Integer Register-Immediate Operations
+ CADD $-32, X5 // 8112
+ CADD $31, X5 // fd02
+ CADDI $-32, X5 // 8112
+ CADDI $31, X5 // fd02
+ CADDW $-32, X5 // 8132
+ CADDW $31, X5 // fd22
+ CADDIW $-32, X5 // 8132
+ CADDIW $31, X5 // fd22
+ CADDI16SP $-512, SP // 0171
+ CADDI16SP $496, SP // 7d61
+ CADDI4SPN $4, SP, X10 // 4800
+ CADDI4SPN $1020, SP, X10 // e81f
+ CSLLI $63, X5 // fe12
+ CSRLI $63, X10 // 7d91
+ CSRAI $63, X10 // 7d95
+ CAND $-32, X10 // 0199
+ CAND $31, X10 // 7d89
+ CANDI $-32, X10 // 0199
+ CANDI $31, X10 // 7d89
+
+ // 28.5.3: Compressed Integer Register-Register Operations
+ CMV X6, X5 // 9a82
+ CADD X9, X8 // 2694
+ CAND X9, X8 // 658c
+ COR X9, X8 // 458c
+ CXOR X9, X8 // 258c
+ CSUB X9, X8 // 058c
+ CADDW X9, X8 // 259c
+ CSUBW X9, X8 // 059c
+
+ // 28.5.5: Compressed NOP Instruction
+ CNOP // 0100
+
+ // 28.5.6: Compressed Breakpoint Instruction
+ CEBREAK // 0290
+
+ // 30.2: Address Generation Instructions (Zba)
ADDUW X10, X11, X12 // 3b86a508
ADDUW X10, X11 // bb85a508
SH1ADD X11, X12, X13 // b326b620
@@ -396,7 +488,7 @@ start:
// "B" Extension for Bit Manipulation, Version 1.0.0
//
- // 28.4.2: Basic Bit Manipulation (Zbb)
+ // 30.3: Basic Bit Manipulation (Zbb)
ANDN X19, X20, X21 // b37a3a41 or 93caf9ffb37a5a01
ANDN X19, X20 // 337a3a41 or 93cff9ff337afa01
CLZ X20, X21 // 931a0a60
@@ -420,8 +512,6 @@ start:
XNOR X18, X19, X20 // 33ca2941 or 33ca2901134afaff
XNOR X18, X19 // b3c92941 or b3c9290193c9f9ff
ZEXTH X19, X20 // 3bca0908
-
- // 28.4.2: Bitwise Rotation (Zbb)
ROL X8, X9, X10 // 33958460 or b30f8040b3dff4013395840033e5af00
ROL X8, X9 // b3948460 or b30f8040b3dff401b3948400b3e49f00
ROLW X9, X10, X11 // bb159560 or b30f9040bb5ff501bb159500b3e5bf00
@@ -439,7 +529,15 @@ start:
ORCB X5, X6 // 13d37228
REV8 X7, X8 // 13d4836b
- // 28.4.4: Single-bit Instructions (Zbs)
+ // 30.4: Carry-less multiplication (Zbc)
+ CLMUL X5, X6, X7 // b313530a
+ CLMUL X5, X6 // 3313530a
+ CLMULH X5, X6, X7 // b333530a
+ CLMULH X5, X6 // 3333530a
+ CLMULR X5, X6, X7 // b323530a
+ CLMULR X5, X6 // 3323530a
+
+ // 30.5: Single-bit Instructions (Zbs)
BCLR X23, X24, X25 // b31c7c49
BCLR $63, X24 // 131cfc4b
BCLRI $1, X25, X26 // 139d1c48
@@ -497,7 +595,7 @@ start:
VLMV (X10), V3 // 8701b502
VSMV V3, (X10) // a701b502
- // 31.7.5: Vector Strided Instructions
+ // 31.7.5: Vector Constant-Stride Instructions
VLSE8V (X10), X11, V3 // 8701b50a
VLSE8V (X10), X11, V0, V3 // 8701b508
VLSE16V (X10), X11, V3 // 8751b50a
@@ -549,7 +647,7 @@ start:
VSOXEI64V V3, V2, (X10) // a771250e
VSOXEI64V V3, V2, V0, (X10) // a771250c
- // 31.7.7: Unit-stride Fault-Only-First Loads
+ // 31.7.7: Vector Unit-stride Fault-Only-First Loads
VLE8FFV (X10), V8 // 07040503
VLE16FFV (X10), V8 // 07540503
VLE32FFV (X10), V8 // 07640503
@@ -559,8 +657,6 @@ start:
VLE32FFV (X10), V0, V8 // 07640501
VLE64FFV (X10), V0, V8 // 07740501
- // 31.7.8: Vector Load/Store Segment Instructions
-
// 31.7.8.1: Vector Unit-Stride Segment Loads and Stores
VLSEG2E8V (X10), V8 // 07040522
VLSEG2E16V (X10), V8 // 07540522
@@ -751,7 +847,7 @@ start:
VLSEG8E32FFV (X10), V0, V8 // 076405e1
VLSEG8E64FFV (X10), V0, V8 // 077405e1
- // 31.7.8.2: Vector Strided Segment Loads and Stores
+ // 31.7.8.2: Vector Constant-Stride Segment Loads and Stores
VLSSEG2E8V (X10), X11, V8 // 0704b52a
VLSSEG2E16V (X10), X11, V8 // 0754b52a
VLSSEG2E32V (X10), X11, V8 // 0764b52a
@@ -879,7 +975,6 @@ start:
VSSSEG8E64V V24, X11, V0, (X10) // 277cb5e8
// 31.7.8.3: Vector Indexed Segment Loads and Stores
-
VLUXSEG2EI8V (X10), V4, V8 // 07044526
VLUXSEG2EI16V (X10), V4, V8 // 07544526
VLUXSEG2EI32V (X10), V4, V8 // 07644526
@@ -1895,6 +1990,50 @@ start:
VMV4RV V8, V4 // 57b2819e
VMV8RV V8, V0 // 57b0839e
+ // 33.2.1: Vector Basic Bit-manipulation (Zvbb)
+ VANDNVV V1, V2, V3 // d7812006
+ VANDNVV V1, V2, V0, V3 // d7812004
+ VANDNVX X10, V2, V3 // d7412506
+ VANDNVX X10, V2, V0, V3 // d7412504
+ VBREVV V2, V3 // d721254a
+ VBREVV V2, V0, V3 // d7212548
+ VBREV8V V2, V3 // d721244a
+ VBREV8V V2, V0, V3 // d7212448
+ VREV8V V2, V3 // d7a1244a
+ VREV8V V2, V0, V3 // d7a12448
+ VCLZV V2, V3 // d721264a
+ VCLZV V2, V0, V3 // d7212648
+ VCTZV V2, V3 // d7a1264a
+ VCTZV V2, V0, V3 // d7a12648
+ VCPOPV V2, V3 // d721274a
+ VCPOPV V2, V0, V3 // d7212748
+ VROLVV V1, V2, V3 // d7812056
+ VROLVV V1, V2, V0, V3 // d7812054
+ VROLVX X10, V2, V3 // d7412556
+ VROLVX X10, V2, V0, V3 // d7412554
+ VRORVV V1, V2, V3 // d7812052
+ VRORVV V1, V2, V0, V3 // d7812050
+ VRORVX X10, V2, V3 // d7412552
+ VRORVX X10, V2, V0, V3 // d7412550
+ VRORVI $16, V2, V3 // d7312852
+ VRORVI $16, V2, V0, V3 // d7312850
+ VWSLLVV V1, V2, V3 // d78120d6
+ VWSLLVV V1, V2, V0, V3 // d78120d4
+ VWSLLVX X10, V2, V3 // d74125d6
+ VWSLLVX X10, V2, V0, V3 // d74125d4
+ VWSLLVI $16, V2, V3 // d73128d6
+ VWSLLVI $16, V2, V0, V3 // d73128d4
+
+ // 33.2.2: Vector Carryless Multiplication (Zvbc)
+ VCLMULVV V1, V2, V3 // d7a12032
+ VCLMULVV V1, V2, V0, V3 // d7a12030
+ VCLMULVX X10, V2, V3 // d7612532
+ VCLMULVX X10, V2, V0, V3 // d7612530
+ VCLMULHVV V1, V2, V3 // d7a12036
+ VCLMULHVV V1, V2, V0, V3 // d7a12034
+ VCLMULHVX X10, V2, V3 // d7612536
+ VCLMULHVX X10, V2, V0, V3 // d7612534
+
//
// Privileged ISA
//
@@ -1923,9 +2062,12 @@ start:
// Converted to load and shift(s)
MOV $0xffffffff, X5 // MOV $4294967295, X5 // 9302f0ff93d20202
+ MOV $0x80000001, X5 // MOV $2147483649, X5 // b70200809b8212009392020293d20202 or b70200809b821200bb820208
MOV $0x100000000, X5 // MOV $4294967296, X5 // 9302100093920202
MOV $0xfffffffffffda, X5 // MOV $4503599627370458, X5 // 9302d0fe9392d20093d2c200
MOV $0xffffffffffffe, X5 // MOV $4503599627370494, X5 // 9302f0ff9392d20093d2c200
+ MOV $0x0800000010000000, X5 // MOV $576460752571858944, X5 // b70200809b8212009392020293d24200
+ MOV $0x0abcdabcd0000000, X5 // MOV $773733740479250432, X5 // b7b2cdab9b82d2bc9392020293d24200
MOV $0x7fffffff00000000, X5 // MOV $9223372032559808512, X5 // b70200809b82f2ff93920202
MOV $0x8000000100000000, X5 // MOV $-9223372032559808512, X5 // b70200809b82120093920202
MOV $0xffffffff00000000, X5 // MOV $-4294967296, X5 // 9302f0ff93920202
@@ -1933,11 +2075,8 @@ start:
MOV $0x7fffffffffffffff, X5 // MOV $9223372036854775807, X5 // 9302f0ff93d21200
// Converted to load of symbol (AUIPC + LD)
- MOV $0x80000001, X5 // MOV $2147483649, X5 // 9702000083b20200
MOV $0x100000001, X5 // MOV $4294967297, X5 // 9702000083b20200
- MOV $0x0800000010000000, X5 // MOV $576460752571858944, X5 // 9702000083b20200
MOV $0x8000000010000000, X5 // MOV $-9223372036586340352, X5 // 9702000083b20200
- MOV $0x0abcdabcd0000000, X5 // MOV $773733740479250432, X5 // 9702000083b20200
MOV $0x8abcdabcd0000000, X5 // MOV $-8449638296375525376, X5 // 9702000083b20200
MOV $0xfff0000000ffffff, X5 // MOV $-4503599610593281, X5 // 9702000083b20200
diff --git a/src/cmd/asm/internal/asm/testdata/riscv64error.s b/src/cmd/asm/internal/asm/testdata/riscv64error.s
index 3c09770d2a2dcd..fd5e4acdf37c50 100644
--- a/src/cmd/asm/internal/asm/testdata/riscv64error.s
+++ b/src/cmd/asm/internal/asm/testdata/riscv64error.s
@@ -3,9 +3,14 @@
// license that can be found in the LICENSE file.
TEXT errors(SB),$0
+ CSRC X5, TU // ERROR "unknown CSR"
+ CSRC (X5), CYCLE // ERROR "integer register or immediate expected for 1st operand"
+ CSRC $-1, TIME // ERROR "immediate out of range 0 to 31"
+ CSRR TU, X5 // ERROR "unknown CSR"
+ CSRR CYCLE, (X5) // ERROR "needs an integer register output"
CSRRC (X10), CYCLE, X5 // ERROR "integer register or immediate expected for 1st operand"
CSRRC X0, TU, X5 // ERROR "unknown CSR"
- CSRRC X0, CYCLE // ERROR "missing CSR name"
+ CSRRC X0, X5 // ERROR "missing CSR name"
CSRRC X0, CYCLE, (X10) // ERROR "needs an integer register output"
CSRRC $-1, TIME, X15 // ERROR "immediate out of range 0 to 31"
CSRRCI $32, TIME, X15 // ERROR "immediate out of range 0 to 31"
@@ -13,17 +18,25 @@ TEXT errors(SB),$0
CSRRS (X10), CYCLE, X5 // ERROR "integer register or immediate expected for 1st operand"
CSRRS X0, CYCLE, (X10) // ERROR "needs an integer register output"
CSRRS X0, TU, X5 // ERROR "unknown CSR"
- CSRRS X0, CYCLE // ERROR "missing CSR name"
+ CSRRS X0, X5 // ERROR "missing CSR name"
CSRRS $-1, TIME, X15 // ERROR "immediate out of range 0 to 31"
CSRRSI $32, TIME, X15 // ERROR "immediate out of range 0 to 31"
CSRRSI $1, TIME, (X15) // ERROR "needs an integer register output"
CSRRW (X10), CYCLE, X5 // ERROR "integer register or immediate expected for 1st operand"
CSRRW X0, TU, X5 // ERROR "unknown CSR"
- CSRRW X0, CYCLE // ERROR "missing CSR name"
+ CSRRW X0, X5 // ERROR "missing CSR name"
CSRRW X0, CYCLE, (X5) // ERROR "needs an integer register output"
CSRRW $-1, TIME, X15 // ERROR "immediate out of range 0 to 31"
CSRRWI $32, TIME, X15 // ERROR "immediate out of range 0 to 31"
CSRRWI $1, TIME, (X15) // ERROR "needs an integer register output"
+ CSRW X5, TU // ERROR "unknown CSR"
+ CSRW $-1, TIME // ERROR "immediate out of range 0 to 31"
+ CSRWI $32, TIME // ERROR "immediate out of range 0 to 31"
+ CSRS (X5), CYCLE // ERROR "integer register or immediate expected for 1st operand"
+ CSRS X5, TU // ERROR "unknown CSR"
+ CSRS $-1, TIME // ERROR "immediate out of range 0 to 31"
+ CSRSI $32, TIME // ERROR "immediate out of range 0 to 31"
+ CSRW (X5), CYCLE // ERROR "integer register or immediate expected for 1st operand"
MOV $errors(SB), (X5) // ERROR "address load must target register"
MOV $8(SP), (X5) // ERROR "address load must target register"
MOVB $8(SP), X5 // ERROR "unsupported address load"
@@ -68,6 +81,13 @@ TEXT errors(SB),$0
SD X5, 4294967296(X6) // ERROR "constant 4294967296 too large"
FNES F1, (X5) // ERROR "needs an integer register output"
+ // Memory Ordering Instructions
+ FENCE X1, R // ERROR "invalid FENCE predecessor operand"
+ FENCE R, X2 // ERROR "invalid FENCE successor operand"
+ FENCE $1, R // ERROR "invalid FENCE predecessor operand"
+ FENCE R, $2 // ERROR "invalid FENCE successor operand"
+ FENCE.TSO R, R // ERROR "FENCE.TSO must not have operands"
+
//
// "V" Standard Extension for Vector Operations, Version 1.0
//
@@ -416,5 +436,25 @@ TEXT errors(SB),$0
VRGATHEREI16VV V1, V2, V4, V3 // ERROR "invalid vector mask register"
VRGATHERVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
VRGATHERVI $16, V2, V4, V3 // ERROR "invalid vector mask register"
+ VANDNVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VANDNVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
+ VBREVV V2, V3, V4 // ERROR "invalid vector mask register"
+ VBREV8V V2, V3, V4 // ERROR "invalid vector mask register"
+ VREV8V V2, V3, V4 // ERROR "invalid vector mask register"
+ VCLZV V2, V3, V4 // ERROR "invalid vector mask register"
+ VCTZV V2, V3, V4 // ERROR "invalid vector mask register"
+ VCPOPV V2, V3, V4 // ERROR "invalid vector mask register"
+ VROLVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VROLVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
+ VRORVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VRORVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
+ VRORVI $16, V2, V4, V3 // ERROR "invalid vector mask register"
+ VWSLLVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VWSLLVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
+ VWSLLVI $16, V2, V4, V3 // ERROR "invalid vector mask register"
+ VCLMULVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VCLMULVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
+ VCLMULHVV V1, V2, V4, V3 // ERROR "invalid vector mask register"
+ VCLMULHVX X10, V2, V4, V3 // ERROR "invalid vector mask register"
RET
diff --git a/src/cmd/asm/internal/asm/testdata/riscv64validation.s b/src/cmd/asm/internal/asm/testdata/riscv64validation.s
index c8ae4c92114bbc..4e4e16286b6355 100644
--- a/src/cmd/asm/internal/asm/testdata/riscv64validation.s
+++ b/src/cmd/asm/internal/asm/testdata/riscv64validation.s
@@ -15,6 +15,144 @@ TEXT validation(SB),$0
WORD $-1 // ERROR "must be in range [0x0, 0xffffffff]"
WORD $0x100000000 // ERROR "must be in range [0x0, 0xffffffff]"
+ //
+ // "C" Extension for Compressed Instructions, Version 2.0
+ //
+ CLWSP 20(X5), X10 // ERROR "rs2 must be SP/X2"
+ CLWSP 20(SP), X0 // ERROR "cannot use register X0"
+ CLWSP 20(SP), F10 // ERROR "expected integer register in rd position"
+ CLWSP 22(SP), X10 // ERROR "must be a multiple of 4"
+ CLDSP 24(X5), X10 // ERROR "rs2 must be SP/X2"
+ CLDSP 24(SP), X0 // ERROR "cannot use register X0"
+ CLDSP 24(SP), F10 // ERROR "expected integer register in rd position"
+ CLDSP 28(SP), X10 // ERROR "must be a multiple of 8"
+ CFLDSP 32(X5), F10 // ERROR "rs2 must be SP/X2"
+ CFLDSP 32(SP), X10 // ERROR "expected float register in rd position"
+ CFLDSP 36(SP), F10 // ERROR "must be a multiple of 8"
+ CSWSP X10, 20(X5) // ERROR "rd must be SP/X2"
+ CSWSP F10, 20(SP) // ERROR "expected integer register in rs2 position"
+ CSWSP X10, 22(SP) // ERROR "must be a multiple of 4"
+ CSDSP X10, 24(X5) // ERROR "rd must be SP/X2"
+ CSDSP F10, 24(SP) // ERROR "expected integer register in rs2 position"
+ CSDSP X10, 28(SP) // ERROR "must be a multiple of 8"
+ CFSDSP F10, 32(X5) // ERROR "rd must be SP/X2"
+ CFSDSP X10, 32(SP) // ERROR "expected float register in rs2 position"
+ CFSDSP F10, 36(SP) // ERROR "must be a multiple of 8"
+ CLW 20(X10), F11 // ERROR "expected integer prime register in rd position"
+ CLW 20(X5), X11 // ERROR "expected integer prime register in rs1 position"
+ CLW 20(X10), X5 // ERROR "expected integer prime register in rd position"
+ CLW -1(X10), X11 // ERROR "must be in range [0, 127]"
+ CLW 22(X10), X11 // ERROR "must be a multiple of 4"
+ CLW 128(X10), X11 // ERROR "must be in range [0, 127]"
+ CLD 24(X10), F11 // ERROR "expected integer prime register in rd position"
+ CLD 24(X5), X11 // ERROR "expected integer prime register in rs1 position"
+ CLD -1(X10), X11 // ERROR "must be in range [0, 255]"
+ CLD 30(X10), X11 // ERROR "must be a multiple of 8"
+ CLD 256(X10), X11 // ERROR "must be in range [0, 255]"
+ CFLD 32(X10), X11 // ERROR "expected float prime register in rd position"
+ CFLD 32(X5), F11 // ERROR "expected integer prime register in rs1 position"
+ CFLD -1(X10), F11 // ERROR "must be in range [0, 255]"
+ CFLD 34(X10), F11 // ERROR "must be a multiple of 8"
+ CFLD 256(X10), F11 // ERROR "must be in range [0, 255]"
+ CSW F11, 20(X10) // ERROR "expected integer prime register in rs2 position"
+ CSW X11, -1(X10) // ERROR "must be in range [0, 127]"
+ CSW X11, 22(X10) // ERROR "must be a multiple of 4"
+ CSW X11, 128(X10) // ERROR "must be in range [0, 127]"
+ CSD F11, 24(X10) // ERROR "expected integer prime register in rs2 position"
+ CSD X11, -1(X10) // ERROR "must be in range [0, 255]"
+ CSD X11, 28(X10) // ERROR "must be a multiple of 8"
+ CSD X11, 256(X10) // ERROR "must be in range [0, 255]"
+ CFSD X11, 32(X10) // ERROR "expected float prime register in rs2 position"
+ CFSD F11, -1(X10) // ERROR "must be in range [0, 255]"
+ CFSD F11, 36(X10) // ERROR "must be a multiple of 8"
+ CFSD F11, 256(X10) // ERROR "must be in range [0, 255]"
+ CJR X0 // ERROR "cannot use register X0 in rs1"
+ CJR X10, X11 // ERROR "expected no register in rs2"
+ CJALR X0 // ERROR "cannot use register X0 in rs1"
+ CJALR X10, X11 // ERROR "expected no register in rd"
+ CBEQZ X5, 1(PC) // ERROR "expected integer prime register in rs1"
+ CBNEZ X5, 1(PC) // ERROR "expected integer prime register in rs1"
+ CLI $3, X0 // ERROR "cannot use register X0 in rd"
+ CLI $-33, X5 // ERROR "must be in range [-32, 31]"
+ CLI $32, X5 // ERROR "must be in range [-32, 31]"
+ CLUI $0, X5 // ERROR "immediate cannot be zero"
+ CLUI $3, X0 // ERROR "cannot use register X0 in rd"
+ CLUI $3, X2 // ERROR "cannot use register SP/X2 in rd"
+ CLUI $-33, X5 // ERROR "must be in range [-32, 31]"
+ CLUI $32, X5 // ERROR "must be in range [-32, 31]"
+ CADD $31, X5, X6 // ERROR "rd must be the same as rs1"
+ CADD $-33, X5 // ERROR "must be in range [-32, 31]"
+ CADD $32, X5 // ERROR "must be in range [-32, 31]"
+ CADDI $0, X5 // ERROR "immediate cannot be zero"
+ CADDI $31, X5, X6 // ERROR "rd must be the same as rs1"
+ CADDI $-33, X5 // ERROR "must be in range [-32, 31]"
+ CADDI $32, X5 // ERROR "must be in range [-32, 31]"
+ CADDW $-33, X5 // ERROR "must be in range [-32, 31]"
+ CADDW $32, X5 // ERROR "must be in range [-32, 31]"
+ CADDIW $-33, X5 // ERROR "must be in range [-32, 31]"
+ CADDIW $32, X5 // ERROR "must be in range [-32, 31]"
+ CADDI16SP $0, SP // ERROR "immediate cannot be zero"
+ CADDI16SP $16, X5 // ERROR "rd must be SP/X2"
+ CADDI16SP $-513, SP // ERROR "must be in range [-512, 511]"
+ CADDI16SP $20, SP // ERROR "must be a multiple of 16"
+ CADDI16SP $512, SP // ERROR "must be in range [-512, 511]"
+ CADDI4SPN $4, SP, X5 // ERROR "expected integer prime register in rd"
+ CADDI4SPN $4, X5, X10 // ERROR "SP/X2 must be in rs1"
+ CADDI4SPN $-1, SP, X10 // ERROR "must be in range [0, 1023]"
+ CADDI4SPN $0, SP, X10 // ERROR "immediate cannot be zero"
+ CADDI4SPN $6, SP, X10 // ERROR "must be a multiple of 4"
+ CADDI4SPN $1024, SP, X10 // ERROR "must be in range [0, 1023]"
+ CSLLI $63, X5, X6 // ERROR "rd must be the same as rs1"
+ CSLLI $-1, X5 // ERROR "must be in range [0, 63]"
+ CSLLI $0, X5 // ERROR "immediate cannot be zero"
+ CSLLI $64, X5 // ERROR "must be in range [0, 63]"
+ CSRLI $63, X10, X11 // ERROR "rd must be the same as rs1"
+ CSRLI $63, X5 // ERROR "expected integer prime register in rd"
+ CSRLI $-1, X10 // ERROR "must be in range [0, 63]"
+ CSRLI $0, X10 // ERROR "immediate cannot be zero"
+ CSRLI $64, X10 // ERROR "must be in range [0, 63]"
+ CSRAI $63, X10, X11 // ERROR "rd must be the same as rs1"
+ CSRAI $63, X5 // ERROR "expected integer prime register in rd"
+ CSRAI $-1, X10 // ERROR "must be in range [0, 63]"
+ CSRAI $0, X10 // ERROR "immediate cannot be zero"
+ CSRAI $64, X10 // ERROR "must be in range [0, 63]"
+ CAND $1, X10, X11 // ERROR "rd must be the same as rs1"
+ CAND $1, X5 // ERROR "expected integer prime register in rd"
+ CAND $-64, X10 // ERROR "must be in range [-32, 31]"
+ CAND $63, X10 // ERROR "must be in range [-32, 31]"
+ CANDI $1, X10, X11 // ERROR "rd must be the same as rs1"
+ CANDI $1, X5 // ERROR "expected integer prime register in rd"
+ CANDI $-64, X10 // ERROR "must be in range [-32, 31]"
+ CANDI $63, X10 // ERROR "must be in range [-32, 31]"
+ CMV X0, X5 // ERROR "cannot use register X0 in rs2"
+ CMV X5, X6, X7 // ERROR "expected no register in rs1"
+ CMV X5, X0 // ERROR "cannot use register X0 in rd"
+ CMV F1, X5 // ERROR "expected integer register in rs2"
+ CMV X5, F1 // ERROR "expected integer register in rd"
+ CADD X5, X6, X7 // ERROR "rd must be the same as rs1"
+ CADD X0, X8 // ERROR "cannot use register X0 in rs2"
+ CADD X8, X0 // ERROR "cannot use register X0 in rd"
+ CAND X10, X11, X12 // ERROR "rd must be the same as rs1"
+ CAND X5, X11 // ERROR "expected integer prime register in rs2"
+ CAND X10, X5 // ERROR "expected integer prime register in rd"
+ COR X10, X11, X12 // ERROR "rd must be the same as rs1"
+ COR X5, X11 // ERROR "expected integer prime register in rs2"
+ COR X10, X5 // ERROR "expected integer prime register in rd"
+ CXOR X10, X11, X12 // ERROR "rd must be the same as rs1"
+ CXOR X5, X11 // ERROR "expected integer prime register in rs2"
+ CXOR X10, X5 // ERROR "expected integer prime register in rd"
+ CSUB X10, X11, X12 // ERROR "rd must be the same as rs1"
+ CSUB X5, X11 // ERROR "expected integer prime register in rs2"
+ CSUB X10, X5 // ERROR "expected integer prime register in rd"
+ CADDW X10, X11, X12 // ERROR "rd must be the same as rs1"
+ CADDW X5, X11 // ERROR "expected integer prime register in rs2"
+ CADDW X10, X5 // ERROR "expected integer prime register in rd"
+ CSUBW X10, X11, X12 // ERROR "rd must be the same as rs1"
+ CSUBW X5, X11 // ERROR "expected integer prime register in rs2"
+ CSUBW X10, X5 // ERROR "expected integer prime register in rd"
+ CNOP X10 // ERROR "expected no register in rs2"
+ CEBREAK X10 // ERROR "expected no register in rs2"
+
//
// "V" Standard Extension for Vector Operations, Version 1.0
//
@@ -459,5 +597,9 @@ TEXT validation(SB),$0
VMV2RV X10, V10 // ERROR "expected vector register in vs2 position"
VMV4RV X10, V4 // ERROR "expected vector register in vs2 position"
VMV8RV X10, V0 // ERROR "expected vector register in vs2 position"
+ VRORVI $32, V2, V4 // ERROR "unsigned immediate 32 must be in range [0, 31]"
+ VRORVI $-1, V2, V4 // ERROR "unsigned immediate -1 must be in range [0, 31]"
+ VWSLLVI $32, V2, V4 // ERROR "unsigned immediate 32 must be in range [0, 31]"
+ VWSLLVI $-1, V2, V4 // ERROR "unsigned immediate -1 must be in range [0, 31]"
RET
diff --git a/src/cmd/asm/internal/asm/testdata/s390x.s b/src/cmd/asm/internal/asm/testdata/s390x.s
index 93c3ec9ea7f22e..cb9a9917a5fde5 100644
--- a/src/cmd/asm/internal/asm/testdata/s390x.s
+++ b/src/cmd/asm/internal/asm/testdata/s390x.s
@@ -456,6 +456,7 @@ TEXT main·foo(SB),DUPOK|NOSPLIT,$16-0 // TEXT main.foo(SB), DUPOK|NOSPLIT, $16-
VST V1, (R15) // e710f000000e
VL (R15), V31 // e7f0f0000806
VST V31, (R15) // e7f0f000080e
+ VSTRL V4, $6, 1(R4) // e6064001403d
VESLB $5, V14 // e7ee00050030
VESRAG $0, V15, V16 // e70f0000383a
VLM (R15), V8, V23 // e787f0000436
diff --git a/src/cmd/asm/internal/flags/flags.go b/src/cmd/asm/internal/flags/flags.go
index e15a062749fab5..a02dcae2c6f0ed 100644
--- a/src/cmd/asm/internal/flags/flags.go
+++ b/src/cmd/asm/internal/flags/flags.go
@@ -26,11 +26,13 @@ var (
SymABIs = flag.Bool("gensymabis", false, "write symbol ABI information to output file, don't assemble")
Importpath = flag.String("p", obj.UnlinkablePkg, "set expected package import to path")
Spectre = flag.String("spectre", "", "enable spectre mitigations in `list` (all, ret)")
+ Std = flag.Bool("std", false, "building standard library")
)
var DebugFlags struct {
- MayMoreStack string `help:"call named function before all stack growth checks"`
- PCTab string `help:"print named pc-value table\nOne of: pctospadj, pctofile, pctoline, pctoinline, pctopcdata"`
+ CompressInstructions int `help:"use compressed instructions when possible (if supported by architecture)"`
+ MayMoreStack string `help:"call named function before all stack growth checks"`
+ PCTab string `help:"print named pc-value table\nOne of: pctospadj, pctofile, pctoline, pctoinline, pctopcdata"`
}
var (
@@ -47,6 +49,8 @@ func init() {
flag.Var(objabi.NewDebugFlag(&DebugFlags, nil), "d", "enable debugging settings; try -d help")
objabi.AddVersionFlag() // -V
objabi.Flagcount("S", "print assembly and machine code", &PrintOut)
+
+ DebugFlags.CompressInstructions = 1
}
// MultiFlag allows setting a value multiple times to collect a list, as in -I=dir1 -I=dir2.
diff --git a/src/cmd/asm/internal/lex/input.go b/src/cmd/asm/internal/lex/input.go
index 789e229a779916..342ac5ac483beb 100644
--- a/src/cmd/asm/internal/lex/input.go
+++ b/src/cmd/asm/internal/lex/input.go
@@ -68,7 +68,7 @@ func predefine(defines flags.MultiFlag) map[string]*Macro {
var panicOnError bool // For testing.
-func (in *Input) Error(args ...interface{}) {
+func (in *Input) Error(args ...any) {
if panicOnError {
panic(fmt.Errorf("%s:%d: %s", in.File(), in.Line(), fmt.Sprintln(args...)))
}
@@ -77,7 +77,7 @@ func (in *Input) Error(args ...interface{}) {
}
// expectText is like Error but adds "got XXX" where XXX is a quoted representation of the most recent token.
-func (in *Input) expectText(args ...interface{}) {
+func (in *Input) expectText(args ...any) {
in.Error(append(args, "; got", strconv.Quote(in.Stack.Text()))...)
}
diff --git a/src/cmd/asm/main.go b/src/cmd/asm/main.go
index 9db1fec8beac3c..d553fc8c32c0c2 100644
--- a/src/cmd/asm/main.go
+++ b/src/cmd/asm/main.go
@@ -40,6 +40,7 @@ func main() {
log.Fatalf("unrecognized architecture %s", GOARCH)
}
ctxt := obj.Linknew(architecture.LinkArch)
+ ctxt.CompressInstructions = flags.DebugFlags.CompressInstructions != 0
ctxt.Debugasm = flags.PrintOut
ctxt.Debugvlog = flags.DebugV
ctxt.Flag_dynlink = *flags.Dynlink
@@ -49,6 +50,7 @@ func main() {
ctxt.Debugpcln = flags.DebugFlags.PCTab
ctxt.IsAsm = true
ctxt.Pkgpath = *flags.Importpath
+ ctxt.Std = *flags.Std
ctxt.DwTextCount = objabi.DummyDwarfFunctionCountForAssembler()
switch *flags.Spectre {
default:
@@ -93,7 +95,7 @@ func main() {
for _, f := range flag.Args() {
lexer := lex.NewLexer(f)
parser := asm.NewParser(ctxt, architecture, lexer)
- ctxt.DiagFunc = func(format string, args ...interface{}) {
+ ctxt.DiagFunc = func(format string, args ...any) {
diag = true
log.Printf(format, args...)
}
diff --git a/src/cmd/cgo/ast.go b/src/cmd/cgo/ast.go
index 861479db7acf5b..df0552f525adad 100644
--- a/src/cmd/cgo/ast.go
+++ b/src/cmd/cgo/ast.go
@@ -199,7 +199,7 @@ func commentText(g *ast.CommentGroup) string {
return strings.Join(pieces, "")
}
-func (f *File) validateIdents(x interface{}, context astContext) {
+func (f *File) validateIdents(x any, context astContext) {
if x, ok := x.(*ast.Ident); ok {
if f.isMangledName(x.Name) {
error_(x.Pos(), "identifier %q may conflict with identifiers generated by cgo", x.Name)
@@ -208,7 +208,7 @@ func (f *File) validateIdents(x interface{}, context astContext) {
}
// Save various references we are going to need later.
-func (f *File) saveExprs(x interface{}, context astContext) {
+func (f *File) saveExprs(x any, context astContext) {
switch x := x.(type) {
case *ast.Expr:
switch (*x).(type) {
@@ -278,7 +278,7 @@ func (f *File) saveCall(call *ast.CallExpr, context astContext) {
}
// If a function should be exported add it to ExpFunc.
-func (f *File) saveExport(x interface{}, context astContext) {
+func (f *File) saveExport(x any, context astContext) {
n, ok := x.(*ast.FuncDecl)
if !ok {
return
@@ -301,24 +301,19 @@ func (f *File) saveExport(x interface{}, context astContext) {
error_(c.Pos(), "export comment has wrong name %q, want %q", name, n.Name.Name)
}
- doc := ""
- for _, c1 := range n.Doc.List {
- if c1 != c {
- doc += c1.Text + "\n"
- }
- }
-
f.ExpFunc = append(f.ExpFunc, &ExpFunc{
Func: n,
ExpName: name,
- Doc: doc,
+ // Caution: Do not set the Doc field on purpose
+ // to ensure that there are no unintended artifacts
+ // in the binary. See https://go.dev/issue/76697.
})
break
}
}
// Make f.ExpFunc[i] point at the Func from this AST instead of the other one.
-func (f *File) saveExport2(x interface{}, context astContext) {
+func (f *File) saveExport2(x any, context astContext) {
n, ok := x.(*ast.FuncDecl)
if !ok {
return
@@ -355,7 +350,7 @@ const (
)
// walk walks the AST x, calling visit(f, x, context) for each node.
-func (f *File) walk(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
+func (f *File) walk(x any, context astContext, visit func(*File, any, astContext)) {
visit(f, x, context)
switch n := x.(type) {
case *ast.Expr:
@@ -363,7 +358,8 @@ func (f *File) walk(x interface{}, context astContext, visit func(*File, interfa
// everything else just recurs
default:
- f.walkUnexpected(x, context, visit)
+ error_(token.NoPos, "unexpected type %T in walk", x)
+ panic("unexpected type")
case nil:
@@ -396,6 +392,9 @@ func (f *File) walk(x interface{}, context astContext, visit func(*File, interfa
case *ast.IndexExpr:
f.walk(&n.X, ctxExpr, visit)
f.walk(&n.Index, ctxExpr, visit)
+ case *ast.IndexListExpr:
+ f.walk(&n.X, ctxExpr, visit)
+ f.walk(n.Indices, ctxExpr, visit)
case *ast.SliceExpr:
f.walk(&n.X, ctxExpr, visit)
if n.Low != nil {
@@ -434,8 +433,8 @@ func (f *File) walk(x interface{}, context astContext, visit func(*File, interfa
case *ast.StructType:
f.walk(n.Fields, ctxField, visit)
case *ast.FuncType:
- if tparams := funcTypeTypeParams(n); tparams != nil {
- f.walk(tparams, ctxParam, visit)
+ if n.TypeParams != nil {
+ f.walk(n.TypeParams, ctxParam, visit)
}
f.walk(n.Params, ctxParam, visit)
if n.Results != nil {
@@ -524,8 +523,8 @@ func (f *File) walk(x interface{}, context astContext, visit func(*File, interfa
f.walk(n.Values, ctxExpr, visit)
}
case *ast.TypeSpec:
- if tparams := typeSpecTypeParams(n); tparams != nil {
- f.walk(tparams, ctxParam, visit)
+ if n.TypeParams != nil {
+ f.walk(n.TypeParams, ctxParam, visit)
}
f.walk(&n.Type, ctxType, visit)
diff --git a/src/cmd/cgo/ast_go1.go b/src/cmd/cgo/ast_go1.go
deleted file mode 100644
index 2f65f0f718356f..00000000000000
--- a/src/cmd/cgo/ast_go1.go
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build compiler_bootstrap
-
-package main
-
-import (
- "go/ast"
- "go/token"
-)
-
-func (f *File) walkUnexpected(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
- error_(token.NoPos, "unexpected type %T in walk", x)
- panic("unexpected type")
-}
-
-func funcTypeTypeParams(n *ast.FuncType) *ast.FieldList {
- return nil
-}
-
-func typeSpecTypeParams(n *ast.TypeSpec) *ast.FieldList {
- return nil
-}
diff --git a/src/cmd/cgo/ast_go118.go b/src/cmd/cgo/ast_go118.go
deleted file mode 100644
index ced30728dc9a79..00000000000000
--- a/src/cmd/cgo/ast_go118.go
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !compiler_bootstrap
-
-package main
-
-import (
- "go/ast"
- "go/token"
-)
-
-func (f *File) walkUnexpected(x interface{}, context astContext, visit func(*File, interface{}, astContext)) {
- switch n := x.(type) {
- default:
- error_(token.NoPos, "unexpected type %T in walk", x)
- panic("unexpected type")
-
- case *ast.IndexListExpr:
- f.walk(&n.X, ctxExpr, visit)
- f.walk(n.Indices, ctxExpr, visit)
- }
-}
-
-func funcTypeTypeParams(n *ast.FuncType) *ast.FieldList {
- return n.TypeParams
-}
-
-func typeSpecTypeParams(n *ast.TypeSpec) *ast.FieldList {
- return n.TypeParams
-}
diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go
index ef5272299bbf07..47d243d1a16190 100644
--- a/src/cmd/cgo/doc.go
+++ b/src/cmd/cgo/doc.go
@@ -127,7 +127,7 @@ environment variable when running the go tool: set it to 1 to enable
the use of cgo, and to 0 to disable it. The go tool will set the
build constraint "cgo" if cgo is enabled. The special import "C"
implies the "cgo" build constraint, as though the file also said
-"//go:build cgo". Therefore, if cgo is disabled, files that import
+"//go:build cgo". Therefore, if cgo is disabled, files that import
"C" will not be built by the go tool. (For more about build constraints
see https://golang.org/pkg/go/build/#hdr-Build_Constraints).
@@ -425,8 +425,8 @@ and of course there is nothing stopping the C code from doing anything
it likes. However, programs that break these rules are likely to fail
in unexpected and unpredictable ways.
-The runtime/cgo.Handle type can be used to safely pass Go values
-between Go and C. See the runtime/cgo package documentation for details.
+The type [runtime/cgo.Handle] can be used to safely pass Go values
+between Go and C.
Note: the current implementation has a bug. While Go code is permitted
to write nil or a C pointer (but not a Go pointer) to C memory, the
diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
index 886ddf2d461b65..b8b8d6baec6554 100644
--- a/src/cmd/cgo/gcc.go
+++ b/src/cmd/cgo/gcc.go
@@ -24,6 +24,7 @@ import (
"math"
"os"
"os/exec"
+ "path/filepath"
"slices"
"strconv"
"strings"
@@ -1056,7 +1057,7 @@ func (p *Package) rewriteCall(f *File, call *Call) (string, bool) {
func (p *Package) needsPointerCheck(f *File, t ast.Expr, arg ast.Expr) bool {
// An untyped nil does not need a pointer check, and when
// _cgoCheckPointer returns the untyped nil the type assertion we
- // are going to insert will fail. Easier to just skip nil arguments.
+ // are going to insert will fail. Easier to just skip nil arguments.
// TODO: Note that this fails if nil is shadowed.
if id, ok := arg.(*ast.Ident); ok && id.Name == "nil" {
return false
@@ -1121,6 +1122,9 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
if t.Name == "error" {
return true
}
+ if t.Name == "any" {
+ return true
+ }
if goTypes[t.Name] != nil {
return false
}
@@ -1158,7 +1162,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
// If addPosition is true, add position info to the idents of C names in arg.
func (p *Package) mangle(f *File, arg *ast.Expr, addPosition bool) (ast.Expr, bool) {
needsUnsafe := false
- f.walk(arg, ctxExpr, func(f *File, arg interface{}, context astContext) {
+ f.walk(arg, ctxExpr, func(f *File, arg any, context astContext) {
px, ok := arg.(*ast.Expr)
if !ok {
return
@@ -1792,7 +1796,7 @@ var n atomic.Int64
func gccTmp() string {
c := strconv.Itoa(int(n.Add(1)))
- return *objDir + "_cgo_" + c + ".o"
+ return filepath.Join(outputDir(), "_cgo_"+c+".o")
}
// gccCmd returns the gcc command line to use for compiling
@@ -2439,7 +2443,7 @@ func (tr *TypeRepr) Empty() bool {
// Set modifies the type representation.
// If fargs are provided, repr is used as a format for fmt.Sprintf.
// Otherwise, repr is used unprocessed as the type representation.
-func (tr *TypeRepr) Set(repr string, fargs ...interface{}) {
+func (tr *TypeRepr) Set(repr string, fargs ...any) {
tr.Repr = repr
tr.FormatArgs = fargs
}
@@ -2713,7 +2717,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
// so execute the basic things that the struct case would do
// other than try to determine a Go representation.
tt := *t
- tt.C = &TypeRepr{"%s %s", []interface{}{dt.Kind, tag}}
+ tt.C = &TypeRepr{"%s %s", []any{dt.Kind, tag}}
// We don't know what the representation of this struct is, so don't let
// anyone allocate one on the Go side. As a side effect of this annotation,
// pointers to this type will not be considered pointers in Go. They won't
@@ -2743,7 +2747,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
t.Align = align
tt := *t
if tag != "" {
- tt.C = &TypeRepr{"struct %s", []interface{}{tag}}
+ tt.C = &TypeRepr{"struct %s", []any{tag}}
}
tt.Go = g
if c.incompleteStructs[tag] {
@@ -3010,7 +3014,7 @@ func (c *typeConv) FuncType(dtype *dwarf.FuncType, pos token.Pos) *FuncType {
for i, f := range dtype.ParamType {
// gcc's DWARF generator outputs a single DotDotDotType parameter for
// function pointers that specify no parameters (e.g. void
- // (*__cgo_0)()). Treat this special case as void. This case is
+ // (*__cgo_0)()). Treat this special case as void. This case is
// invalid according to ISO C anyway (i.e. void (*__cgo_1)(...) is not
// legal).
if _, ok := f.(*dwarf.DotDotDotType); ok && i == 0 {
@@ -3081,7 +3085,7 @@ func (c *typeConv) Struct(dt *dwarf.StructType, pos token.Pos) (expr *ast.Struct
off := int64(0)
// Rename struct fields that happen to be named Go keywords into
- // _{keyword}. Create a map from C ident -> Go ident. The Go ident will
+ // _{keyword}. Create a map from C ident -> Go ident. The Go ident will
// be mangled. Any existing identifier that already has the same name on
// the C-side will cause the Go-mangled version to be prefixed with _.
// (e.g. in a struct with fields '_type' and 'type', the latter would be
@@ -3309,7 +3313,7 @@ func godefsFields(fld []*ast.Field) {
// fieldPrefix returns the prefix that should be removed from all the
// field names when generating the C or Go code. For generated
// C, we leave the names as is (tv_sec, tv_usec), since that's what
-// people are used to seeing in C. For generated Go code, such as
+// people are used to seeing in C. For generated Go code, such as
// package syscall's data structures, we drop a common prefix
// (so sec, usec, which will get turned into Sec, Usec for exporting).
func fieldPrefix(fld []*ast.Field) string {
@@ -3456,7 +3460,7 @@ func (c *typeConv) badCFType(dt *dwarf.TypedefType) bool {
// Tagged pointer support
// Low-bit set means tagged object, next 3 bits (currently)
// define the tagged object class, next 4 bits are for type
-// information for the specific tagged object class. Thus,
+// information for the specific tagged object class. Thus,
// the low byte is for type info, and the rest of a pointer
// (32 or 64-bit) is for payload, whatever the tagged class.
//
diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
index 9cf626c17322c2..93f9027157183b 100644
--- a/src/cmd/cgo/godefs.go
+++ b/src/cmd/cgo/godefs.go
@@ -117,7 +117,7 @@ func (p *Package) godefs(f *File, args []string) string {
var gofmtBuf strings.Builder
// gofmt returns the gofmt-formatted string for an AST node.
-func gofmt(n interface{}) string {
+func gofmt(n any) string {
gofmtBuf.Reset()
err := printer.Fprint(&gofmtBuf, fset, n)
if err != nil {
diff --git a/src/cmd/cgo/internal/test/buildid_linux.go b/src/cmd/cgo/internal/test/buildid_linux.go
index 84d3edb664eb25..7e0fd0fd126a02 100644
--- a/src/cmd/cgo/internal/test/buildid_linux.go
+++ b/src/cmd/cgo/internal/test/buildid_linux.go
@@ -4,9 +4,9 @@
package cgotest
-// Test that we have no more than one build ID. In the past we used
+// Test that we have no more than one build ID. In the past we used
// to generate a separate build ID for each package using cgo, and the
-// linker concatenated them all. We don't want that--we only want
+// linker concatenated them all. We don't want that--we only want
// one.
import (
@@ -42,7 +42,7 @@ sections:
for len(d) > 0 {
// ELF standards differ as to the sizes in
- // note sections. Both the GNU linker and
+ // note sections. Both the GNU linker and
// gold always generate 32-bit sizes, so that
// is what we assume here.
diff --git a/src/cmd/cgo/internal/test/callback.go b/src/cmd/cgo/internal/test/callback.go
index 478bf8294af3a5..8f8dd8fded6f15 100644
--- a/src/cmd/cgo/internal/test/callback.go
+++ b/src/cmd/cgo/internal/test/callback.go
@@ -40,7 +40,7 @@ func nestedCall(f func()) {
callbackMutex.Unlock()
// Pass the address of i because the C function was written to
- // take a pointer. We could pass an int if we felt like
+ // take a pointer. We could pass an int if we felt like
// rewriting the C code.
C.callback(unsafe.Pointer(&i))
diff --git a/src/cmd/cgo/internal/test/callstub_linux_ppc64le.go b/src/cmd/cgo/internal/test/callstub_linux_ppc64le.go
deleted file mode 100644
index 93c29e1c7cdd97..00000000000000
--- a/src/cmd/cgo/internal/test/callstub_linux_ppc64le.go
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2023 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cgotest
-
-// extern int notoc_func(void);
-// int TestPPC64Stubs(void) {
-// return notoc_func();
-// }
-import "C"
-import "testing"
-
-func testPPC64CallStubs(t *testing.T) {
- // Verify the trampolines run on the testing machine. If they
- // do not, or are missing, a crash is expected.
- if C.TestPPC64Stubs() != 0 {
- t.Skipf("This test requires binutils 2.35 or newer.")
- }
-}
diff --git a/src/cmd/cgo/internal/test/callstub_linux_ppc64x.go b/src/cmd/cgo/internal/test/callstub_linux_ppc64x.go
new file mode 100644
index 00000000000000..b0dd69fd64ba39
--- /dev/null
+++ b/src/cmd/cgo/internal/test/callstub_linux_ppc64x.go
@@ -0,0 +1,22 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build (ppc64 || ppc64le) && linux && cgo
+
+package cgotest
+
+// extern int notoc_func(void);
+// int TestPPC64Stubs(void) {
+// return notoc_func();
+// }
+import "C"
+import "testing"
+
+func testPPC64CallStubs(t *testing.T) {
+ // Verify the trampolines run on the testing machine. If they
+ // do not, or are missing, a crash is expected.
+ if C.TestPPC64Stubs() != 0 {
+ t.Skipf("This test requires binutils 2.35 or newer.")
+ }
+}
diff --git a/src/cmd/cgo/internal/test/cgo_darwin_test.go b/src/cmd/cgo/internal/test/cgo_darwin_test.go
new file mode 100644
index 00000000000000..e6c67a13f22f13
--- /dev/null
+++ b/src/cmd/cgo/internal/test/cgo_darwin_test.go
@@ -0,0 +1,12 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build cgo && darwin
+
+package cgotest
+
+import "testing"
+
+func TestIssue76023(t *testing.T) { issue76023(t) }
+func TestGlobalDataDynimport(t *testing.T) { unsignedRelocDynimport(t) }
diff --git a/src/cmd/cgo/internal/test/cgo_test.go b/src/cmd/cgo/internal/test/cgo_test.go
index 5393552e07a4d1..17f9225cf9bf83 100644
--- a/src/cmd/cgo/internal/test/cgo_test.go
+++ b/src/cmd/cgo/internal/test/cgo_test.go
@@ -12,100 +12,106 @@ import "testing"
// so that they can use cgo (import "C").
// These wrappers are here for gotest to find.
-func Test1328(t *testing.T) { test1328(t) }
-func Test1560(t *testing.T) { test1560(t) }
-func Test1635(t *testing.T) { test1635(t) }
-func Test3250(t *testing.T) { test3250(t) }
-func Test3729(t *testing.T) { test3729(t) }
-func Test3775(t *testing.T) { test3775(t) }
-func Test4029(t *testing.T) { test4029(t) }
-func Test4339(t *testing.T) { test4339(t) }
-func Test5227(t *testing.T) { test5227(t) }
-func Test5242(t *testing.T) { test5242(t) }
-func Test5337(t *testing.T) { test5337(t) }
-func Test5548(t *testing.T) { test5548(t) }
-func Test5603(t *testing.T) { test5603(t) }
-func Test5986(t *testing.T) { test5986(t) }
-func Test6390(t *testing.T) { test6390(t) }
-func Test6833(t *testing.T) { test6833(t) }
-func Test6907(t *testing.T) { test6907(t) }
-func Test6907Go(t *testing.T) { test6907Go(t) }
-func Test7560(t *testing.T) { test7560(t) }
-func Test7665(t *testing.T) { test7665(t) }
-func Test7978(t *testing.T) { test7978(t) }
-func Test8092(t *testing.T) { test8092(t) }
-func Test8517(t *testing.T) { test8517(t) }
-func Test8694(t *testing.T) { test8694(t) }
-func Test8756(t *testing.T) { test8756(t) }
-func Test8811(t *testing.T) { test8811(t) }
-func Test9026(t *testing.T) { test9026(t) }
-func Test9510(t *testing.T) { test9510(t) }
-func Test9557(t *testing.T) { test9557(t) }
-func Test10303(t *testing.T) { test10303(t, 10) }
-func Test11925(t *testing.T) { test11925(t) }
-func Test12030(t *testing.T) { test12030(t) }
-func Test14838(t *testing.T) { test14838(t) }
-func Test17065(t *testing.T) { test17065(t) }
-func Test17537(t *testing.T) { test17537(t) }
-func Test18126(t *testing.T) { test18126(t) }
-func Test18720(t *testing.T) { test18720(t) }
-func Test20129(t *testing.T) { test20129(t) }
-func Test20266(t *testing.T) { test20266(t) }
-func Test20369(t *testing.T) { test20369(t) }
-func Test20910(t *testing.T) { test20910(t) }
-func Test21708(t *testing.T) { test21708(t) }
-func Test21809(t *testing.T) { test21809(t) }
-func Test21897(t *testing.T) { test21897(t) }
-func Test22906(t *testing.T) { test22906(t) }
-func Test23356(t *testing.T) { test23356(t) }
-func Test24206(t *testing.T) { test24206(t) }
-func Test25143(t *testing.T) { test25143(t) }
-func Test26066(t *testing.T) { test26066(t) }
-func Test26213(t *testing.T) { test26213(t) }
-func Test27660(t *testing.T) { test27660(t) }
-func Test28896(t *testing.T) { test28896(t) }
-func Test30065(t *testing.T) { test30065(t) }
-func Test32579(t *testing.T) { test32579(t) }
-func Test31891(t *testing.T) { test31891(t) }
-func Test42018(t *testing.T) { test42018(t) }
-func Test45451(t *testing.T) { test45451(t) }
-func Test49633(t *testing.T) { test49633(t) }
-func Test69086(t *testing.T) { test69086(t) }
-func TestAlign(t *testing.T) { testAlign(t) }
-func TestAtol(t *testing.T) { testAtol(t) }
-func TestBlocking(t *testing.T) { testBlocking(t) }
-func TestBoolAlign(t *testing.T) { testBoolAlign(t) }
-func TestCallGoWithString(t *testing.T) { testCallGoWithString(t) }
-func TestCallback(t *testing.T) { testCallback(t) }
-func TestCallbackCallers(t *testing.T) { testCallbackCallers(t) }
-func TestCallbackGC(t *testing.T) { testCallbackGC(t) }
-func TestCallbackPanic(t *testing.T) { testCallbackPanic(t) }
-func TestCallbackPanicLocked(t *testing.T) { testCallbackPanicLocked(t) }
-func TestCallbackPanicLoop(t *testing.T) { testCallbackPanicLoop(t) }
-func TestCallbackStack(t *testing.T) { testCallbackStack(t) }
-func TestCflags(t *testing.T) { testCflags(t) }
-func TestCheckConst(t *testing.T) { testCheckConst(t) }
-func TestConst(t *testing.T) { testConst(t) }
-func TestCthread(t *testing.T) { testCthread(t) }
-func TestEnum(t *testing.T) { testEnum(t) }
-func TestNamedEnum(t *testing.T) { testNamedEnum(t) }
-func TestCastToEnum(t *testing.T) { testCastToEnum(t) }
-func TestErrno(t *testing.T) { testErrno(t) }
-func TestFpVar(t *testing.T) { testFpVar(t) }
-func TestGCC68255(t *testing.T) { testGCC68255(t) }
-func TestHandle(t *testing.T) { testHandle(t) }
-func TestHelpers(t *testing.T) { testHelpers(t) }
-func TestLibgcc(t *testing.T) { testLibgcc(t) }
-func TestMultipleAssign(t *testing.T) { testMultipleAssign(t) }
-func TestNaming(t *testing.T) { testNaming(t) }
-func TestPanicFromC(t *testing.T) { testPanicFromC(t) }
-func TestPrintf(t *testing.T) { testPrintf(t) }
-func TestReturnAfterGrow(t *testing.T) { testReturnAfterGrow(t) }
-func TestReturnAfterGrowFromGo(t *testing.T) { testReturnAfterGrowFromGo(t) }
-func TestSetEnv(t *testing.T) { testSetEnv(t) }
-func TestThreadLock(t *testing.T) { testThreadLockFunc(t) }
-func TestUnsignedInt(t *testing.T) { testUnsignedInt(t) }
-func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) }
+func Test1328(t *testing.T) { test1328(t) }
+func Test1560(t *testing.T) { test1560(t) }
+func Test1635(t *testing.T) { test1635(t) }
+func Test3250(t *testing.T) { test3250(t) }
+func Test3729(t *testing.T) { test3729(t) }
+func Test3775(t *testing.T) { test3775(t) }
+func Test4029(t *testing.T) { test4029(t) }
+func Test4339(t *testing.T) { test4339(t) }
+func Test5227(t *testing.T) { test5227(t) }
+func Test5242(t *testing.T) { test5242(t) }
+func Test5337(t *testing.T) { test5337(t) }
+func Test5548(t *testing.T) { test5548(t) }
+func Test5603(t *testing.T) { test5603(t) }
+func Test5986(t *testing.T) { test5986(t) }
+func Test6390(t *testing.T) { test6390(t) }
+func Test6833(t *testing.T) { test6833(t) }
+func Test6907(t *testing.T) { test6907(t) }
+func Test6907Go(t *testing.T) { test6907Go(t) }
+func Test7560(t *testing.T) { test7560(t) }
+func Test7665(t *testing.T) { test7665(t) }
+func Test7978(t *testing.T) { test7978(t) }
+func Test8092(t *testing.T) { test8092(t) }
+func Test8517(t *testing.T) { test8517(t) }
+func Test8694(t *testing.T) { test8694(t) }
+func Test8756(t *testing.T) { test8756(t) }
+func Test8811(t *testing.T) { test8811(t) }
+func Test9026(t *testing.T) { test9026(t) }
+func Test9510(t *testing.T) { test9510(t) }
+func Test9557(t *testing.T) { test9557(t) }
+func Test10303(t *testing.T) { test10303(t, 10) }
+func Test11925(t *testing.T) { test11925(t) }
+func Test12030(t *testing.T) { test12030(t) }
+func Test14838(t *testing.T) { test14838(t) }
+func Test17065(t *testing.T) { test17065(t) }
+func Test17537(t *testing.T) { test17537(t) }
+func Test18126(t *testing.T) { test18126(t) }
+func Test18720(t *testing.T) { test18720(t) }
+func Test20129(t *testing.T) { test20129(t) }
+func Test20266(t *testing.T) { test20266(t) }
+func Test20369(t *testing.T) { test20369(t) }
+func Test20910(t *testing.T) { test20910(t) }
+func Test21708(t *testing.T) { test21708(t) }
+func Test21809(t *testing.T) { test21809(t) }
+func Test21897(t *testing.T) { test21897(t) }
+func Test22906(t *testing.T) { test22906(t) }
+func Test23356(t *testing.T) { test23356(t) }
+func Test24206(t *testing.T) { test24206(t) }
+func Test25143(t *testing.T) { test25143(t) }
+func Test26066(t *testing.T) { test26066(t) }
+func Test26213(t *testing.T) { test26213(t) }
+func Test27660(t *testing.T) { test27660(t) }
+func Test28896(t *testing.T) { test28896(t) }
+func Test30065(t *testing.T) { test30065(t) }
+func Test32579(t *testing.T) { test32579(t) }
+func Test31891(t *testing.T) { test31891(t) }
+func Test42018(t *testing.T) { test42018(t) }
+func Test45451(t *testing.T) { test45451(t) }
+func Test49633(t *testing.T) { test49633(t) }
+func Test69086(t *testing.T) { test69086(t) }
+func TestAlign(t *testing.T) { testAlign(t) }
+func TestAtol(t *testing.T) { testAtol(t) }
+func TestBlocking(t *testing.T) { testBlocking(t) }
+func TestBoolAlign(t *testing.T) { testBoolAlign(t) }
+func TestCallGoWithString(t *testing.T) { testCallGoWithString(t) }
+func TestCallback(t *testing.T) { testCallback(t) }
+func TestCallbackCallers(t *testing.T) { testCallbackCallers(t) }
+func TestCallbackInInit(t *testing.T) { testCallbackInInit(t) }
+func TestCallbackGC(t *testing.T) { testCallbackGC(t) }
+func TestCallbackPanic(t *testing.T) { testCallbackPanic(t) }
+func TestCallbackPanicLocked(t *testing.T) { testCallbackPanicLocked(t) }
+func TestCallbackPanicLoop(t *testing.T) { testCallbackPanicLoop(t) }
+func TestCallbackStack(t *testing.T) { testCallbackStack(t) }
+func TestCflags(t *testing.T) { testCflags(t) }
+func TestCheckConst(t *testing.T) { testCheckConst(t) }
+func TestConst(t *testing.T) { testConst(t) }
+func TestCthread(t *testing.T) { testCthread(t) }
+func TestEnum(t *testing.T) { testEnum(t) }
+func TestNamedEnum(t *testing.T) { testNamedEnum(t) }
+func TestCastToEnum(t *testing.T) { testCastToEnum(t) }
+func TestErrno(t *testing.T) { testErrno(t) }
+func TestFpVar(t *testing.T) { testFpVar(t) }
+func TestGCC68255(t *testing.T) { testGCC68255(t) }
+func TestHandle(t *testing.T) { testHandle(t) }
+func TestHelpers(t *testing.T) { testHelpers(t) }
+func TestLibgcc(t *testing.T) { testLibgcc(t) }
+func TestMultipleAssign(t *testing.T) { testMultipleAssign(t) }
+func TestNaming(t *testing.T) { testNaming(t) }
+func TestPanicFromC(t *testing.T) { testPanicFromC(t) }
+func TestPrintf(t *testing.T) { testPrintf(t) }
+func TestReturnAfterGrow(t *testing.T) { testReturnAfterGrow(t) }
+func TestReturnAfterGrowFromGo(t *testing.T) { testReturnAfterGrowFromGo(t) }
+func TestSetEnv(t *testing.T) { testSetEnv(t) }
+func TestThreadLock(t *testing.T) { testThreadLockFunc(t) }
+func TestUnsignedInt(t *testing.T) { testUnsignedInt(t) }
+func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) }
+func Test76340(t *testing.T) { test76340(t) }
+func TestDITCgo(t *testing.T) { testDITCgo(t) }
+func TestDITCgoCallback(t *testing.T) { testDITCgoCallback(t) }
+func TestDITCgoCallbackEnableDIT(t *testing.T) { testDITCgoCallbackEnableDIT(t) }
+func TestDITCgoCallbackDisableDIT(t *testing.T) { testDITCgoCallbackDisableDIT(t) }
func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) }
func BenchmarkGoString(b *testing.B) { benchGoString(b) }
diff --git a/src/cmd/cgo/internal/test/cthread_unix.c b/src/cmd/cgo/internal/test/cthread_unix.c
index d0da643158e5f4..1901c6437f7b3d 100644
--- a/src/cmd/cgo/internal/test/cthread_unix.c
+++ b/src/cmd/cgo/internal/test/cthread_unix.c
@@ -56,3 +56,18 @@ callGoInCThread(int max)
return max;
}
+
+static void*
+callbackInInitCThread(void* unused)
+{
+ callbackInInit();
+ return NULL;
+}
+
+void
+callbackInInitC(void)
+{
+ pthread_t thread;
+
+ pthread_create(&thread, NULL, callbackInInitCThread, NULL);
+}
diff --git a/src/cmd/cgo/internal/test/cthread_windows.c b/src/cmd/cgo/internal/test/cthread_windows.c
index 4e52209dee1d18..5bfc6f2c485976 100644
--- a/src/cmd/cgo/internal/test/cthread_windows.c
+++ b/src/cmd/cgo/internal/test/cthread_windows.c
@@ -57,3 +57,19 @@ callGoInCThread(int max)
CloseHandle((HANDLE)thread_id);
return max;
}
+
+__stdcall
+static unsigned int
+callbackInInitCThread(void* unused)
+{
+ callbackInInit();
+ return 0;
+}
+
+void
+callbackInInitC(void)
+{
+ uintptr_t thread_id;
+ thread_id = _beginthreadex(0, 0, callbackInInitCThread, NULL, 0, 0);
+ CloseHandle((HANDLE)thread_id);
+}
diff --git a/src/cmd/cgo/internal/test/gcc68255/a.go b/src/cmd/cgo/internal/test/gcc68255/a.go
index e106dee3ec023d..cc4804b90bd122 100644
--- a/src/cmd/cgo/internal/test/gcc68255/a.go
+++ b/src/cmd/cgo/internal/test/gcc68255/a.go
@@ -3,7 +3,7 @@
// license that can be found in the LICENSE file.
// Test that it's OK to have C code that does nothing other than
-// initialize a global variable. This used to fail with gccgo.
+// initialize a global variable. This used to fail with gccgo.
package gcc68255
diff --git a/src/cmd/cgo/internal/test/issue76023.go b/src/cmd/cgo/internal/test/issue76023.go
new file mode 100644
index 00000000000000..7fe8ae53f7eb66
--- /dev/null
+++ b/src/cmd/cgo/internal/test/issue76023.go
@@ -0,0 +1,27 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin
+
+package cgotest
+
+/*
+#cgo LDFLAGS: -Wl,-undefined,dynamic_lookup
+
+extern void __gotest_cgo_null_api(void) __attribute__((weak_import));
+
+int issue76023(void) {
+ if (__gotest_cgo_null_api) return 1;
+ return 0;
+}
+*/
+import "C"
+import "testing"
+
+func issue76023(t *testing.T) {
+ r := C.issue76023()
+ if r != 0 {
+ t.Error("found __gotest_cgo_null_api")
+ }
+}
diff --git a/src/cmd/cgo/internal/test/issue76861.go b/src/cmd/cgo/internal/test/issue76861.go
new file mode 100644
index 00000000000000..225e2acc3f64be
--- /dev/null
+++ b/src/cmd/cgo/internal/test/issue76861.go
@@ -0,0 +1,12 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build cgo
+
+package cgotest
+
+// Issue 43639: No runtime test needed, make sure package
+// cmd/cgo/internal/test/issue76861 compiles without error.
+
+import _ "cmd/cgo/internal/test/issue76861"
diff --git a/src/cmd/cgo/internal/test/issue76861/a.go b/src/cmd/cgo/internal/test/issue76861/a.go
new file mode 100644
index 00000000000000..18a7bda490c52b
--- /dev/null
+++ b/src/cmd/cgo/internal/test/issue76861/a.go
@@ -0,0 +1,13 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package issue76861
+
+// #cgo CFLAGS: -Wall -Werror
+// void issue76861(void) {}
+import "C"
+
+func Issue76861() {
+ C.issue76861()
+}
diff --git a/src/cmd/cgo/internal/test/linux_ppc64le_test.go b/src/cmd/cgo/internal/test/linux_ppc64le_test.go
deleted file mode 100644
index 67b6b161d695fc..00000000000000
--- a/src/cmd/cgo/internal/test/linux_ppc64le_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright 2023 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build ppc64le && linux && cgo
-
-package cgotest
-
-import "testing"
-
-func TestPPC64CallStubs(t *testing.T) {
- testPPC64CallStubs(t)
-}
diff --git a/src/cmd/cgo/internal/test/linux_ppc64x_test.go b/src/cmd/cgo/internal/test/linux_ppc64x_test.go
new file mode 100644
index 00000000000000..a6e85b6ea34d08
--- /dev/null
+++ b/src/cmd/cgo/internal/test/linux_ppc64x_test.go
@@ -0,0 +1,13 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build (ppc64 || ppc64le) && linux && cgo
+
+package cgotest
+
+import "testing"
+
+func TestPPC64CallStubs(t *testing.T) {
+ testPPC64CallStubs(t)
+}
diff --git a/src/cmd/cgo/internal/test/seh_internal_windows_test.go b/src/cmd/cgo/internal/test/seh_internal_windows_test.go
deleted file mode 100644
index 708ffdc6f60bda..00000000000000
--- a/src/cmd/cgo/internal/test/seh_internal_windows_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2024 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build cgo && windows && internal
-
-package cgotest
-
-import (
- "internal/testenv"
- "testing"
-)
-
-func TestCallbackCallersSEH(t *testing.T) {
- testenv.SkipFlaky(t, 65116)
-}
diff --git a/src/cmd/cgo/internal/test/seh_windows_test.go b/src/cmd/cgo/internal/test/seh_windows_test.go
index 4a8d5bbd4dcc02..7bbed5b04e3f6f 100644
--- a/src/cmd/cgo/internal/test/seh_windows_test.go
+++ b/src/cmd/cgo/internal/test/seh_windows_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-//go:build cgo && windows && !internal
+//go:build cgo && windows
package cgotest
diff --git a/src/cmd/cgo/internal/test/stubtest_linux_ppc64le.S b/src/cmd/cgo/internal/test/stubtest_linux_ppc64le.S
deleted file mode 100644
index 0c519705a508f2..00000000000000
--- a/src/cmd/cgo/internal/test/stubtest_linux_ppc64le.S
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2023 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// When linking C ELFv2 objects, the Go linker may need to insert calling stubs.
-// A call stub is usually needed when the ELFv2 st_other attribute is different
-// between caller and callee.
-//
-// The type of call stub inserted will vary depending on GOPPC64 and the
-// buildmode (e.g pie builds shared code, default builds fixed-position code).
-// CI is set up to run for P8 and P10 machines, and this test is run in both
-// pie and default modes.
-//
-// Several functions are written with interesting st_other attributes, and
-// call each other to test various calling combinations which require stubs.
-//
-// The call tree is as follows, starting from TestPPC64Stubs (A C function):
-// TestPPC64Stubs (compiled PIC by default by Go)
-// notoc_func [called TOC -> NOTOC (but R2 is preserved)]
-// toc_func [called NOTOC -> TOC]
-// notoc_nor2_func [called TOC -> NOTOC]
-// random [dynamic TOC call]
-// random [dynamic NOTOC call]
-//
-// Depending on the GOPPC64/buildmode used, and type of call, one of 7 stubs may need inserted:
-//
-// TOC -> NOTOC: Save R2, call global entry. (valid for any GOPPC64)
-// TOC save slot is rewrittent to restore TOC.
-// NOTOC -> TOC [P10]: A PIC call stub using P10 instructions to call the global entry
-// NOTOC -> TOC [P8]: A PIC call stub using P8 instructions to call the global entry
-//
-// TOC -> dynamic: A PLT call stub is generated which saves R2.
-// TOC save slot is rewritten to restore TOC.
-// NOTOC -> dynamic [P10]: A stub using pcrel instructions is generated.
-// NOTOC -> dynamic [P8/default]: A P8 compatible, non-PIC stub is generated
-// NOTOC -> dynamic [P8/pie]: A P8 compatible, PIC stub is generated
-//
-//
-// Some notes about other cases:
-// TOC -> TOC, NOTOC -> NOTOC, NOTOC -> TOC local calls do not require require call stubs.
-// TOC -> NOTOC (R2 is preserved, st_other==0): A special case where a call stub is not needed.
-
-// This test requires a binutils with power10 and ELFv2 1.5 support. This is earliest verified version.
-.if .gasversion. >= 23500
-
-// A function which does not guarantee R2 is preserved.
-// R2 is clobbered here to ensure the stubs preserve it.
- .globl notoc_nor2_func
- .type notoc_nor2_func, @function
-notoc_nor2_func:
- .localentry notoc_nor2_func,1
- li 2,0
- blr
-
-// A function which expects R2 to hold TOC, and has a distinct local entry.
- .globl toc_func
- .type toc_func, @function
-toc_func:
- addis 2,12,.TOC.-toc_func@ha
- addi 2,2,.TOC.-toc_func@l
- .localentry toc_func, .-toc_func
- mflr 0
- std 0,16(1)
- stdu 1,-32(1)
-
- // Call a NOTOC function which clobbers R2.
- bl notoc_nor2_func
- nop
-
- // Call libc random. This should generate a TOC relative plt stub.
- bl random
- nop
-
- addi 1,1,32
- ld 0,16(1)
- mtlr 0
- blr
-
-// An ELFv2 st_other==0 function. It preserves R2 (TOC), but does not use it.
- .globl notoc_func
- .type notoc_func, @function
-notoc_func:
- // Save R2 and LR and stack a frame.
- mflr 0
- std 0,16(1)
- stdu 1,-32(1)
-
- // Save R2 in TOC save slot.
- std 2,24(1)
-
- // clobber R2
- li 2,0
-
- // Call type2_func. A call stub from notoc to toc should be inserted.
- bl toc_func@notoc
-
- // Call libc random. A notoc plt stub should be inserted.
- bl random@notoc
-
- // Return 0 to indicate the test ran.
- li 3,0
-
- // Restore R2
- ld 2,24(1)
-
- // Restore LR and pop stack
- addi 1,1,32
- ld 0,16(1)
- mtlr 0
- blr
-
-.else
-
-// A stub for older binutils
- .globl notoc_func
- .type notoc_func, @function
-notoc_func:
- // Return 1 to indicate the test was skipped.
- li 3,1
- blr
-
-.endif
diff --git a/src/cmd/cgo/internal/test/stubtest_linux_ppc64x.S b/src/cmd/cgo/internal/test/stubtest_linux_ppc64x.S
new file mode 100644
index 00000000000000..668eb3224e6dc1
--- /dev/null
+++ b/src/cmd/cgo/internal/test/stubtest_linux_ppc64x.S
@@ -0,0 +1,124 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build (ppc64 || ppc64le) && linux && cgo
+
+// When linking C ELFv2 objects, the Go linker may need to insert calling stubs.
+// A call stub is usually needed when the ELFv2 st_other attribute is different
+// between caller and callee.
+//
+// The type of call stub inserted will vary depending on GOPPC64 and the
+// buildmode (e.g pie builds shared code, default builds fixed-position code).
+// CI is set up to run for P8 and P10 machines, and this test is run in both
+// pie and default modes.
+//
+// Several functions are written with interesting st_other attributes, and
+// call each other to test various calling combinations which require stubs.
+//
+// The call tree is as follows, starting from TestPPC64Stubs (A C function):
+// TestPPC64Stubs (compiled PIC by default by Go)
+// notoc_func [called TOC -> NOTOC (but R2 is preserved)]
+// toc_func [called NOTOC -> TOC]
+// notoc_nor2_func [called TOC -> NOTOC]
+// random [dynamic TOC call]
+// random [dynamic NOTOC call]
+//
+// Depending on the GOPPC64/buildmode used, and type of call, one of 7 stubs may need inserted:
+//
+// TOC -> NOTOC: Save R2, call global entry. (valid for any GOPPC64)
+// TOC save slot is rewrittent to restore TOC.
+// NOTOC -> TOC [P10]: A PIC call stub using P10 instructions to call the global entry
+// NOTOC -> TOC [P8]: A PIC call stub using P8 instructions to call the global entry
+//
+// TOC -> dynamic: A PLT call stub is generated which saves R2.
+// TOC save slot is rewritten to restore TOC.
+// NOTOC -> dynamic [P10]: A stub using pcrel instructions is generated.
+// NOTOC -> dynamic [P8/default]: A P8 compatible, non-PIC stub is generated
+// NOTOC -> dynamic [P8/pie]: A P8 compatible, PIC stub is generated
+//
+//
+// Some notes about other cases:
+// TOC -> TOC, NOTOC -> NOTOC, NOTOC -> TOC local calls do not require require call stubs.
+// TOC -> NOTOC (R2 is preserved, st_other==0): A special case where a call stub is not needed.
+
+// This test requires a binutils with power10 and ELFv2 1.5 support. This is earliest verified version.
+.if .gasversion. >= 23500
+
+// A function which does not guarantee R2 is preserved.
+// R2 is clobbered here to ensure the stubs preserve it.
+ .globl notoc_nor2_func
+ .type notoc_nor2_func, @function
+notoc_nor2_func:
+ .localentry notoc_nor2_func,1
+ li 2,0
+ blr
+
+// A function which expects R2 to hold TOC, and has a distinct local entry.
+ .globl toc_func
+ .type toc_func, @function
+toc_func:
+ addis 2,12,.TOC.-toc_func@ha
+ addi 2,2,.TOC.-toc_func@l
+ .localentry toc_func, .-toc_func
+ mflr 0
+ std 0,16(1)
+ stdu 1,-32(1)
+
+ // Call a NOTOC function which clobbers R2.
+ bl notoc_nor2_func
+ nop
+
+ // Call libc random. This should generate a TOC relative plt stub.
+ bl random
+ nop
+
+ addi 1,1,32
+ ld 0,16(1)
+ mtlr 0
+ blr
+
+// An ELFv2 st_other==0 function. It preserves R2 (TOC), but does not use it.
+ .globl notoc_func
+ .type notoc_func, @function
+notoc_func:
+ // Save R2 and LR and stack a frame.
+ mflr 0
+ std 0,16(1)
+ stdu 1,-32(1)
+
+ // Save R2 in TOC save slot.
+ std 2,24(1)
+
+ // clobber R2
+ li 2,0
+
+ // Call type2_func. A call stub from notoc to toc should be inserted.
+ bl toc_func@notoc
+
+ // Call libc random. A notoc plt stub should be inserted.
+ bl random@notoc
+
+ // Return 0 to indicate the test ran.
+ li 3,0
+
+ // Restore R2
+ ld 2,24(1)
+
+ // Restore LR and pop stack
+ addi 1,1,32
+ ld 0,16(1)
+ mtlr 0
+ blr
+
+.else
+
+// A stub for older binutils
+ .globl notoc_func
+ .type notoc_func, @function
+notoc_func:
+ // Return 1 to indicate the test was skipped.
+ li 3,1
+ blr
+
+.endif
diff --git a/src/cmd/cgo/internal/test/test.go b/src/cmd/cgo/internal/test/test.go
index 9626407d882ef2..3f0732b428b2fc 100644
--- a/src/cmd/cgo/internal/test/test.go
+++ b/src/cmd/cgo/internal/test/test.go
@@ -953,13 +953,61 @@ typedef struct {
} issue69086struct;
static int issue690861(issue69086struct* p) { p->b = 1234; return p->c; }
static int issue690862(unsigned long ul1, unsigned long ul2, unsigned int u, issue69086struct s) { return (int)(s.b); }
+
+char issue75751v = 1;
+char * const issue75751p = &issue75751v;
+#define issue75751m issue75751p
+char * const volatile issue75751p2 = &issue75751v;
+#define issue75751m2 issue75751p2
+
+typedef struct { void *t; void *v; } GoInterface;
+extern int exportAny76340Param(GoInterface);
+extern GoInterface exportAny76340Return(int);
+
+int issue76340testFromC(GoInterface obj) {
+ return exportAny76340Param(obj);
+}
+
+GoInterface issue76340returnFromC(int val) {
+ return exportAny76340Return(val);
+}
+
+static void enableDIT() {
+ #ifdef __arm64__
+ __asm__ __volatile__("msr dit, #1");
+ #endif
+}
+
+static void disableDIT() {
+ #ifdef __arm64__
+ __asm__ __volatile__("msr dit, #0");
+ #endif
+}
+
+extern uint8_t ditCallback();
+
+static uint8_t ditCallbackTest() {
+ return ditCallback();
+}
+
+static void ditCallbackEnableDIT() {
+ enableDIT();
+ ditCallback();
+}
+
+static void ditCallbackDisableDIT() {
+ disableDIT();
+ ditCallback();
+}
*/
import "C"
import (
"context"
+ "crypto/subtle"
"fmt"
"internal/asan"
+ "internal/runtime/sys"
"math"
"math/rand"
"os"
@@ -2396,3 +2444,100 @@ func test69086(t *testing.T) {
t.Errorf("call: got %d, want 1234", got)
}
}
+
+// Issue 75751: no runtime test, just make sure it compiles.
+func test75751() int {
+ return int(*C.issue75751m) + int(*C.issue75751m2)
+}
+
+// Issue 76340.
+func test76340(t *testing.T) {
+ var emptyInterface C.GoInterface
+ r1 := C.issue76340testFromC(emptyInterface)
+ if r1 != 0 {
+ t.Errorf("issue76340testFromC with nil interface: got %d, want 0", r1)
+ }
+
+ r2 := C.issue76340returnFromC(42)
+ if r2.t == nil && r2.v == nil {
+ t.Error("issue76340returnFromC(42) returned nil interface")
+ }
+
+ r3 := C.issue76340returnFromC(0)
+ if r3.t != nil || r3.v != nil {
+ t.Errorf("issue76340returnFromC(0) returned non-nil interface: got %v, want nil", r3)
+ }
+}
+
+func testDITCgo(t *testing.T) {
+ if !sys.DITSupported {
+ t.Skip("CPU does not support DIT")
+ }
+
+ ditAlreadyEnabled := sys.DITEnabled()
+ C.enableDIT()
+
+ if ditAlreadyEnabled != sys.DITEnabled() {
+ t.Fatalf("DIT state not preserved across cgo call: before %t, after %t", ditAlreadyEnabled, sys.DITEnabled())
+ }
+
+ subtle.WithDataIndependentTiming(func() {
+ C.disableDIT()
+
+ if !sys.DITEnabled() {
+ t.Fatal("DIT disabled after disabling in cgo call")
+ }
+ })
+}
+
+func testDITCgoCallback(t *testing.T) {
+ if !sys.DITSupported {
+ t.Skip("CPU does not support DIT")
+ }
+
+ ditAlreadyEnabled := sys.DITEnabled()
+
+ subtle.WithDataIndependentTiming(func() {
+ if C.ditCallbackTest() != 1 {
+ t.Fatal("DIT not enabled in cgo callback within WithDataIndependentTiming")
+ }
+ })
+
+ if ditAlreadyEnabled != sys.DITEnabled() {
+ t.Fatalf("DIT state not preserved across cgo callback: before %t, after %t", ditAlreadyEnabled, sys.DITEnabled())
+ }
+}
+
+func testDITCgoCallbackEnableDIT(t *testing.T) {
+ if !sys.DITSupported {
+ t.Skip("CPU does not support DIT")
+ }
+
+ ditAlreadyEnabled := sys.DITEnabled()
+
+ C.ditCallbackEnableDIT()
+
+ if ditAlreadyEnabled != sys.DITEnabled() {
+ t.Fatalf("DIT state not preserved across cgo callback: before %t, after %t", ditAlreadyEnabled, sys.DITEnabled())
+ }
+}
+
+func testDITCgoCallbackDisableDIT(t *testing.T) {
+ if !sys.DITSupported {
+ t.Skip("CPU does not support DIT")
+ }
+
+ ditAlreadyEnabled := sys.DITEnabled()
+
+ subtle.WithDataIndependentTiming(func() {
+ C.ditCallbackDisableDIT()
+
+ if !sys.DITEnabled() {
+ t.Fatal("DIT disabled after disabling in cgo call")
+ }
+ })
+
+ if ditAlreadyEnabled != sys.DITEnabled() {
+ t.Fatalf("DIT state not preserved across cgo callback: before %t, after %t", ditAlreadyEnabled, sys.DITEnabled())
+ }
+}
diff --git a/src/cmd/cgo/internal/test/testx.go b/src/cmd/cgo/internal/test/testx.go
index 9a63b9e10087ad..1c5ac99bd28596 100644
--- a/src/cmd/cgo/internal/test/testx.go
+++ b/src/cmd/cgo/internal/test/testx.go
@@ -11,6 +11,7 @@
package cgotest
import (
+ "internal/runtime/sys"
"runtime"
"runtime/cgo"
"runtime/debug"
@@ -25,6 +26,7 @@ import (
// threads
extern void doAdd(int, int);
extern int callGoInCThread(int);
+extern void callbackInInitC(void);
// issue 1328
void IntoC(void);
@@ -595,3 +597,54 @@ func test49633(t *testing.T) {
t.Errorf("msg = %q, want 'hello'", v.msg)
}
}
+
+//export exportAny76340Param
+func exportAny76340Param(obj any) C.int {
+ if obj == nil {
+ return 0
+ }
+
+ return 1
+}
+
+//export exportAny76340Return
+func exportAny76340Return(val C.int) any {
+ if val == 0 {
+ return nil
+ }
+
+ return int(val)
+}
+
+//export ditCallback
+func ditCallback() uint8 {
+ if sys.DITEnabled() {
+ return 1
+ }
+ return 0
+}
+
+// Test C calling back into Go before init is done.
+// In particular, this does not trigger false race
+// (the fix is in CL 746581).
+
+var callbackInInitVar int
+
+func init() {
+ C.callbackInInitC()
+ callbackInInitVar = 123
+}
+
+var callbackInInitChan = make(chan int)
+
+//export callbackInInit
+func callbackInInit() {
+ if callbackInInitVar != 123 {
+ panic("callbackInInitVar not initialized to 123")
+ }
+ callbackInInitChan <- 1
+}
+
+func testCallbackInInit(t *testing.T) {
+ <-callbackInInitChan // make sure callbackInInit runs
+}
diff --git a/src/cmd/cgo/internal/test/unsigned_reloc_darwin.go b/src/cmd/cgo/internal/test/unsigned_reloc_darwin.go
new file mode 100644
index 00000000000000..3db5ed19161ea8
--- /dev/null
+++ b/src/cmd/cgo/internal/test/unsigned_reloc_darwin.go
@@ -0,0 +1,41 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build darwin
+
+package cgotest
+
+/*
+#include
+
+// Global function pointer to a dynamically-linked libc function.
+// When compiled to a Mach-O object, this produces a RELOC_UNSIGNED
+// relocation targeting the external symbol _puts.
+int (*_cgo_test_dynref_puts)(const char *) = puts;
+
+static int cgo_call_dynref(void) {
+ if (_cgo_test_dynref_puts == 0) {
+ return -1;
+ }
+ // Call the resolved function pointer. puts returns a non-negative
+ // value on success.
+ return _cgo_test_dynref_puts("cgo unsigned reloc test");
+}
+*/
+import "C"
+
+import "testing"
+
+// unsignedRelocDynimport verifies that the Go internal linker correctly
+// handles Mach-O UNSIGNED relocations targeting dynamic import symbols.
+// The C preamble above contains a global function pointer initialized
+// to puts, which produces a RELOC_UNSIGNED relocation to the external
+// symbol _puts. If the linker can't handle this, the test binary
+// won't link at all.
+func unsignedRelocDynimport(t *testing.T) {
+ got := C.cgo_call_dynref()
+ if got < 0 {
+ t.Fatal("C function pointer to puts not resolved")
+ }
+}
diff --git a/src/cmd/cgo/internal/testcarchive/carchive_test.go b/src/cmd/cgo/internal/testcarchive/carchive_test.go
index 7c6e2b1d848def..a04959e85979fc 100644
--- a/src/cmd/cgo/internal/testcarchive/carchive_test.go
+++ b/src/cmd/cgo/internal/testcarchive/carchive_test.go
@@ -502,7 +502,7 @@ func TestInstall(t *testing.T) {
func TestEarlySignalHandler(t *testing.T) {
switch GOOS {
- case "darwin", "ios":
+ case "ios":
switch GOARCH {
case "arm64":
t.Skipf("skipping on %s/%s; see https://golang.org/issue/13701", GOOS, GOARCH)
@@ -1351,7 +1351,7 @@ func TestDeepStack(t *testing.T) {
}
func BenchmarkCgoCallbackMainThread(b *testing.B) {
- // Benchmark for calling into Go fron C main thread.
+ // Benchmark for calling into Go from C main thread.
// See issue #68587.
//
// It uses a subprocess, which is a C binary that calls
diff --git a/src/cmd/cgo/internal/testcshared/cshared_test.go b/src/cmd/cgo/internal/testcshared/cshared_test.go
index c01f5cf2cfbe7a..abaa75587eb3a1 100644
--- a/src/cmd/cgo/internal/testcshared/cshared_test.go
+++ b/src/cmd/cgo/internal/testcshared/cshared_test.go
@@ -462,7 +462,7 @@ func GoFunc%d() {}
// NumberOfNames is the number of functions exported with a unique name.
// NumberOfFunctions can be higher than that because it also counts
- // functions exported only by ordinal, a unique number asigned by the linker,
+ // functions exported only by ordinal, a unique number assigned by the linker,
// and linkers might add an unknown number of their own ordinal-only functions.
if wantAll {
if e.NumberOfNames <= uint32(exportedSymbols) {
@@ -942,3 +942,29 @@ func TestIssue68411(t *testing.T) {
t.Error("missing functions")
}
}
+
+func TestSymbolicFunctions(t *testing.T) {
+ // Test that we can build a c-shared library with -Wl,-Bsymbolic-functions,
+ // see issue 80632.
+ globalSkip(t)
+ testenv.MustHaveGoBuild(t)
+ testenv.MustHaveCGO(t)
+ testenv.MustHaveBuildMode(t, "c-shared")
+ if GOOS != "linux" {
+ t.Skip("Skipping on non-Linux OS")
+ }
+
+ t.Parallel()
+
+ tmpdir := t.TempDir()
+ libname := filepath.Join(tmpdir, "libbsymbolic.a")
+
+ run(t,
+ nil,
+ "go", "build",
+ "-buildmode=c-shared",
+ "-installsuffix", "testcshared",
+ "-ldflags=-extldflags=-Wl,-Bsymbolic-functions",
+ "-o", libname, "./libgo",
+ )
+}
diff --git a/src/cmd/cgo/internal/testerrors/argposition_test.go b/src/cmd/cgo/internal/testerrors/argposition_test.go
index 035552127b8f46..30447e7a68807e 100644
--- a/src/cmd/cgo/internal/testerrors/argposition_test.go
+++ b/src/cmd/cgo/internal/testerrors/argposition_test.go
@@ -96,7 +96,7 @@ func TestArgumentsPositions(t *testing.T) {
t.Fatal(err)
}
fset := token.NewFileSet()
- f, err := parser.ParseFile(fset, "", mainProcessed, parser.AllErrors)
+ f, err := parser.ParseFile(fset, "", mainProcessed, parser.AllErrors|parser.SkipObjectResolution)
if err != nil {
fmt.Println(err)
return
diff --git a/src/cmd/cgo/internal/testerrors/errors_test.go b/src/cmd/cgo/internal/testerrors/errors_test.go
index 80d2c402cec0c2..6af7c94168a14b 100644
--- a/src/cmd/cgo/internal/testerrors/errors_test.go
+++ b/src/cmd/cgo/internal/testerrors/errors_test.go
@@ -72,7 +72,7 @@ func expect(t *testing.T, errors []*regexp.Regexp, files ...string) {
defer os.RemoveAll(dir)
dst := filepath.Join(dir, strings.TrimSuffix(files[0], ".go"))
- args := []string{"build", "-gcflags=-L -e", "-o=" + dst} // TODO(gri) no need for -gcflags=-L if go tool is adjusted
+ args := []string{"build", "-gcflags=-e", "-o=" + dst}
for _, file := range files {
args = append(args, path(file))
}
@@ -152,7 +152,6 @@ func TestToleratesOptimizationFlag(t *testing.T) {
"",
"-O",
} {
- cflags := cflags
t.Run(cflags, func(t *testing.T) {
testenv.MustHaveGoBuild(t)
testenv.MustHaveCGO(t)
diff --git a/src/cmd/cgo/internal/testerrors/ptr_test.go b/src/cmd/cgo/internal/testerrors/ptr_test.go
index beba0d26ac1818..bc1cc1c6e08ed8 100644
--- a/src/cmd/cgo/internal/testerrors/ptr_test.go
+++ b/src/cmd/cgo/internal/testerrors/ptr_test.go
@@ -14,6 +14,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "regexp"
"slices"
"strings"
"sync/atomic"
@@ -24,15 +25,16 @@ var tmp = flag.String("tmp", "", "use `dir` for temporary files and do not clean
// ptrTest is the tests without the boilerplate.
type ptrTest struct {
- name string // for reporting
- c string // the cgo comment
- c1 string // cgo comment forced into non-export cgo file
- imports []string // a list of imports
- support string // supporting functions
- body string // the body of the main function
- extra []extra // extra files
- fail bool // whether the test should fail
- expensive bool // whether the test requires the expensive check
+ name string // for reporting
+ c string // the cgo comment
+ c1 string // cgo comment forced into non-export cgo file
+ imports []string // a list of imports
+ support string // supporting functions
+ body string // the body of the main function
+ extra []extra // extra files
+ fail bool // whether the test should fail
+ expensive bool // whether the test requires the expensive check
+ errTextRegexp string // error text regexp; if empty, use the pattern `.*unpinned Go.*`
}
type extra struct {
@@ -489,6 +491,27 @@ var ptrTests = []ptrTest{
body: `i := 0; a := &[2]unsafe.Pointer{nil, unsafe.Pointer(&i)}; C.f45(&a[0])`,
fail: true,
},
+ {
+ // Passing a Go map as argument to C.
+ name: "argmap",
+ c: `void f46(void* p) {}`,
+ imports: []string{"unsafe"},
+ body: `m := map[int]int{0: 1,}; C.f46(unsafe.Pointer(&m))`,
+ fail: true,
+ errTextRegexp: `.*argument of cgo function has Go pointer to unpinned Go map`,
+ },
+ {
+ // Returning a Go map to C.
+ name: "retmap",
+ c: `extern void f47();`,
+ support: `//export GoMap47
+ func GoMap47() map[int]int { return map[int]int{0: 1,} }`,
+ body: `C.f47()`,
+ c1: `extern void* GoMap47();
+ void f47() { GoMap47(); }`,
+ fail: true,
+ errTextRegexp: `.*result of Go function GoMap47 called from cgo is unpinned Go map or points to unpinned Go map.*`,
+ },
}
func TestPointerChecks(t *testing.T) {
@@ -519,7 +542,6 @@ func TestPointerChecks(t *testing.T) {
// after testOne finishes.
var pending int32
for _, pt := range ptrTests {
- pt := pt
t.Run(pt.name, func(t *testing.T) {
atomic.AddInt32(&pending, +1)
defer func() {
@@ -690,11 +712,17 @@ func testOne(t *testing.T, pt ptrTest, exe, exe2 string) {
}
buf, err := runcmd(cgocheck)
+
+ var pattern string = pt.errTextRegexp
+ if pt.errTextRegexp == "" {
+ pattern = `.*unpinned Go.*`
+ }
+
if pt.fail {
if err == nil {
t.Logf("%s", buf)
t.Fatalf("did not fail as expected")
- } else if !bytes.Contains(buf, []byte("Go pointer")) {
+ } else if ok, _ := regexp.Match(pattern, buf); !ok {
t.Logf("%s", buf)
t.Fatalf("did not print expected error (failed with %v)", err)
}
diff --git a/src/cmd/cgo/internal/testfortran/fortran_test.go b/src/cmd/cgo/internal/testfortran/fortran_test.go
index 9ccf916d841d66..771590e34adaa5 100644
--- a/src/cmd/cgo/internal/testfortran/fortran_test.go
+++ b/src/cmd/cgo/internal/testfortran/fortran_test.go
@@ -47,14 +47,19 @@ func TestFortran(t *testing.T) {
switch runtime.GOOS {
case "darwin":
libExt = "dylib"
- case "aix":
+ case "aix", "openbsd", "windows":
libExt = "a"
}
- libPath, err := exec.Command(fc, append([]string{"-print-file-name=libgfortran." + libExt}, fcExtra...)...).CombinedOutput()
+ libName := "libgfortran." + libExt
+ b, err := exec.Command(fc, append([]string{"-print-file-name=" + libName}, fcExtra...)...).CombinedOutput()
if err != nil {
- t.Errorf("error invoking %s: %s", fc, err)
+ t.Fatalf("error invoking %s: %s", fc, err)
}
- libDir := filepath.Dir(string(libPath))
+ libPath := strings.TrimSpace(string(b))
+ if libPath == libName {
+ t.Fatalf("Failed to get full library path for %q", libName)
+ }
+ libDir := filepath.Dir(libPath)
cgoLDFlags := os.Getenv("CGO_LDFLAGS")
cgoLDFlags += " -L " + libDir
if runtime.GOOS != "aix" {
diff --git a/src/cmd/cgo/internal/testout/out_test.go b/src/cmd/cgo/internal/testout/out_test.go
index e8ea5092a3562b..ff506c8cd97fbd 100644
--- a/src/cmd/cgo/internal/testout/out_test.go
+++ b/src/cmd/cgo/internal/testout/out_test.go
@@ -18,6 +18,32 @@ import (
"testing"
)
+// TestDisallowSmuggledCode tests that
+// docstrings do not smuggle code into
+// files generated by Cgo.
+func TestDisallowSmuggledCode(t *testing.T) {
+ testenv.MustHaveGoRun(t)
+ testenv.MustHaveCGO(t)
+ objDir := cgo(t, "comments.go")
+
+ file, err := os.Open(filepath.Join(objDir, "_cgo_export.h"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if strings.Contains(line, `"Hello, I am exploiting CVE-2025-61732!\n"`) {
+ t.Fatalf(`got %q, want ""`, line)
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ t.Fatal(err)
+ }
+}
+
type methodAlign struct {
Method string
Align int
@@ -43,23 +69,7 @@ var wantAligns = map[string]int{
func TestAligned(t *testing.T) {
testenv.MustHaveGoRun(t)
testenv.MustHaveCGO(t)
-
- testdata, err := filepath.Abs("testdata")
- if err != nil {
- t.Fatal(err)
- }
-
- objDir := t.TempDir()
-
- cmd := testenv.Command(t, testenv.GoToolPath(t), "tool", "cgo",
- "-objdir", objDir,
- filepath.Join(testdata, "aligned.go"))
- cmd.Stderr = new(bytes.Buffer)
-
- err = cmd.Run()
- if err != nil {
- t.Fatalf("%#q: %v\n%s", cmd, err, cmd.Stderr)
- }
+ objDir := cgo(t, "aligned.go")
haveAligns, err := parseAlign(filepath.Join(objDir, "_cgo_export.c"))
if err != nil {
@@ -84,6 +94,28 @@ func TestAligned(t *testing.T) {
}
}
+// cgo executes 'go tool cgo' on testFile
+// and returns the objdir containing the
+// generated files.
+func cgo(t *testing.T, testFile string) string {
+ objDir := t.TempDir()
+ testdata, err := filepath.Abs("testdata")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cmd := testenv.Command(t, testenv.GoToolPath(t), "tool", "cgo",
+ "-objdir", objDir,
+ filepath.Join(testdata, testFile))
+
+ cmd.Stderr = new(bytes.Buffer)
+ if err = cmd.Run(); err != nil {
+ t.Fatalf("%#q: %v\n%s", cmd, err, cmd.Stderr)
+ }
+
+ return objDir
+}
+
func parseAlign(filename string) ([]methodAlign, error) {
file, err := os.Open(filename)
if err != nil {
diff --git a/src/cmd/cgo/internal/testout/testdata/comments.go b/src/cmd/cgo/internal/testout/testdata/comments.go
new file mode 100644
index 00000000000000..c1fcaeea4a2f1c
--- /dev/null
+++ b/src/cmd/cgo/internal/testout/testdata/comments.go
@@ -0,0 +1,47 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+/*
+#include
+
+#pragma once
+
+extern void go_func();
+
+
+void print(const char *str) {
+ printf("%s", str);
+ go_func();
+}
+*/
+import "C"
+import "fmt"
+
+func main() {
+ str := C.CString("Hello from C\n")
+ C.print(str)
+}
+
+// \
+/*
+
+#ifndef AUTO_PRINT_H
+#define AUTO_PRINT_H
+
+#include
+
+__attribute__((constructor))
+static void inject(void) {
+ printf("Hello, I am exploiting CVE-2025-61732!\n");
+}
+
+#endif
+
+/* */
+//export go_func
+func go_func() {
+ fmt.Println("Hello from Go")
+}
diff --git a/src/cmd/cgo/internal/testplugin/plugin_test.go b/src/cmd/cgo/internal/testplugin/plugin_test.go
index 2afb542ec4f17c..4f0458b7dee472 100644
--- a/src/cmd/cgo/internal/testplugin/plugin_test.go
+++ b/src/cmd/cgo/internal/testplugin/plugin_test.go
@@ -37,7 +37,7 @@ func TestMain(m *testing.M) {
var tmpDir string
// prettyPrintf prints lines with tmpDir sanitized.
-func prettyPrintf(format string, args ...interface{}) {
+func prettyPrintf(format string, args ...any) {
s := fmt.Sprintf(format, args...)
if tmpDir != "" {
s = strings.ReplaceAll(s, tmpDir, "$TMPDIR")
@@ -430,3 +430,14 @@ func TestIssue75102(t *testing.T) {
goCmd(t, "build", "-o", "issue75102.exe", "./issue75102/main.go")
run(t, "./issue75102.exe")
}
+
+func TestIssue81303(t *testing.T) {
+ // Issue 81303: the itab copies of a plugin must not hide the itabs
+ // of the host for the same interface/type pairs.
+ globalSkip(t)
+ goCmd(t, "build", "-buildmode=plugin", "-o", "issue81303p1.so", "./issue81303/plugin1.go")
+ goCmd(t, "build", "-buildmode=plugin", "-o", "issue81303p2.so", "./issue81303/plugin2.go")
+ goCmd(t, "build", "-buildmode=plugin", "-o", "issue81303p3.so", "./issue81303/plugin3.go")
+ goCmd(t, "build", "-o", "issue81303.exe", "./issue81303/main.go")
+ run(t, "./issue81303.exe")
+}
diff --git a/src/cmd/cgo/internal/testplugin/testdata/issue81303/main.go b/src/cmd/cgo/internal/testplugin/testdata/issue81303/main.go
new file mode 100644
index 00000000000000..7b2ea1595f18ac
--- /dev/null
+++ b/src/cmd/cgo/internal/testplugin/testdata/issue81303/main.go
@@ -0,0 +1,39 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Issue 81303: a plugin has its own copies of the itabs of the host.
+// The runtime added these copies to the itab table as second entries
+// for the same interface/type pairs. After the table grew, a lookup
+// could return the copy from the plugin. A type switch compares the
+// itab with the itab of the host, so it took the default case.
+//
+// Each plugin imports package p, which uses go/ast, so each plugin
+// adds a copy of every go/ast itab of the host. The host walks a
+// syntax tree before and after each plugin loads. The three plugins
+// are the same because plugin.Open loads a plugin path only once.
+
+package main
+
+import (
+ "log"
+ "plugin"
+
+ "testplugin/issue81303/p"
+)
+
+func main() {
+ p.Walk()
+ for _, name := range []string{"issue81303p1.so", "issue81303p2.so", "issue81303p3.so"} {
+ pl, err := plugin.Open(name)
+ if err != nil {
+ log.Fatal(err)
+ }
+ f, err := pl.Lookup("F")
+ if err != nil {
+ log.Fatal(err)
+ }
+ f.(func())()
+ p.Walk()
+ }
+}
diff --git a/src/cmd/cgo/internal/testplugin/testdata/issue81303/p/p.go b/src/cmd/cgo/internal/testplugin/testdata/issue81303/p/p.go
new file mode 100644
index 00000000000000..960d35e4376d5d
--- /dev/null
+++ b/src/cmd/cgo/internal/testplugin/testdata/issue81303/p/p.go
@@ -0,0 +1,48 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "log"
+)
+
+const src = `package p
+
+import "fmt"
+
+type T struct{ x int }
+
+func (t *T) M(n int) (r int) {
+ defer func() { r++ }()
+ go fmt.Println(n)
+ if n > 0 {
+ r = n
+ } else {
+ r = -n
+ }
+ for i := 0; i < n; i++ {
+ r += t.x
+ }
+ switch v := any(n).(type) {
+ case int:
+ r = v
+ }
+ return
+}
+`
+
+// Walk parses src and walks the syntax tree. ast.Walk converts each
+// node to ast.Node, which looks up the itab in the itab table, and
+// then compares that itab with the itab of the host in a type switch.
+func Walk() {
+ f, err := parser.ParseFile(token.NewFileSet(), "src.go", src, parser.SkipObjectResolution)
+ if err != nil {
+ log.Fatal(err)
+ }
+ ast.Inspect(f, func(ast.Node) bool { return true })
+}
diff --git a/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin1.go b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin1.go
new file mode 100644
index 00000000000000..c97a98583684ea
--- /dev/null
+++ b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin1.go
@@ -0,0 +1,11 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "testplugin/issue81303/p"
+
+func main() {}
+
+func F() { p.Walk() }
diff --git a/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin2.go b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin2.go
new file mode 100644
index 00000000000000..c97a98583684ea
--- /dev/null
+++ b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin2.go
@@ -0,0 +1,11 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "testplugin/issue81303/p"
+
+func main() {}
+
+func F() { p.Walk() }
diff --git a/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin3.go b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin3.go
new file mode 100644
index 00000000000000..c97a98583684ea
--- /dev/null
+++ b/src/cmd/cgo/internal/testplugin/testdata/issue81303/plugin3.go
@@ -0,0 +1,11 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "testplugin/issue81303/p"
+
+func main() {}
+
+func F() { p.Walk() }
diff --git a/src/cmd/cgo/internal/testsanitizers/asan_test.go b/src/cmd/cgo/internal/testsanitizers/asan_test.go
index cb7d857280416f..a2d420402018e7 100644
--- a/src/cmd/cgo/internal/testsanitizers/asan_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/asan_test.go
@@ -48,7 +48,6 @@ func TestASAN(t *testing.T) {
{src: "arena_fail.go", memoryAccessError: "use-after-poison", errorLocation: "arena_fail.go:26", experiments: []string{"arenas"}},
}
for _, tc := range cases {
- tc := tc
name := strings.TrimSuffix(tc.src, ".go")
t.Run(name, func(t *testing.T) {
t.Parallel()
diff --git a/src/cmd/cgo/internal/testsanitizers/cc_test.go b/src/cmd/cgo/internal/testsanitizers/cc_test.go
index f2239fb9cffc2e..cfd670585637b2 100644
--- a/src/cmd/cgo/internal/testsanitizers/cc_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/cc_test.go
@@ -13,6 +13,7 @@ package sanitizers_test
import (
"bytes"
+ "context"
"encoding/json"
"errors"
"fmt"
@@ -110,6 +111,24 @@ func appendExperimentEnv(cmd *exec.Cmd, experiments []string) {
cmd.Env = append(cmd.Env, "GOEXPERIMENT="+exps)
}
+func appendASANOptions(cmd *exec.Cmd, opts ...string) {
+ if cmd.Env == nil {
+ cmd.Env = cmd.Environ()
+ }
+ var asanOptions string
+ for _, evar := range cmd.Env {
+ name, value, ok := strings.Cut(evar, "=")
+ if ok && name == "ASAN_OPTIONS" {
+ asanOptions = value
+ }
+ }
+ if asanOptions != "" {
+ asanOptions += ":"
+ }
+ asanOptions += strings.Join(opts, ":")
+ cmd.Env = append(cmd.Env, "ASAN_OPTIONS="+asanOptions)
+}
+
// mustRun executes t and fails cmd with a well-formatted message if it fails.
func mustRun(t *testing.T, cmd *exec.Cmd) {
t.Helper()
@@ -137,7 +156,7 @@ func mustRun(t *testing.T, cmd *exec.Cmd) {
}
// cc returns a cmd that executes `$(go env CC) $(go env GOGCCFLAGS) $args`.
-func cc(args ...string) (*exec.Cmd, error) {
+func cc(ctx context.Context, args ...string) (*exec.Cmd, error) {
CC, err := goEnv("CC")
if err != nil {
return nil, err
@@ -186,7 +205,7 @@ func cc(args ...string) (*exec.Cmd, error) {
flags = append(flags, GOGCCFLAGS[start:])
}
- cmd := exec.Command(CC, flags...)
+ cmd := exec.CommandContext(ctx, CC, flags...)
cmd.Args = append(cmd.Args, args...)
return cmd, nil
}
@@ -211,7 +230,7 @@ func compilerVersion() (version, error) {
compiler.err = func() error {
compiler.name = "unknown"
- cmd, err := cc("--version")
+ cmd, err := cc(context.Background(), "--version")
if err != nil {
return err
}
@@ -224,7 +243,7 @@ func compilerVersion() (version, error) {
var match [][]byte
if bytes.HasPrefix(out, []byte("gcc")) {
compiler.name = "gcc"
- cmd, err := cc("-dumpfullversion", "-dumpversion")
+ cmd, err := cc(context.Background(), "-dumpfullversion", "-dumpversion")
if err != nil {
return err
}
@@ -464,6 +483,11 @@ int LLVMFuzzerTestOneInput(char *data, size_t size) {
`)
func (c *config) checkCSanitizer() (skip bool, err error) {
+ // The sanitizer probes compile and run tiny C programs. If either step
+ // takes longer than this, treat the C sanitizer configuration as broken
+ // instead of letting the package-level test timeout fire.
+ probeTimeout := 20 * time.Second
+
dir, err := os.MkdirTemp("", c.sanitizer)
if err != nil {
return false, fmt.Errorf("failed to create temp directory: %v", err)
@@ -481,14 +505,26 @@ func (c *config) checkCSanitizer() (skip bool, err error) {
}
dst := filepath.Join(dir, "return0")
- cmd, err := cc(c.cFlags...)
+ compileCtx, cancelCompile := context.WithTimeout(context.Background(), probeTimeout)
+ defer cancelCompile()
+ cmd, err := cc(compileCtx, c.cFlags...)
if err != nil {
return false, err
}
cmd.Args = append(cmd.Args, c.ldFlags...)
cmd.Args = append(cmd.Args, "-o", dst, src)
+ if c.sanitizer == "address" {
+ // This is only a compiler support probe for ASAN. Some libasan versions
+ // run a slow LeakSanitizer check at exit even for this no-op C program,
+ // which can hang TestASAN before it starts testing Go binaries.
+ appendASANOptions(cmd, "leak_check_at_exit=0")
+ }
+ makeHangProne(cmd)
out, err := cmd.CombinedOutput()
if err != nil {
+ if errors.Is(compileCtx.Err(), context.DeadlineExceeded) {
+ return true, fmt.Errorf("%#q timed out after %v", cmd, probeTimeout)
+ }
if bytes.Contains(out, []byte("-fsanitize")) &&
(bytes.Contains(out, []byte("unrecognized")) ||
bytes.Contains(out, []byte("unsupported"))) {
@@ -502,7 +538,20 @@ func (c *config) checkCSanitizer() (skip bool, err error) {
return false, nil
}
- if out, err := exec.Command(dst).CombinedOutput(); err != nil {
+ runCtx, cancelRun := context.WithTimeout(context.Background(), probeTimeout)
+ defer cancelRun()
+ cmd = exec.CommandContext(runCtx, dst)
+ makeHangProne(cmd)
+ if c.sanitizer == "address" {
+ // Match the compile-time probe above: avoid libasan's implicit LSan exit
+ // scan for this standalone C binary. The explicit LSAN tests still run
+ // with leak checking enabled.
+ appendASANOptions(cmd, "leak_check_at_exit=0")
+ }
+ if out, err := cmd.CombinedOutput(); err != nil {
+ if errors.Is(runCtx.Err(), context.DeadlineExceeded) {
+ return true, fmt.Errorf("%#q timed out after %v", cmd, probeTimeout)
+ }
if os.IsNotExist(err) {
return true, fmt.Errorf("%#q failed to produce executable: %v", cmd, err)
}
@@ -537,7 +586,7 @@ func (c *config) checkRuntime() (skip bool, err error) {
// libcgo.h sets CGO_TSAN if it detects TSAN support in the C compiler.
// Dump the preprocessor defines to check that works.
// (Sometimes it doesn't: see https://golang.org/issue/15983.)
- cmd, err := cc(c.cFlags...)
+ cmd, err := cc(context.Background(), c.cFlags...)
if err != nil {
return false, err
}
@@ -594,8 +643,14 @@ func newTempDir(t *testing.T) *tempDir {
// leak.
func hangProneCmd(name string, arg ...string) *exec.Cmd {
cmd := exec.Command(name, arg...)
+ makeHangProne(cmd)
+ return cmd
+}
+
+// makeHangProne configures cmd to receive SIGKILL when the parent process receives SIGINT.
+// See [hangProneCmd] for details.
+func makeHangProne(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
- return cmd
}
diff --git a/src/cmd/cgo/internal/testsanitizers/cshared_test.go b/src/cmd/cgo/internal/testsanitizers/cshared_test.go
index 0cf094ead79e66..76559679a00542 100644
--- a/src/cmd/cgo/internal/testsanitizers/cshared_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/cshared_test.go
@@ -54,7 +54,6 @@ func TestShared(t *testing.T) {
}
for _, tc := range cases {
- tc := tc
name := strings.TrimSuffix(tc.src, ".go")
//The memory sanitizer tests require support for the -msan option.
if tc.sanitizer == "memory" && !platform.MSanSupported(GOOS, GOARCH) {
@@ -83,7 +82,7 @@ func TestShared(t *testing.T) {
}
dstBin := dir.Join(name)
- cmd, err := cc(config.cFlags...)
+ cmd, err := cc(t.Context(), config.cFlags...)
if err != nil {
t.Fatal(err)
}
diff --git a/src/cmd/cgo/internal/testsanitizers/libfuzzer_test.go b/src/cmd/cgo/internal/testsanitizers/libfuzzer_test.go
index 9f548d66eace03..b53c6aafcc1b7f 100644
--- a/src/cmd/cgo/internal/testsanitizers/libfuzzer_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/libfuzzer_test.go
@@ -43,7 +43,6 @@ func TestLibFuzzer(t *testing.T) {
{goSrc: "libfuzzer2.go", cSrc: "libfuzzer2.c", expectedError: "panic: found it"},
}
for _, tc := range cases {
- tc := tc
name := strings.TrimSuffix(tc.goSrc, ".go")
t.Run(name, func(t *testing.T) {
t.Parallel()
@@ -57,7 +56,7 @@ func TestLibFuzzer(t *testing.T) {
mustRun(t, config.goCmd("build", "-buildmode=c-archive", "-o", archivePath, srcPath(tc.goSrc)))
// build C code (if any) and link with Go code
- cmd, err := cc(config.cFlags...)
+ cmd, err := cc(t.Context(), config.cFlags...)
if err != nil {
t.Fatalf("error running cc: %v", err)
}
diff --git a/src/cmd/cgo/internal/testsanitizers/msan_test.go b/src/cmd/cgo/internal/testsanitizers/msan_test.go
index 554cceaff88669..811a1438397eb0 100644
--- a/src/cmd/cgo/internal/testsanitizers/msan_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/msan_test.go
@@ -62,7 +62,6 @@ func TestMSAN(t *testing.T) {
{src: "arena_fail.go", wantErr: true, experiments: []string{"arenas"}},
}
for _, tc := range cases {
- tc := tc
name := strings.TrimSuffix(tc.src, ".go")
t.Run(name, func(t *testing.T) {
t.Parallel()
diff --git a/src/cmd/cgo/internal/testsanitizers/tsan_test.go b/src/cmd/cgo/internal/testsanitizers/tsan_test.go
index 589db2e6bc4e34..81fc00900f9e37 100644
--- a/src/cmd/cgo/internal/testsanitizers/tsan_test.go
+++ b/src/cmd/cgo/internal/testsanitizers/tsan_test.go
@@ -59,7 +59,6 @@ func TestTSAN(t *testing.T) {
{src: "tsan_tracebackctxt", needsRuntime: true}, // Subdirectory
}
for _, tc := range cases {
- tc := tc
name := strings.TrimSuffix(tc.src, ".go")
t.Run(name, func(t *testing.T) {
t.Parallel()
diff --git a/src/cmd/cgo/internal/testshared/shared_test.go b/src/cmd/cgo/internal/testshared/shared_test.go
index 3d401b604ebbba..f785f423ac36fd 100644
--- a/src/cmd/cgo/internal/testshared/shared_test.go
+++ b/src/cmd/cgo/internal/testshared/shared_test.go
@@ -1175,10 +1175,10 @@ func TestStd(t *testing.T) {
tmpDir := t.TempDir()
// Use a temporary pkgdir to not interfere with other tests, and not write to GOROOT.
// Cannot use goCmd as it runs with cloned GOROOT which is incomplete.
- runWithEnv(t, "building std", []string{"GOROOT=" + oldGOROOT},
+ runWithEnv(t, "building std", []string{"GOROOT=" + oldGOROOT, "GOEXPERIMENT=nosimd"},
filepath.Join(oldGOROOT, "bin", "go"), "install", "-buildmode=shared", "-pkgdir="+tmpDir, "std")
// Issue #58966.
- runWithEnv(t, "testing issue #58966", []string{"GOROOT=" + oldGOROOT},
+ runWithEnv(t, "testing issue #58966", []string{"GOROOT=" + oldGOROOT, "GOEXPERIMENT=nosimd"},
filepath.Join(oldGOROOT, "bin", "go"), "run", "-linkshared", "-pkgdir="+tmpDir, "./issue58966/main.go")
}
diff --git a/src/cmd/cgo/internal/teststdio/stdio_test.go b/src/cmd/cgo/internal/teststdio/stdio_test.go
index 9150281ae87ff3..f9e7fe507989f4 100644
--- a/src/cmd/cgo/internal/teststdio/stdio_test.go
+++ b/src/cmd/cgo/internal/teststdio/stdio_test.go
@@ -56,7 +56,6 @@ func TestTestRun(t *testing.T) {
"fib.go",
"hello.go",
} {
- file := file
wantFile := strings.Replace(file, ".go", ".out", 1)
t.Run(file, func(t *testing.T) {
cmd := exec.Command(testenv.GoToolPath(t), "run", file)
diff --git a/src/cmd/cgo/internal/teststdio/testdata/fib.go b/src/cmd/cgo/internal/teststdio/testdata/fib.go
index 96173683353151..69147880c20df2 100644
--- a/src/cmd/cgo/internal/teststdio/testdata/fib.go
+++ b/src/cmd/cgo/internal/teststdio/testdata/fib.go
@@ -5,7 +5,7 @@
//go:build test_run
// Compute Fibonacci numbers with two goroutines
-// that pass integers back and forth. No actual
+// that pass integers back and forth. No actual
// concurrency, just threads and synchronization
// and foreign code on multiple pthreads.
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
index 5e08427daf9cc2..452840a4cfbb19 100644
--- a/src/cmd/cgo/main.go
+++ b/src/cmd/cgo/main.go
@@ -72,8 +72,8 @@ type File struct {
ExpFunc []*ExpFunc // exported functions for this file
Name map[string]*Name // map from Go name to Name
NamePos map[*Name]token.Pos // map from Name to position of the first reference
- NoCallbacks map[string]bool // C function names that with #cgo nocallback directive
- NoEscapes map[string]bool // C function names that with #cgo noescape directive
+ NoCallbacks map[string]bool // C function names with #cgo nocallback directive
+ NoEscapes map[string]bool // C function names with #cgo noescape directive
Edit *edit.Buffer
debugs []*debug // debug data from iterations of gccDebug. Initialized by File.loadDebug.
@@ -148,7 +148,7 @@ type ExpFunc struct {
// A TypeRepr contains the string representation of a type.
type TypeRepr struct {
Repr string
- FormatArgs []interface{}
+ FormatArgs []any
}
// A Type collects information about a type in both the C and Go worlds.
@@ -320,6 +320,10 @@ func main() {
conf.Mode &^= printer.SourcePos
}
+ if *objDir == "" {
+ *objDir = "_obj"
+ }
+
args := flag.Args()
if len(args) < 1 {
usage()
@@ -446,14 +450,6 @@ func main() {
cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6])
- if *objDir == "" {
- *objDir = "_obj"
- }
- // make sure that `objDir` directory exists, so that we can write
- // all the output files there.
- os.MkdirAll(*objDir, 0o700)
- *objDir += string(filepath.Separator)
-
for i, input := range goFiles {
f := fs[i]
p.Translate(f)
diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
index a2bcdf89c5ad44..151000f5813c3d 100644
--- a/src/cmd/cgo/out.go
+++ b/src/cmd/cgo/out.go
@@ -33,15 +33,15 @@ var (
// writeDefs creates output files to be compiled by gc and gcc.
func (p *Package) writeDefs() {
var fgo2, fc io.Writer
- f := creat(*objDir + "_cgo_gotypes.go")
+ f := creat("_cgo_gotypes.go")
defer f.Close()
fgo2 = f
if *gccgo {
- f := creat(*objDir + "_cgo_defun.c")
+ f := creat("_cgo_defun.c")
defer f.Close()
fc = f
}
- fm := creat(*objDir + "_cgo_main.c")
+ fm := creat("_cgo_main.c")
var gccgoInit strings.Builder
@@ -50,7 +50,7 @@ func (p *Package) writeDefs() {
fmt.Fprintf(fgo2, "//go:cgo_ldflag %q\n", arg)
}
} else {
- fflg := creat(*objDir + "_cgo_flags")
+ fflg := creat("_cgo_flags")
for _, arg := range p.LdFlags {
fmt.Fprintf(fflg, "_CGO_LDFLAGS=%s\n", arg)
}
@@ -242,8 +242,8 @@ func (p *Package) writeDefs() {
}
}
- fgcc := creat(*objDir + "_cgo_export.c")
- fgcch := creat(*objDir + "_cgo_export.h")
+ fgcc := creat("_cgo_export.c")
+ fgcch := creat("_cgo_export.h")
if *gccgo {
p.writeGccgoExports(fgo2, fm, fgcc, fgcch)
} else {
@@ -263,8 +263,11 @@ func (p *Package) writeDefs() {
}
if *exportHeader != "" && len(p.ExpFunc) > 0 {
- fexp := creat(*exportHeader)
- fgcch, err := os.Open(*objDir + "_cgo_export.h")
+ fexp, err := os.Create(*exportHeader)
+ if err != nil {
+ fatalf("%s", err)
+ }
+ fgcch, err := os.Open(filepath.Join(outputDir(), "_cgo_export.h"))
if err != nil {
fatalf("%s", err)
}
@@ -457,6 +460,36 @@ func checkImportSymName(s string) {
// Also assumes that gc convention is to word-align the
// input and output parameters.
func (p *Package) structType(n *Name) (string, int64) {
+ // It's possible for us to see a type with a top-level const here,
+ // which will give us an unusable struct type. See #75751.
+ // The top-level const will always appear as a final qualifier,
+ // constructed by typeConv.loadType in the dwarf.QualType case.
+ // The top-level const is meaningless here and can simply be removed.
+ stripConst := func(s string) string {
+ i := strings.LastIndex(s, "const")
+ if i == -1 {
+ return s
+ }
+
+ // A top-level const can only be followed by other qualifiers.
+ if r, ok := strings.CutSuffix(s, "const"); ok {
+ return strings.TrimSpace(r)
+ }
+
+ var nonConst []string
+ for _, f := range strings.Fields(s[i:]) {
+ switch f {
+ case "const":
+ case "restrict", "volatile":
+ nonConst = append(nonConst, f)
+ default:
+ return s
+ }
+ }
+
+ return strings.TrimSpace(s[:i]) + " " + strings.Join(nonConst, " ")
+ }
+
var buf strings.Builder
fmt.Fprint(&buf, "struct {\n")
off := int64(0)
@@ -468,7 +501,7 @@ func (p *Package) structType(n *Name) (string, int64) {
}
c := t.Typedef
if c == "" {
- c = t.C.String()
+ c = stripConst(t.C.String())
}
fmt.Fprintf(&buf, "\t\t%s p%d;\n", c, i)
off += t.Size
@@ -484,7 +517,7 @@ func (p *Package) structType(n *Name) (string, int64) {
fmt.Fprintf(&buf, "\t\tchar __pad%d[%d];\n", off, pad)
off += pad
}
- fmt.Fprintf(&buf, "\t\t%s r;\n", t.C)
+ fmt.Fprintf(&buf, "\t\t%s r;\n", stripConst(t.C.String()))
off += t.Size
}
if off%p.PtrSize != 0 {
@@ -649,13 +682,15 @@ func (p *Package) writeDefsFunc(fgo2 io.Writer, n *Name, callsMalloc *bool) {
if p.noEscapes[n.C] && p.noCallbacks[n.C] {
touchFunc = "_Cgo_keepalive"
}
- fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n")
- if d.Type.Params != nil {
+
+ if len(paramnames) > 0 {
+ fmt.Fprintf(fgo2, "\tif _Cgo_always_false {\n")
for _, name := range paramnames {
fmt.Fprintf(fgo2, "\t\t%s(%s)\n", touchFunc, name)
}
+ fmt.Fprintf(fgo2, "\t}\n")
}
- fmt.Fprintf(fgo2, "\t}\n")
+
fmt.Fprintf(fgo2, "\treturn\n")
fmt.Fprintf(fgo2, "}\n")
}
@@ -665,8 +700,8 @@ func (p *Package) writeOutput(f *File, srcfile string) {
base := srcfile
base = strings.TrimSuffix(base, ".go")
base = filepath.Base(base)
- fgo1 := creat(*objDir + base + ".cgo1.go")
- fgcc := creat(*objDir + base + ".cgo2.c")
+ fgo1 := creat(base + ".cgo1.go")
+ fgcc := creat(base + ".cgo2.c")
p.GoFiles = append(p.GoFiles, base+".cgo1.go")
p.GccFiles = append(p.GccFiles, base+".cgo2.c")
@@ -751,13 +786,13 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
// We're trying to write a gcc struct that matches gc's layout.
// Use packed attribute to force no padding in this struct in case
// gcc has different packing requirements.
- fmt.Fprintf(fgcc, "\t%s %v *_cgo_a = v;\n", ctype, p.packedAttribute())
- if n.FuncType.Result != nil {
- // Save the stack top for use below.
- fmt.Fprintf(fgcc, "\tchar *_cgo_stktop = _cgo_topofstack();\n")
- }
tr := n.FuncType.Result
+ if (n.Kind != "macro" && len(n.FuncType.Params) > 0) || tr != nil {
+ fmt.Fprintf(fgcc, "\t%s %v *_cgo_a = v;\n", ctype, p.packedAttribute())
+ }
if tr != nil {
+ // Save the stack top for use below.
+ fmt.Fprintf(fgcc, "\tchar *_cgo_stktop = _cgo_topofstack();\n")
fmt.Fprintf(fgcc, "\t__typeof__(_cgo_a->r) _cgo_r;\n")
}
fmt.Fprintf(fgcc, "\t_cgo_tsan_acquire();\n")
@@ -787,7 +822,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
fmt.Fprintf(fgcc, "\t_cgo_errno = errno;\n")
}
fmt.Fprintf(fgcc, "\t_cgo_tsan_release();\n")
- if n.FuncType.Result != nil {
+ if tr != nil {
// The cgo call may have caused a stack copy (via a callback).
// Adjust the return value pointer appropriately.
fmt.Fprintf(fgcc, "\t_cgo_a = (void*)((char*)_cgo_a + (_cgo_topofstack() - _cgo_stktop));\n")
@@ -951,7 +986,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
npad := 0
// the align is at least 1 (for char)
maxAlign := int64(1)
- argField := func(typ ast.Expr, namePat string, args ...interface{}) {
+ argField := func(typ ast.Expr, namePat string, args ...any) {
name := fmt.Sprintf(namePat, args...)
t := p.cgoType(typ)
if off%t.Align != 0 {
@@ -1144,6 +1179,10 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) {
if !p.hasPointer(nil, atype, false) {
return
}
+
+ // Use the export'ed file/line in error messages.
+ pos := fset.Position(exp.Func.Pos())
+ fmt.Fprintf(fgo2, "//line %s:%d\n", pos.Filename, pos.Line)
fmt.Fprintf(fgo2, "\t_cgoCheckResult(a.r%d)\n", i)
})
}
@@ -1359,7 +1398,7 @@ func gccgoToSymbol(ppath string) string {
fatalf("unable to locate gccgo: %v", err)
}
}
- gccgoMangler, err = pkgpath.ToSymbolFunc(cmd, *objDir)
+ gccgoMangler, err = pkgpath.ToSymbolFunc(cmd, outputDir())
if err != nil {
fatalf("%v", err)
}
@@ -1406,7 +1445,7 @@ func forFieldList(fl *ast.FieldList, fn func(int, string, ast.Expr)) {
}
}
-func c(repr string, args ...interface{}) *TypeRepr {
+func c(repr string, args ...any) *TypeRepr {
return &TypeRepr{repr, args}
}
@@ -1522,6 +1561,9 @@ func (p *Package) doCgoType(e ast.Expr, m map[ast.Expr]bool) *Type {
if t.Name == "error" {
return &Type{Size: 2 * p.PtrSize, Align: p.PtrSize, C: c("GoInterface")}
}
+ if t.Name == "any" {
+ return &Type{Size: 2 * p.PtrSize, Align: p.PtrSize, C: c("GoInterface")}
+ }
if r, ok := goTypes[t.Name]; ok {
return goTypesFixup(r)
}
diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
index 23b4a414db7562..bf34d61e902dca 100644
--- a/src/cmd/cgo/util.go
+++ b/src/cmd/cgo/util.go
@@ -10,6 +10,7 @@ import (
"go/token"
"os"
"os/exec"
+ "path/filepath"
"slices"
)
@@ -75,7 +76,7 @@ func lineno(pos token.Pos) string {
}
// Die with an error message.
-func fatalf(msg string, args ...interface{}) {
+func fatalf(msg string, args ...any) {
// If we've already printed other errors, they might have
// caused the fatal condition. Assume they're enough.
if nerrors == 0 {
@@ -86,7 +87,7 @@ func fatalf(msg string, args ...interface{}) {
var nerrors int
-func error_(pos token.Pos, msg string, args ...interface{}) {
+func error_(pos token.Pos, msg string, args ...any) {
nerrors++
if pos.IsValid() {
fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
@@ -97,10 +98,17 @@ func error_(pos token.Pos, msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "\n")
}
+// create creates a file in the output directory.
func creat(name string) *os.File {
- f, err := os.Create(name)
+ f, err := os.Create(filepath.Join(outputDir(), name))
if err != nil {
fatalf("%s", err)
}
return f
}
+
+// outputDir returns the output directory, making sure that it exists.
+func outputDir() string {
+ os.MkdirAll(*objDir, 0o700)
+ return *objDir
+}
diff --git a/src/cmd/compile/README.md b/src/cmd/compile/README.md
index 02429d5688659e..de667e60e37f2e 100644
--- a/src/cmd/compile/README.md
+++ b/src/cmd/compile/README.md
@@ -157,7 +157,8 @@ The format of the export data file has gone through a number of
iterations. Its current form is called "unified", and it is a
serialized representation of an object graph, with an index allowing
lazy decoding of parts of the whole (since most imports are used to
-provide only a handful of symbols).
+provide only a handful of symbols). See [here](internal/noder/README.md)
+for details on making changes to unified IR.
The GOROOT repository contains a reader and a writer for the unified
format; it encodes from/decodes to the compiler's IR.
diff --git a/src/cmd/compile/abi-internal.md b/src/cmd/compile/abi-internal.md
index eae230dc070d86..535acedd84b4ea 100644
--- a/src/cmd/compile/abi-internal.md
+++ b/src/cmd/compile/abi-internal.md
@@ -643,8 +643,6 @@ Registers R20 - R21, R23 – R28, R30 - R31, F16 – F31 are permanent scratch r
Register R2 is reserved and never used.
-Register R20, R21 is Used by runtime.duffcopy, runtime.duffzero.
-
Special-purpose registers used within Go generated code and Go assembly code
are as follows:
@@ -653,7 +651,6 @@ are as follows:
| R0 | Zero value | Same | Same |
| R1 | Link register | Link register | Scratch |
| R3 | Stack pointer | Same | Same |
-| R20,R21 | Scratch | Scratch | Used by duffcopy, duffzero |
| R22 | Current goroutine | Same | Same |
| R29 | Closure context pointer | Same | Same |
| R30, R31 | used by the assembler | Same | Same |
@@ -799,17 +796,15 @@ assembly code are as follows:
| X2 | Stack pointer | Same | Same |
| X3 | Global pointer | Same | Used by dynamic linker |
| X4 | TLS (thread pointer) | TLS | Scratch |
-| X24,X25 | Scratch | Scratch | Used by duffcopy, duffzero |
| X26 | Closure context pointer | Scratch | Scratch |
| X27 | Current goroutine | Same | Same |
| X31 | Scratch | Scratch | Scratch |
*Rationale*: These register meanings are compatible with Go’s
-stack-based calling convention. Context register X20 will change to X26,
-duffcopy, duffzero register will change to X24, X25 before this register ABI been adopted.
+stack-based calling convention.
X10 – X17, X8, X9, X18 – X23, is the same order as A0 – A7, S0 – S7 in platform ABI.
F10 – F17, F8, F9, F18 – F23, is the same order as FA0 – FA7, FS0 – FS7 in platform ABI.
-X8 – X23, F8 – F15 are used for compressed instruction (RVC) which will benefit code size in the future.
+X8 – X23, F8 – F15 are used for compressed instruction (RVC) which benefits code size.
#### Stack layout
@@ -833,6 +828,51 @@ The riscv64 has Zicsr extension for control and status register (CSR) and
treated as scratch register.
All bits in CSR are system flags and are not modified by Go.
+### s390x architecture
+
+The s390x architecture uses R2 – R9 for integer arguments and integer results.
+
+It uses F0 – F15 for floating-point arguments and results.
+
+Special-purpose registers used within Go generated code and Go assembly code
+are as follows:
+
+| Register | Call meaning | Return meaning | Body meaning |
+| --- | --- | --- | --- |
+| R0 | Zero value | Same | Same |
+| R1 | Scratch | Scratch | Scratch |
+| R10, R11 | used by the assembler | Same | Same |
+| R12 | Closure context pointer | Same | Same |
+| R13 | Current goroutine | Same | Same |
+| R14 | Link register | Link register | Scratch |
+| R15 | Stack pointer | Same | Same |
+
+*Rationale*: These register meanings are compatible with Go’s stack-based
+calling convention.
+
+#### Stack layout
+
+The stack pointer, R15, grows down and is aligned to 8 bytes.
+
+A function's stack frame, after the frame is created, is laid out as
+follows:
+
+ +------------------------------+
+ | ... locals ... |
+ | ... outgoing arguments ... |
+ | return PC | ← R15 points to
+ +------------------------------+ ↓ lower addresses
+
+This stack layout is used by both register-based (ABIInternal) and
+stack-based (ABI0) calling conventions.
+
+The "return PC" is loaded to the link register R14, as part of the
+s390x `BL` operation.
+
+#### Flags
+The s390x architecture maintains a single condition code (CC) field in the Program Status Word (PSW).
+Go-generated code sets and tests this condition code to control conditional branches.
+
## Future directions
### Spill path improvements
diff --git a/src/cmd/compile/default.pgo b/src/cmd/compile/default.pgo
index 2c2588704f3f6e..06e63b47811f70 100644
Binary files a/src/cmd/compile/default.pgo and b/src/cmd/compile/default.pgo differ
diff --git a/src/cmd/compile/doc.go b/src/cmd/compile/doc.go
index 81e6189a262381..dee509218965f8 100644
--- a/src/cmd/compile/doc.go
+++ b/src/cmd/compile/doc.go
@@ -198,9 +198,15 @@ all other compiler directives are of the form
// is peeled off the same way if it is valid. Anything before that is considered the filename
// (possibly including blanks and colons). Invalid line or column values are reported as errors.
//
+// A relative filename is interpreted relative to the directory of the file
+// containing the directive. Absolute filenames are used as given. A filename
+// inherited from a previous directive (the empty-filename form //line :line:col)
+// is reused verbatim and is not re-resolved.
+//
// Examples:
//
-// //line foo.go:10 the filename is foo.go, and the line number is 10 for the next line
+// //line foo.go:10 the (relative) filename is foo.go, and the line number is 10 for the next line
+// //line ../foo.go:10 relative filenames are resolved against the directive's source directory
// //line C:foo.go:10 colons are permitted in filenames, here the filename is C:foo.go, and the line is 10
// //line a:100 :10 blanks are permitted in filenames, here the filename is " a:100 " (excluding quotes)
// /*line :10:20*/x the position of x is in the current file with line number 10 and column number 20
diff --git a/src/cmd/compile/internal/abi/abiutils.go b/src/cmd/compile/internal/abi/abiutils.go
index dacaad3f300833..0e1cf88757ea21 100644
--- a/src/cmd/compile/internal/abi/abiutils.go
+++ b/src/cmd/compile/internal/abi/abiutils.go
@@ -150,12 +150,12 @@ func appendParamTypes(rts []*types.Type, t *types.Type) []*types.Type {
if w == 0 {
return rts
}
- if t.IsScalar() || t.IsPtrShaped() {
+ if t.IsScalar() || t.IsPtrShaped() || t.IsSIMD() {
if t.IsComplex() {
c := types.FloatForComplex(t)
return append(rts, c, c)
} else {
- if int(t.Size()) <= types.RegSize {
+ if int(t.Size()) <= types.RegSize || t.IsSIMD() {
return append(rts, t)
}
// assume 64bit int on 32-bit machine
@@ -199,6 +199,9 @@ func appendParamOffsets(offsets []int64, at int64, t *types.Type) ([]int64, int6
if w == 0 {
return offsets, at
}
+ if t.IsSIMD() {
+ return append(offsets, at), at + w
+ }
if t.IsScalar() || t.IsPtrShaped() {
if t.IsComplex() || int(t.Size()) > types.RegSize { // complex and *int64 on 32-bit
s := w / 2
@@ -275,13 +278,6 @@ func NewABIConfig(iRegsCount, fRegsCount int, offsetForLocals int64, which uint8
return &ABIConfig{offsetForLocals: offsetForLocals, regAmounts: RegAmounts{iRegsCount, fRegsCount}, which: obj.ABI(which)}
}
-// Copy returns config.
-//
-// TODO(mdempsky): Remove.
-func (config *ABIConfig) Copy() *ABIConfig {
- return config
-}
-
// Which returns the ABI number
func (config *ABIConfig) Which() obj.ABI {
return config.which
@@ -521,11 +517,11 @@ func (state *assignState) allocateRegs(regs []RegIndex, t *types.Type) []RegInde
}
ri := state.rUsed.intRegs
rf := state.rUsed.floatRegs
- if t.IsScalar() || t.IsPtrShaped() {
+ if t.IsScalar() || t.IsPtrShaped() || t.IsSIMD() {
if t.IsComplex() {
regs = append(regs, RegIndex(rf+state.rTotal.intRegs), RegIndex(rf+1+state.rTotal.intRegs))
rf += 2
- } else if t.IsFloat() {
+ } else if t.IsFloat() || t.IsSIMD() {
regs = append(regs, RegIndex(rf+state.rTotal.intRegs))
rf += 1
} else {
diff --git a/src/cmd/compile/internal/abt/avlint32.go b/src/cmd/compile/internal/abt/avlint32.go
index ddfca346a2f751..e41a6c0ca40bd8 100644
--- a/src/cmd/compile/internal/abt/avlint32.go
+++ b/src/cmd/compile/internal/abt/avlint32.go
@@ -28,7 +28,7 @@ type T struct {
type node32 struct {
// Standard conventions hold for left = smaller, right = larger
left, right *node32
- data interface{}
+ data any
key int32
height_ int8
}
@@ -49,21 +49,21 @@ func (t *T) IsSingle() bool {
// VisitInOrder applies f to the key and data pairs in t,
// with keys ordered from smallest to largest.
-func (t *T) VisitInOrder(f func(int32, interface{})) {
+func (t *T) VisitInOrder(f func(int32, any)) {
if t.root == nil {
return
}
t.root.visitInOrder(f)
}
-func (n *node32) nilOrData() interface{} {
+func (n *node32) nilOrData() any {
if n == nil {
return nil
}
return n.data
}
-func (n *node32) nilOrKeyAndData() (k int32, d interface{}) {
+func (n *node32) nilOrKeyAndData() (k int32, d any) {
if n == nil {
k = NOT_KEY32
d = nil
@@ -83,7 +83,7 @@ func (n *node32) height() int8 {
// Find returns the data associated with x in the tree, or
// nil if x is not in the tree.
-func (t *T) Find(x int32) interface{} {
+func (t *T) Find(x int32) any {
return t.root.find(x).nilOrData()
}
@@ -92,7 +92,7 @@ func (t *T) Find(x int32) interface{} {
// x was already a key in the tree. The previous data associated
// with x is returned, and is nil if x was not previously a
// key in the tree.
-func (t *T) Insert(x int32, data interface{}) interface{} {
+func (t *T) Insert(x int32, data any) any {
if x == NOT_KEY32 {
panic("Cannot use sentinel value -0x80000000 as key")
}
@@ -105,7 +105,7 @@ func (t *T) Insert(x int32, data interface{}) interface{} {
} else {
newroot, n, o = n.aInsert(x)
}
- var r interface{}
+ var r any
if o != nil {
r = o.data
} else {
@@ -121,7 +121,7 @@ func (t *T) Copy() *T {
return &u
}
-func (t *T) Delete(x int32) interface{} {
+func (t *T) Delete(x int32) any {
n := t.root
if n == nil {
return nil
@@ -135,7 +135,7 @@ func (t *T) Delete(x int32) interface{} {
return d.data
}
-func (t *T) DeleteMin() (int32, interface{}) {
+func (t *T) DeleteMin() (int32, any) {
n := t.root
if n == nil {
return NOT_KEY32, nil
@@ -149,7 +149,7 @@ func (t *T) DeleteMin() (int32, interface{}) {
return d.key, d.data
}
-func (t *T) DeleteMax() (int32, interface{}) {
+func (t *T) DeleteMax() (int32, any) {
n := t.root
if n == nil {
return NOT_KEY32, nil
@@ -172,7 +172,7 @@ func (t *T) Size() int {
// not be symmetric. If f returns nil, then the key and data are not
// added to the result. If f itself is nil, then whatever value was
// already present in the smaller set is used.
-func (t *T) Intersection(u *T, f func(x, y interface{}) interface{}) *T {
+func (t *T) Intersection(u *T, f func(x, y any) any) *T {
if t.Size() == 0 || u.Size() == 0 {
return &T{}
}
@@ -227,7 +227,7 @@ func (t *T) Intersection(u *T, f func(x, y interface{}) interface{}) *T {
// is given by f(t's data, u's data) -- f need not be symmetric. If f returns nil,
// then the key and data are not added to the result. If f itself is nil, then
// whatever value was already present in the larger set is used.
-func (t *T) Union(u *T, f func(x, y interface{}) interface{}) *T {
+func (t *T) Union(u *T, f func(x, y any) any) *T {
if t.Size() == 0 {
return u
}
@@ -284,7 +284,7 @@ func (t *T) Union(u *T, f func(x, y interface{}) interface{}) *T {
// of f applied to data corresponding to equal keys. If f returns nil
// (or if f is nil) then the key+data are excluded, as usual. If f
// returns not-nil, then that key+data pair is inserted. instead.
-func (t *T) Difference(u *T, f func(x, y interface{}) interface{}) *T {
+func (t *T) Difference(u *T, f func(x, y any) any) *T {
if t.Size() == 0 {
return &T{}
}
@@ -365,7 +365,7 @@ func (t *node32) equals(u *node32) bool {
return it.done() == iu.done()
}
-func (t *T) Equiv(u *T, eqv func(x, y interface{}) bool) bool {
+func (t *T) Equiv(u *T, eqv func(x, y any) bool) bool {
if t == u {
return true
}
@@ -375,7 +375,7 @@ func (t *T) Equiv(u *T, eqv func(x, y interface{}) bool) bool {
return t.root.equiv(u.root, eqv)
}
-func (t *node32) equiv(u *node32, eqv func(x, y interface{}) bool) bool {
+func (t *node32) equiv(u *node32, eqv func(x, y any) bool) bool {
if t == u {
return true
}
@@ -404,7 +404,7 @@ type Iterator struct {
it iterator
}
-func (it *Iterator) Next() (int32, interface{}) {
+func (it *Iterator) Next() (int32, any) {
x := it.it.next()
if x == nil {
return NOT_KEY32, nil
@@ -461,37 +461,37 @@ func (it *iterator) next() *node32 {
// Min returns the minimum element of t.
// If t is empty, then (NOT_KEY32, nil) is returned.
-func (t *T) Min() (k int32, d interface{}) {
+func (t *T) Min() (k int32, d any) {
return t.root.min().nilOrKeyAndData()
}
// Max returns the maximum element of t.
// If t is empty, then (NOT_KEY32, nil) is returned.
-func (t *T) Max() (k int32, d interface{}) {
+func (t *T) Max() (k int32, d any) {
return t.root.max().nilOrKeyAndData()
}
// Glb returns the greatest-lower-bound-exclusive of x and the associated
// data. If x has no glb in the tree, then (NOT_KEY32, nil) is returned.
-func (t *T) Glb(x int32) (k int32, d interface{}) {
+func (t *T) Glb(x int32) (k int32, d any) {
return t.root.glb(x, false).nilOrKeyAndData()
}
// GlbEq returns the greatest-lower-bound-inclusive of x and the associated
// data. If x has no glbEQ in the tree, then (NOT_KEY32, nil) is returned.
-func (t *T) GlbEq(x int32) (k int32, d interface{}) {
+func (t *T) GlbEq(x int32) (k int32, d any) {
return t.root.glb(x, true).nilOrKeyAndData()
}
// Lub returns the least-upper-bound-exclusive of x and the associated
// data. If x has no lub in the tree, then (NOT_KEY32, nil) is returned.
-func (t *T) Lub(x int32) (k int32, d interface{}) {
+func (t *T) Lub(x int32) (k int32, d any) {
return t.root.lub(x, false).nilOrKeyAndData()
}
// LubEq returns the least-upper-bound-inclusive of x and the associated
// data. If x has no lubEq in the tree, then (NOT_KEY32, nil) is returned.
-func (t *T) LubEq(x int32) (k int32, d interface{}) {
+func (t *T) LubEq(x int32) (k int32, d any) {
return t.root.lub(x, true).nilOrKeyAndData()
}
@@ -499,7 +499,7 @@ func (t *node32) isLeaf() bool {
return t.left == nil && t.right == nil && t.height_ == LEAF_HEIGHT
}
-func (t *node32) visitInOrder(f func(int32, interface{})) {
+func (t *node32) visitInOrder(f func(int32, any)) {
if t.left != nil {
t.left.visitInOrder(f)
}
diff --git a/src/cmd/compile/internal/abt/avlint32_test.go b/src/cmd/compile/internal/abt/avlint32_test.go
index 7fa9ed4fd68236..71962445f2b1d8 100644
--- a/src/cmd/compile/internal/abt/avlint32_test.go
+++ b/src/cmd/compile/internal/abt/avlint32_test.go
@@ -317,7 +317,7 @@ func applicIterator(te *testing.T, x []int32) {
}
}
-func equiv(a, b interface{}) bool {
+func equiv(a, b any) bool {
sa, sb := a.(*sstring), b.(*sstring)
return *sa == *sb
}
@@ -450,16 +450,16 @@ func TestEquals(t *testing.T) {
[]int32{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2})
}
-func first(x, y interface{}) interface{} {
+func first(x, y any) any {
return x
}
-func second(x, y interface{}) interface{} {
+func second(x, y any) any {
return y
}
-func alwaysNil(x, y interface{}) interface{} {
+func alwaysNil(x, y any) any {
return nil
}
-func smaller(x, y interface{}) interface{} {
+func smaller(x, y any) any {
xi, _ := strconv.Atoi(fmt.Sprint(x))
yi, _ := strconv.Atoi(fmt.Sprint(y))
if xi < yi {
@@ -560,7 +560,7 @@ func (s *sstring) String() string {
return s.s
}
-func stringer(s string) interface{} {
+func stringer(s string) any {
return &sstring{s}
}
diff --git a/src/cmd/compile/internal/amd64/simdssa.go b/src/cmd/compile/internal/amd64/simdssa.go
new file mode 100644
index 00000000000000..12f20f15ea4628
--- /dev/null
+++ b/src/cmd/compile/internal/amd64/simdssa.go
@@ -0,0 +1,4389 @@
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -arch amd64 -xedPath $XED_PATH go_amd64.yaml types.yaml categories.yaml'; DO NOT EDIT.
+
+package amd64
+
+import (
+ "cmd/compile/internal/ssa"
+ "cmd/compile/internal/ssagen"
+ "cmd/internal/obj"
+ "cmd/internal/obj/x86"
+)
+
+func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
+ var p *obj.Prog
+ switch v.Op {
+ case ssa.OpAMD64VAESIMC128,
+ ssa.OpAMD64VPABSB128,
+ ssa.OpAMD64VPABSB256,
+ ssa.OpAMD64VPABSB512,
+ ssa.OpAMD64VPABSW128,
+ ssa.OpAMD64VPABSW256,
+ ssa.OpAMD64VPABSW512,
+ ssa.OpAMD64VPABSD128,
+ ssa.OpAMD64VPABSD256,
+ ssa.OpAMD64VPABSD512,
+ ssa.OpAMD64VPABSQ128,
+ ssa.OpAMD64VPABSQ256,
+ ssa.OpAMD64VPABSQ512,
+ ssa.OpAMD64VCVTPD2PSX128,
+ ssa.OpAMD64VCVTPD2PSY128,
+ ssa.OpAMD64VCVTPD2PS256,
+ ssa.OpAMD64VCVTDQ2PS128,
+ ssa.OpAMD64VCVTDQ2PS256,
+ ssa.OpAMD64VCVTDQ2PS512,
+ ssa.OpAMD64VCVTQQ2PSX128,
+ ssa.OpAMD64VCVTQQ2PSY128,
+ ssa.OpAMD64VCVTQQ2PS256,
+ ssa.OpAMD64VCVTUDQ2PS128,
+ ssa.OpAMD64VCVTUDQ2PS256,
+ ssa.OpAMD64VCVTUDQ2PS512,
+ ssa.OpAMD64VCVTUQQ2PSX128,
+ ssa.OpAMD64VCVTUQQ2PSY128,
+ ssa.OpAMD64VCVTUQQ2PS256,
+ ssa.OpAMD64VCVTPS2PD256,
+ ssa.OpAMD64VCVTPS2PD512,
+ ssa.OpAMD64VCVTDQ2PD256,
+ ssa.OpAMD64VCVTDQ2PD512,
+ ssa.OpAMD64VCVTQQ2PD128,
+ ssa.OpAMD64VCVTQQ2PD256,
+ ssa.OpAMD64VCVTQQ2PD512,
+ ssa.OpAMD64VCVTUDQ2PD256,
+ ssa.OpAMD64VCVTUDQ2PD512,
+ ssa.OpAMD64VCVTUQQ2PD128,
+ ssa.OpAMD64VCVTUQQ2PD256,
+ ssa.OpAMD64VCVTUQQ2PD512,
+ ssa.OpAMD64VCVTTPS2DQ128,
+ ssa.OpAMD64VCVTTPS2DQ256,
+ ssa.OpAMD64VCVTTPS2DQ512,
+ ssa.OpAMD64VCVTTPD2DQX128,
+ ssa.OpAMD64VCVTTPD2DQY128,
+ ssa.OpAMD64VCVTTPD2DQ256,
+ ssa.OpAMD64VCVTTPS2QQ256,
+ ssa.OpAMD64VCVTTPS2QQ512,
+ ssa.OpAMD64VCVTTPD2QQ128,
+ ssa.OpAMD64VCVTTPD2QQ256,
+ ssa.OpAMD64VCVTTPD2QQ512,
+ ssa.OpAMD64VCVTTPS2UDQ128,
+ ssa.OpAMD64VCVTTPS2UDQ256,
+ ssa.OpAMD64VCVTTPS2UDQ512,
+ ssa.OpAMD64VCVTTPD2UDQX128,
+ ssa.OpAMD64VCVTTPD2UDQY128,
+ ssa.OpAMD64VCVTTPD2UDQ256,
+ ssa.OpAMD64VCVTTPS2UQQ256,
+ ssa.OpAMD64VCVTTPS2UQQ512,
+ ssa.OpAMD64VCVTTPD2UQQ128,
+ ssa.OpAMD64VCVTTPD2UQQ256,
+ ssa.OpAMD64VCVTTPD2UQQ512,
+ ssa.OpAMD64VPMOVSXBQ128,
+ ssa.OpAMD64VPMOVSXWQ128,
+ ssa.OpAMD64VPMOVSXDQ128,
+ ssa.OpAMD64VPMOVZXBQ128,
+ ssa.OpAMD64VPMOVZXWQ128,
+ ssa.OpAMD64VPMOVZXDQ128,
+ ssa.OpAMD64VPMOVSXBD128,
+ ssa.OpAMD64VPMOVSXWD128,
+ ssa.OpAMD64VPMOVSXBQ256,
+ ssa.OpAMD64VPMOVSXWQ256,
+ ssa.OpAMD64VPMOVZXBD128,
+ ssa.OpAMD64VPMOVZXWD128,
+ ssa.OpAMD64VPMOVZXBQ256,
+ ssa.OpAMD64VPMOVZXWQ256,
+ ssa.OpAMD64VPMOVSXBW128,
+ ssa.OpAMD64VPMOVSXBD256,
+ ssa.OpAMD64VPMOVSXBQ512,
+ ssa.OpAMD64VPMOVZXBW128,
+ ssa.OpAMD64VPMOVZXBD256,
+ ssa.OpAMD64VPMOVZXBQ512,
+ ssa.OpAMD64VPMOVSXBW256,
+ ssa.OpAMD64VPMOVSXBW512,
+ ssa.OpAMD64VPMOVSXBD512,
+ ssa.OpAMD64VPMOVSXWD256,
+ ssa.OpAMD64VPMOVSXWD512,
+ ssa.OpAMD64VPMOVSXWQ512,
+ ssa.OpAMD64VPMOVSXDQ256,
+ ssa.OpAMD64VPMOVSXDQ512,
+ ssa.OpAMD64VPMOVZXBW256,
+ ssa.OpAMD64VPMOVZXBW512,
+ ssa.OpAMD64VPMOVZXBD512,
+ ssa.OpAMD64VPMOVZXWD256,
+ ssa.OpAMD64VPMOVZXWD512,
+ ssa.OpAMD64VPMOVZXWQ512,
+ ssa.OpAMD64VPMOVZXDQ256,
+ ssa.OpAMD64VPMOVZXDQ512,
+ ssa.OpAMD64VPLZCNTD128,
+ ssa.OpAMD64VPLZCNTD256,
+ ssa.OpAMD64VPLZCNTD512,
+ ssa.OpAMD64VPLZCNTQ128,
+ ssa.OpAMD64VPLZCNTQ256,
+ ssa.OpAMD64VPLZCNTQ512,
+ ssa.OpAMD64VPOPCNTB128,
+ ssa.OpAMD64VPOPCNTB256,
+ ssa.OpAMD64VPOPCNTB512,
+ ssa.OpAMD64VPOPCNTW128,
+ ssa.OpAMD64VPOPCNTW256,
+ ssa.OpAMD64VPOPCNTW512,
+ ssa.OpAMD64VPOPCNTD128,
+ ssa.OpAMD64VPOPCNTD256,
+ ssa.OpAMD64VPOPCNTD512,
+ ssa.OpAMD64VPOPCNTQ128,
+ ssa.OpAMD64VPOPCNTQ256,
+ ssa.OpAMD64VPOPCNTQ512,
+ ssa.OpAMD64VRCPPS128,
+ ssa.OpAMD64VRCPPS256,
+ ssa.OpAMD64VRCP14PS512,
+ ssa.OpAMD64VRCP14PD128,
+ ssa.OpAMD64VRCP14PD256,
+ ssa.OpAMD64VRCP14PD512,
+ ssa.OpAMD64VRSQRTPS128,
+ ssa.OpAMD64VRSQRTPS256,
+ ssa.OpAMD64VRSQRT14PS512,
+ ssa.OpAMD64VRSQRT14PD128,
+ ssa.OpAMD64VRSQRT14PD256,
+ ssa.OpAMD64VRSQRT14PD512,
+ ssa.OpAMD64VPMOVSWB128_128,
+ ssa.OpAMD64VPMOVSWB128_256,
+ ssa.OpAMD64VPMOVSWB256,
+ ssa.OpAMD64VPMOVSDB128_128,
+ ssa.OpAMD64VPMOVSDB128_256,
+ ssa.OpAMD64VPMOVSDB128_512,
+ ssa.OpAMD64VPMOVSQB128_128,
+ ssa.OpAMD64VPMOVSQB128_256,
+ ssa.OpAMD64VPMOVSQB128_512,
+ ssa.OpAMD64VPMOVSDW128_128,
+ ssa.OpAMD64VPMOVSDW128_256,
+ ssa.OpAMD64VPMOVSDW256,
+ ssa.OpAMD64VPMOVSQW128_128,
+ ssa.OpAMD64VPMOVSQW128_256,
+ ssa.OpAMD64VPMOVSQW128_512,
+ ssa.OpAMD64VPMOVSQD128_128,
+ ssa.OpAMD64VPMOVSQD128_256,
+ ssa.OpAMD64VPMOVSQD256,
+ ssa.OpAMD64VPMOVUSWB128_128,
+ ssa.OpAMD64VPMOVUSWB128_256,
+ ssa.OpAMD64VPMOVUSWB256,
+ ssa.OpAMD64VPMOVUSDB128_128,
+ ssa.OpAMD64VPMOVUSDB128_256,
+ ssa.OpAMD64VPMOVUSDB128_512,
+ ssa.OpAMD64VPMOVUSQB128_128,
+ ssa.OpAMD64VPMOVUSQB128_256,
+ ssa.OpAMD64VPMOVUSQB128_512,
+ ssa.OpAMD64VPMOVUSDW128_128,
+ ssa.OpAMD64VPMOVUSDW128_256,
+ ssa.OpAMD64VPMOVUSDW256,
+ ssa.OpAMD64VPMOVUSQW128_128,
+ ssa.OpAMD64VPMOVUSQW128_256,
+ ssa.OpAMD64VPMOVUSQW128_512,
+ ssa.OpAMD64VPMOVUSQD128_128,
+ ssa.OpAMD64VPMOVUSQD128_256,
+ ssa.OpAMD64VPMOVUSQD256,
+ ssa.OpAMD64VSQRTPS128,
+ ssa.OpAMD64VSQRTPS256,
+ ssa.OpAMD64VSQRTPS512,
+ ssa.OpAMD64VSQRTPD128,
+ ssa.OpAMD64VSQRTPD256,
+ ssa.OpAMD64VSQRTPD512,
+ ssa.OpAMD64VPMOVWB128_128,
+ ssa.OpAMD64VPMOVWB128_256,
+ ssa.OpAMD64VPMOVWB256,
+ ssa.OpAMD64VPMOVDB128_128,
+ ssa.OpAMD64VPMOVDB128_256,
+ ssa.OpAMD64VPMOVDB128_512,
+ ssa.OpAMD64VPMOVQB128_128,
+ ssa.OpAMD64VPMOVQB128_256,
+ ssa.OpAMD64VPMOVQB128_512,
+ ssa.OpAMD64VPMOVDW128_128,
+ ssa.OpAMD64VPMOVDW128_256,
+ ssa.OpAMD64VPMOVDW256,
+ ssa.OpAMD64VPMOVQW128_128,
+ ssa.OpAMD64VPMOVQW128_256,
+ ssa.OpAMD64VPMOVQW128_512,
+ ssa.OpAMD64VPMOVQD128_128,
+ ssa.OpAMD64VPMOVQD128_256,
+ ssa.OpAMD64VPMOVQD256,
+ ssa.OpAMD64VPBROADCASTQ128,
+ ssa.OpAMD64VBROADCASTSS128,
+ ssa.OpAMD64VBROADCASTSD256,
+ ssa.OpAMD64VPBROADCASTD128,
+ ssa.OpAMD64VPBROADCASTQ256,
+ ssa.OpAMD64VBROADCASTSS256,
+ ssa.OpAMD64VBROADCASTSD512,
+ ssa.OpAMD64VPBROADCASTW128,
+ ssa.OpAMD64VPBROADCASTD256,
+ ssa.OpAMD64VPBROADCASTQ512,
+ ssa.OpAMD64VBROADCASTSS512,
+ ssa.OpAMD64VPBROADCASTB128,
+ ssa.OpAMD64VPBROADCASTW256,
+ ssa.OpAMD64VPBROADCASTD512,
+ ssa.OpAMD64VPBROADCASTB256,
+ ssa.OpAMD64VPBROADCASTW512,
+ ssa.OpAMD64VPBROADCASTB512:
+ p = simdV11(s, v)
+
+ case ssa.OpAMD64VAESDECLAST128,
+ ssa.OpAMD64VAESDECLAST256,
+ ssa.OpAMD64VAESDECLAST512,
+ ssa.OpAMD64VAESDEC128,
+ ssa.OpAMD64VAESDEC256,
+ ssa.OpAMD64VAESDEC512,
+ ssa.OpAMD64VAESENCLAST128,
+ ssa.OpAMD64VAESENCLAST256,
+ ssa.OpAMD64VAESENCLAST512,
+ ssa.OpAMD64VAESENC128,
+ ssa.OpAMD64VAESENC256,
+ ssa.OpAMD64VAESENC512,
+ ssa.OpAMD64VADDPS128,
+ ssa.OpAMD64VADDPS256,
+ ssa.OpAMD64VADDPS512,
+ ssa.OpAMD64VADDPD128,
+ ssa.OpAMD64VADDPD256,
+ ssa.OpAMD64VADDPD512,
+ ssa.OpAMD64VPADDB128,
+ ssa.OpAMD64VPADDB256,
+ ssa.OpAMD64VPADDB512,
+ ssa.OpAMD64VPADDW128,
+ ssa.OpAMD64VPADDW256,
+ ssa.OpAMD64VPADDW512,
+ ssa.OpAMD64VPADDD128,
+ ssa.OpAMD64VPADDD256,
+ ssa.OpAMD64VPADDD512,
+ ssa.OpAMD64VPADDQ128,
+ ssa.OpAMD64VPADDQ256,
+ ssa.OpAMD64VPADDQ512,
+ ssa.OpAMD64VADDSUBPS128,
+ ssa.OpAMD64VADDSUBPS256,
+ ssa.OpAMD64VADDSUBPD128,
+ ssa.OpAMD64VADDSUBPD256,
+ ssa.OpAMD64VPADDSB128,
+ ssa.OpAMD64VPADDSB256,
+ ssa.OpAMD64VPADDSB512,
+ ssa.OpAMD64VPADDSW128,
+ ssa.OpAMD64VPADDSW256,
+ ssa.OpAMD64VPADDSW512,
+ ssa.OpAMD64VPADDUSB128,
+ ssa.OpAMD64VPADDUSB256,
+ ssa.OpAMD64VPADDUSB512,
+ ssa.OpAMD64VPADDUSW128,
+ ssa.OpAMD64VPADDUSW256,
+ ssa.OpAMD64VPADDUSW512,
+ ssa.OpAMD64VPAND128,
+ ssa.OpAMD64VPAND256,
+ ssa.OpAMD64VPANDD512,
+ ssa.OpAMD64VPANDQ512,
+ ssa.OpAMD64VPANDN128,
+ ssa.OpAMD64VPANDN256,
+ ssa.OpAMD64VPANDND512,
+ ssa.OpAMD64VPANDNQ512,
+ ssa.OpAMD64VPAVGB128,
+ ssa.OpAMD64VPAVGB256,
+ ssa.OpAMD64VPAVGB512,
+ ssa.OpAMD64VPAVGW128,
+ ssa.OpAMD64VPAVGW256,
+ ssa.OpAMD64VPAVGW512,
+ ssa.OpAMD64VHADDPS128,
+ ssa.OpAMD64VHADDPD128,
+ ssa.OpAMD64VPHADDW128,
+ ssa.OpAMD64VPHADDD128,
+ ssa.OpAMD64VHADDPS256,
+ ssa.OpAMD64VHADDPD256,
+ ssa.OpAMD64VPHADDW256,
+ ssa.OpAMD64VPHADDD256,
+ ssa.OpAMD64VPHADDSW128,
+ ssa.OpAMD64VPHADDSW256,
+ ssa.OpAMD64VHSUBPS128,
+ ssa.OpAMD64VHSUBPD128,
+ ssa.OpAMD64VPHSUBW128,
+ ssa.OpAMD64VPHSUBD128,
+ ssa.OpAMD64VHSUBPS256,
+ ssa.OpAMD64VHSUBPD256,
+ ssa.OpAMD64VPHSUBW256,
+ ssa.OpAMD64VPHSUBD256,
+ ssa.OpAMD64VPHSUBSW128,
+ ssa.OpAMD64VPHSUBSW256,
+ ssa.OpAMD64VDIVPS128,
+ ssa.OpAMD64VDIVPS256,
+ ssa.OpAMD64VDIVPS512,
+ ssa.OpAMD64VDIVPD128,
+ ssa.OpAMD64VDIVPD256,
+ ssa.OpAMD64VDIVPD512,
+ ssa.OpAMD64VPMADDWD128,
+ ssa.OpAMD64VPMADDWD256,
+ ssa.OpAMD64VPMADDWD512,
+ ssa.OpAMD64VPMADDUBSW128,
+ ssa.OpAMD64VPMADDUBSW256,
+ ssa.OpAMD64VPMADDUBSW512,
+ ssa.OpAMD64VPCMPEQB128,
+ ssa.OpAMD64VPCMPEQB256,
+ ssa.OpAMD64VPCMPEQW128,
+ ssa.OpAMD64VPCMPEQW256,
+ ssa.OpAMD64VPCMPEQD128,
+ ssa.OpAMD64VPCMPEQD256,
+ ssa.OpAMD64VPCMPEQQ128,
+ ssa.OpAMD64VPCMPEQQ256,
+ ssa.OpAMD64VGF2P8MULB128,
+ ssa.OpAMD64VGF2P8MULB256,
+ ssa.OpAMD64VGF2P8MULB512,
+ ssa.OpAMD64VPCMPGTB128,
+ ssa.OpAMD64VPCMPGTB256,
+ ssa.OpAMD64VPCMPGTW128,
+ ssa.OpAMD64VPCMPGTW256,
+ ssa.OpAMD64VPCMPGTD128,
+ ssa.OpAMD64VPCMPGTD256,
+ ssa.OpAMD64VPCMPGTQ128,
+ ssa.OpAMD64VPCMPGTQ256,
+ ssa.OpAMD64VPUNPCKHWD128,
+ ssa.OpAMD64VPUNPCKHDQ128,
+ ssa.OpAMD64VPUNPCKHQDQ128,
+ ssa.OpAMD64VPUNPCKHWD256,
+ ssa.OpAMD64VPUNPCKHWD512,
+ ssa.OpAMD64VPUNPCKHDQ256,
+ ssa.OpAMD64VPUNPCKHDQ512,
+ ssa.OpAMD64VPUNPCKHQDQ256,
+ ssa.OpAMD64VPUNPCKHQDQ512,
+ ssa.OpAMD64VPUNPCKLWD128,
+ ssa.OpAMD64VPUNPCKLDQ128,
+ ssa.OpAMD64VPUNPCKLQDQ128,
+ ssa.OpAMD64VPUNPCKLWD256,
+ ssa.OpAMD64VPUNPCKLWD512,
+ ssa.OpAMD64VPUNPCKLDQ256,
+ ssa.OpAMD64VPUNPCKLDQ512,
+ ssa.OpAMD64VPUNPCKLQDQ256,
+ ssa.OpAMD64VPUNPCKLQDQ512,
+ ssa.OpAMD64VMAXPS128,
+ ssa.OpAMD64VMAXPS256,
+ ssa.OpAMD64VMAXPS512,
+ ssa.OpAMD64VMAXPD128,
+ ssa.OpAMD64VMAXPD256,
+ ssa.OpAMD64VMAXPD512,
+ ssa.OpAMD64VPMAXSB128,
+ ssa.OpAMD64VPMAXSB256,
+ ssa.OpAMD64VPMAXSB512,
+ ssa.OpAMD64VPMAXSW128,
+ ssa.OpAMD64VPMAXSW256,
+ ssa.OpAMD64VPMAXSW512,
+ ssa.OpAMD64VPMAXSD128,
+ ssa.OpAMD64VPMAXSD256,
+ ssa.OpAMD64VPMAXSD512,
+ ssa.OpAMD64VPMAXSQ128,
+ ssa.OpAMD64VPMAXSQ256,
+ ssa.OpAMD64VPMAXSQ512,
+ ssa.OpAMD64VPMAXUB128,
+ ssa.OpAMD64VPMAXUB256,
+ ssa.OpAMD64VPMAXUB512,
+ ssa.OpAMD64VPMAXUW128,
+ ssa.OpAMD64VPMAXUW256,
+ ssa.OpAMD64VPMAXUW512,
+ ssa.OpAMD64VPMAXUD128,
+ ssa.OpAMD64VPMAXUD256,
+ ssa.OpAMD64VPMAXUD512,
+ ssa.OpAMD64VPMAXUQ128,
+ ssa.OpAMD64VPMAXUQ256,
+ ssa.OpAMD64VPMAXUQ512,
+ ssa.OpAMD64VMINPS128,
+ ssa.OpAMD64VMINPS256,
+ ssa.OpAMD64VMINPS512,
+ ssa.OpAMD64VMINPD128,
+ ssa.OpAMD64VMINPD256,
+ ssa.OpAMD64VMINPD512,
+ ssa.OpAMD64VPMINSB128,
+ ssa.OpAMD64VPMINSB256,
+ ssa.OpAMD64VPMINSB512,
+ ssa.OpAMD64VPMINSW128,
+ ssa.OpAMD64VPMINSW256,
+ ssa.OpAMD64VPMINSW512,
+ ssa.OpAMD64VPMINSD128,
+ ssa.OpAMD64VPMINSD256,
+ ssa.OpAMD64VPMINSD512,
+ ssa.OpAMD64VPMINSQ128,
+ ssa.OpAMD64VPMINSQ256,
+ ssa.OpAMD64VPMINSQ512,
+ ssa.OpAMD64VPMINUB128,
+ ssa.OpAMD64VPMINUB256,
+ ssa.OpAMD64VPMINUB512,
+ ssa.OpAMD64VPMINUW128,
+ ssa.OpAMD64VPMINUW256,
+ ssa.OpAMD64VPMINUW512,
+ ssa.OpAMD64VPMINUD128,
+ ssa.OpAMD64VPMINUD256,
+ ssa.OpAMD64VPMINUD512,
+ ssa.OpAMD64VPMINUQ128,
+ ssa.OpAMD64VPMINUQ256,
+ ssa.OpAMD64VPMINUQ512,
+ ssa.OpAMD64VMULPS128,
+ ssa.OpAMD64VMULPS256,
+ ssa.OpAMD64VMULPS512,
+ ssa.OpAMD64VMULPD128,
+ ssa.OpAMD64VMULPD256,
+ ssa.OpAMD64VMULPD512,
+ ssa.OpAMD64VPMULLW128,
+ ssa.OpAMD64VPMULLW256,
+ ssa.OpAMD64VPMULLW512,
+ ssa.OpAMD64VPMULLD128,
+ ssa.OpAMD64VPMULLD256,
+ ssa.OpAMD64VPMULLD512,
+ ssa.OpAMD64VPMULLQ128,
+ ssa.OpAMD64VPMULLQ256,
+ ssa.OpAMD64VPMULLQ512,
+ ssa.OpAMD64VPMULHW128,
+ ssa.OpAMD64VPMULHW256,
+ ssa.OpAMD64VPMULHW512,
+ ssa.OpAMD64VPMULHUW128,
+ ssa.OpAMD64VPMULHUW256,
+ ssa.OpAMD64VPMULHUW512,
+ ssa.OpAMD64VPSIGNB128,
+ ssa.OpAMD64VPSIGNB256,
+ ssa.OpAMD64VPSIGNW128,
+ ssa.OpAMD64VPSIGNW256,
+ ssa.OpAMD64VPSIGND128,
+ ssa.OpAMD64VPSIGND256,
+ ssa.OpAMD64VPMULDQ128,
+ ssa.OpAMD64VPMULDQ256,
+ ssa.OpAMD64VPMULUDQ128,
+ ssa.OpAMD64VPMULUDQ256,
+ ssa.OpAMD64VPOR128,
+ ssa.OpAMD64VPOR256,
+ ssa.OpAMD64VPORD512,
+ ssa.OpAMD64VPORQ512,
+ ssa.OpAMD64VPERMB128,
+ ssa.OpAMD64VPERMB256,
+ ssa.OpAMD64VPERMB512,
+ ssa.OpAMD64VPERMW128,
+ ssa.OpAMD64VPERMW256,
+ ssa.OpAMD64VPERMW512,
+ ssa.OpAMD64VPERMPS256,
+ ssa.OpAMD64VPERMD256,
+ ssa.OpAMD64VPERMPS512,
+ ssa.OpAMD64VPERMD512,
+ ssa.OpAMD64VPERMPD256,
+ ssa.OpAMD64VPERMQ256,
+ ssa.OpAMD64VPERMPD512,
+ ssa.OpAMD64VPERMQ512,
+ ssa.OpAMD64VPSHUFB128,
+ ssa.OpAMD64VPSHUFB256,
+ ssa.OpAMD64VPSHUFB512,
+ ssa.OpAMD64VPROLVD128,
+ ssa.OpAMD64VPROLVD256,
+ ssa.OpAMD64VPROLVD512,
+ ssa.OpAMD64VPROLVQ128,
+ ssa.OpAMD64VPROLVQ256,
+ ssa.OpAMD64VPROLVQ512,
+ ssa.OpAMD64VPRORVD128,
+ ssa.OpAMD64VPRORVD256,
+ ssa.OpAMD64VPRORVD512,
+ ssa.OpAMD64VPRORVQ128,
+ ssa.OpAMD64VPRORVQ256,
+ ssa.OpAMD64VPRORVQ512,
+ ssa.OpAMD64VPACKSSDW128,
+ ssa.OpAMD64VPACKSSDW256,
+ ssa.OpAMD64VPACKSSDW512,
+ ssa.OpAMD64VPACKUSDW128,
+ ssa.OpAMD64VPACKUSDW256,
+ ssa.OpAMD64VPACKUSDW512,
+ ssa.OpAMD64VSCALEFPS128,
+ ssa.OpAMD64VSCALEFPS256,
+ ssa.OpAMD64VSCALEFPS512,
+ ssa.OpAMD64VSCALEFPD128,
+ ssa.OpAMD64VSCALEFPD256,
+ ssa.OpAMD64VSCALEFPD512,
+ ssa.OpAMD64VPSLLVW128,
+ ssa.OpAMD64VPSLLVW256,
+ ssa.OpAMD64VPSLLVW512,
+ ssa.OpAMD64VPSLLVD128,
+ ssa.OpAMD64VPSLLVD256,
+ ssa.OpAMD64VPSLLVD512,
+ ssa.OpAMD64VPSLLVQ128,
+ ssa.OpAMD64VPSLLVQ256,
+ ssa.OpAMD64VPSLLVQ512,
+ ssa.OpAMD64VPSRAVW128,
+ ssa.OpAMD64VPSRAVW256,
+ ssa.OpAMD64VPSRAVW512,
+ ssa.OpAMD64VPSRAVD128,
+ ssa.OpAMD64VPSRAVD256,
+ ssa.OpAMD64VPSRAVD512,
+ ssa.OpAMD64VPSRAVQ128,
+ ssa.OpAMD64VPSRAVQ256,
+ ssa.OpAMD64VPSRAVQ512,
+ ssa.OpAMD64VPSRLVW128,
+ ssa.OpAMD64VPSRLVW256,
+ ssa.OpAMD64VPSRLVW512,
+ ssa.OpAMD64VPSRLVD128,
+ ssa.OpAMD64VPSRLVD256,
+ ssa.OpAMD64VPSRLVD512,
+ ssa.OpAMD64VPSRLVQ128,
+ ssa.OpAMD64VPSRLVQ256,
+ ssa.OpAMD64VPSRLVQ512,
+ ssa.OpAMD64VSUBPS128,
+ ssa.OpAMD64VSUBPS256,
+ ssa.OpAMD64VSUBPS512,
+ ssa.OpAMD64VSUBPD128,
+ ssa.OpAMD64VSUBPD256,
+ ssa.OpAMD64VSUBPD512,
+ ssa.OpAMD64VPSUBB128,
+ ssa.OpAMD64VPSUBB256,
+ ssa.OpAMD64VPSUBB512,
+ ssa.OpAMD64VPSUBW128,
+ ssa.OpAMD64VPSUBW256,
+ ssa.OpAMD64VPSUBW512,
+ ssa.OpAMD64VPSUBD128,
+ ssa.OpAMD64VPSUBD256,
+ ssa.OpAMD64VPSUBD512,
+ ssa.OpAMD64VPSUBQ128,
+ ssa.OpAMD64VPSUBQ256,
+ ssa.OpAMD64VPSUBQ512,
+ ssa.OpAMD64VPSUBSB128,
+ ssa.OpAMD64VPSUBSB256,
+ ssa.OpAMD64VPSUBSB512,
+ ssa.OpAMD64VPSUBSW128,
+ ssa.OpAMD64VPSUBSW256,
+ ssa.OpAMD64VPSUBSW512,
+ ssa.OpAMD64VPSUBUSB128,
+ ssa.OpAMD64VPSUBUSB256,
+ ssa.OpAMD64VPSUBUSB512,
+ ssa.OpAMD64VPSUBUSW128,
+ ssa.OpAMD64VPSUBUSW256,
+ ssa.OpAMD64VPSUBUSW512,
+ ssa.OpAMD64VPSADBW128,
+ ssa.OpAMD64VPSADBW256,
+ ssa.OpAMD64VPSADBW512,
+ ssa.OpAMD64VPXOR128,
+ ssa.OpAMD64VPXOR256,
+ ssa.OpAMD64VPXORD512,
+ ssa.OpAMD64VPXORQ512:
+ p = simdV21(s, v)
+
+ case ssa.OpAMD64VPCMPEQB512,
+ ssa.OpAMD64VPCMPEQW512,
+ ssa.OpAMD64VPCMPEQD512,
+ ssa.OpAMD64VPCMPEQQ512,
+ ssa.OpAMD64VPCMPGTB512,
+ ssa.OpAMD64VPCMPGTW512,
+ ssa.OpAMD64VPCMPGTD512,
+ ssa.OpAMD64VPCMPGTQ512:
+ p = simdV2k(s, v)
+
+ case ssa.OpAMD64VADDPSMasked128,
+ ssa.OpAMD64VADDPSMasked256,
+ ssa.OpAMD64VADDPSMasked512,
+ ssa.OpAMD64VADDPDMasked128,
+ ssa.OpAMD64VADDPDMasked256,
+ ssa.OpAMD64VADDPDMasked512,
+ ssa.OpAMD64VPADDBMasked128,
+ ssa.OpAMD64VPADDBMasked256,
+ ssa.OpAMD64VPADDBMasked512,
+ ssa.OpAMD64VPADDWMasked128,
+ ssa.OpAMD64VPADDWMasked256,
+ ssa.OpAMD64VPADDWMasked512,
+ ssa.OpAMD64VPADDDMasked128,
+ ssa.OpAMD64VPADDDMasked256,
+ ssa.OpAMD64VPADDDMasked512,
+ ssa.OpAMD64VPADDQMasked128,
+ ssa.OpAMD64VPADDQMasked256,
+ ssa.OpAMD64VPADDQMasked512,
+ ssa.OpAMD64VPADDSBMasked128,
+ ssa.OpAMD64VPADDSBMasked256,
+ ssa.OpAMD64VPADDSBMasked512,
+ ssa.OpAMD64VPADDSWMasked128,
+ ssa.OpAMD64VPADDSWMasked256,
+ ssa.OpAMD64VPADDSWMasked512,
+ ssa.OpAMD64VPADDUSBMasked128,
+ ssa.OpAMD64VPADDUSBMasked256,
+ ssa.OpAMD64VPADDUSBMasked512,
+ ssa.OpAMD64VPADDUSWMasked128,
+ ssa.OpAMD64VPADDUSWMasked256,
+ ssa.OpAMD64VPADDUSWMasked512,
+ ssa.OpAMD64VPANDDMasked128,
+ ssa.OpAMD64VPANDDMasked256,
+ ssa.OpAMD64VPANDDMasked512,
+ ssa.OpAMD64VPANDQMasked128,
+ ssa.OpAMD64VPANDQMasked256,
+ ssa.OpAMD64VPANDQMasked512,
+ ssa.OpAMD64VPANDNDMasked128,
+ ssa.OpAMD64VPANDNDMasked256,
+ ssa.OpAMD64VPANDNDMasked512,
+ ssa.OpAMD64VPANDNQMasked128,
+ ssa.OpAMD64VPANDNQMasked256,
+ ssa.OpAMD64VPANDNQMasked512,
+ ssa.OpAMD64VPAVGBMasked128,
+ ssa.OpAMD64VPAVGBMasked256,
+ ssa.OpAMD64VPAVGBMasked512,
+ ssa.OpAMD64VPAVGWMasked128,
+ ssa.OpAMD64VPAVGWMasked256,
+ ssa.OpAMD64VPAVGWMasked512,
+ ssa.OpAMD64VDIVPSMasked128,
+ ssa.OpAMD64VDIVPSMasked256,
+ ssa.OpAMD64VDIVPSMasked512,
+ ssa.OpAMD64VDIVPDMasked128,
+ ssa.OpAMD64VDIVPDMasked256,
+ ssa.OpAMD64VDIVPDMasked512,
+ ssa.OpAMD64VPMADDWDMasked128,
+ ssa.OpAMD64VPMADDWDMasked256,
+ ssa.OpAMD64VPMADDWDMasked512,
+ ssa.OpAMD64VPMADDUBSWMasked128,
+ ssa.OpAMD64VPMADDUBSWMasked256,
+ ssa.OpAMD64VPMADDUBSWMasked512,
+ ssa.OpAMD64VGF2P8MULBMasked128,
+ ssa.OpAMD64VGF2P8MULBMasked256,
+ ssa.OpAMD64VGF2P8MULBMasked512,
+ ssa.OpAMD64VMAXPSMasked128,
+ ssa.OpAMD64VMAXPSMasked256,
+ ssa.OpAMD64VMAXPSMasked512,
+ ssa.OpAMD64VMAXPDMasked128,
+ ssa.OpAMD64VMAXPDMasked256,
+ ssa.OpAMD64VMAXPDMasked512,
+ ssa.OpAMD64VPMAXSBMasked128,
+ ssa.OpAMD64VPMAXSBMasked256,
+ ssa.OpAMD64VPMAXSBMasked512,
+ ssa.OpAMD64VPMAXSWMasked128,
+ ssa.OpAMD64VPMAXSWMasked256,
+ ssa.OpAMD64VPMAXSWMasked512,
+ ssa.OpAMD64VPMAXSDMasked128,
+ ssa.OpAMD64VPMAXSDMasked256,
+ ssa.OpAMD64VPMAXSDMasked512,
+ ssa.OpAMD64VPMAXSQMasked128,
+ ssa.OpAMD64VPMAXSQMasked256,
+ ssa.OpAMD64VPMAXSQMasked512,
+ ssa.OpAMD64VPMAXUBMasked128,
+ ssa.OpAMD64VPMAXUBMasked256,
+ ssa.OpAMD64VPMAXUBMasked512,
+ ssa.OpAMD64VPMAXUWMasked128,
+ ssa.OpAMD64VPMAXUWMasked256,
+ ssa.OpAMD64VPMAXUWMasked512,
+ ssa.OpAMD64VPMAXUDMasked128,
+ ssa.OpAMD64VPMAXUDMasked256,
+ ssa.OpAMD64VPMAXUDMasked512,
+ ssa.OpAMD64VPMAXUQMasked128,
+ ssa.OpAMD64VPMAXUQMasked256,
+ ssa.OpAMD64VPMAXUQMasked512,
+ ssa.OpAMD64VMINPSMasked128,
+ ssa.OpAMD64VMINPSMasked256,
+ ssa.OpAMD64VMINPSMasked512,
+ ssa.OpAMD64VMINPDMasked128,
+ ssa.OpAMD64VMINPDMasked256,
+ ssa.OpAMD64VMINPDMasked512,
+ ssa.OpAMD64VPMINSBMasked128,
+ ssa.OpAMD64VPMINSBMasked256,
+ ssa.OpAMD64VPMINSBMasked512,
+ ssa.OpAMD64VPMINSWMasked128,
+ ssa.OpAMD64VPMINSWMasked256,
+ ssa.OpAMD64VPMINSWMasked512,
+ ssa.OpAMD64VPMINSDMasked128,
+ ssa.OpAMD64VPMINSDMasked256,
+ ssa.OpAMD64VPMINSDMasked512,
+ ssa.OpAMD64VPMINSQMasked128,
+ ssa.OpAMD64VPMINSQMasked256,
+ ssa.OpAMD64VPMINSQMasked512,
+ ssa.OpAMD64VPMINUBMasked128,
+ ssa.OpAMD64VPMINUBMasked256,
+ ssa.OpAMD64VPMINUBMasked512,
+ ssa.OpAMD64VPMINUWMasked128,
+ ssa.OpAMD64VPMINUWMasked256,
+ ssa.OpAMD64VPMINUWMasked512,
+ ssa.OpAMD64VPMINUDMasked128,
+ ssa.OpAMD64VPMINUDMasked256,
+ ssa.OpAMD64VPMINUDMasked512,
+ ssa.OpAMD64VPMINUQMasked128,
+ ssa.OpAMD64VPMINUQMasked256,
+ ssa.OpAMD64VPMINUQMasked512,
+ ssa.OpAMD64VPMULHWMasked128,
+ ssa.OpAMD64VPMULHWMasked256,
+ ssa.OpAMD64VPMULHWMasked512,
+ ssa.OpAMD64VPMULHUWMasked128,
+ ssa.OpAMD64VPMULHUWMasked256,
+ ssa.OpAMD64VPMULHUWMasked512,
+ ssa.OpAMD64VMULPSMasked128,
+ ssa.OpAMD64VMULPSMasked256,
+ ssa.OpAMD64VMULPSMasked512,
+ ssa.OpAMD64VMULPDMasked128,
+ ssa.OpAMD64VMULPDMasked256,
+ ssa.OpAMD64VMULPDMasked512,
+ ssa.OpAMD64VPMULLWMasked128,
+ ssa.OpAMD64VPMULLWMasked256,
+ ssa.OpAMD64VPMULLWMasked512,
+ ssa.OpAMD64VPMULLDMasked128,
+ ssa.OpAMD64VPMULLDMasked256,
+ ssa.OpAMD64VPMULLDMasked512,
+ ssa.OpAMD64VPMULLQMasked128,
+ ssa.OpAMD64VPMULLQMasked256,
+ ssa.OpAMD64VPMULLQMasked512,
+ ssa.OpAMD64VPORDMasked128,
+ ssa.OpAMD64VPORDMasked256,
+ ssa.OpAMD64VPORDMasked512,
+ ssa.OpAMD64VPORQMasked128,
+ ssa.OpAMD64VPORQMasked256,
+ ssa.OpAMD64VPORQMasked512,
+ ssa.OpAMD64VPERMBMasked128,
+ ssa.OpAMD64VPERMBMasked256,
+ ssa.OpAMD64VPERMBMasked512,
+ ssa.OpAMD64VPERMWMasked128,
+ ssa.OpAMD64VPERMWMasked256,
+ ssa.OpAMD64VPERMWMasked512,
+ ssa.OpAMD64VPERMPSMasked256,
+ ssa.OpAMD64VPERMDMasked256,
+ ssa.OpAMD64VPERMPSMasked512,
+ ssa.OpAMD64VPERMDMasked512,
+ ssa.OpAMD64VPERMPDMasked256,
+ ssa.OpAMD64VPERMQMasked256,
+ ssa.OpAMD64VPERMPDMasked512,
+ ssa.OpAMD64VPERMQMasked512,
+ ssa.OpAMD64VPSHUFBMasked256,
+ ssa.OpAMD64VPSHUFBMasked512,
+ ssa.OpAMD64VPSHUFBMasked128,
+ ssa.OpAMD64VPROLVDMasked128,
+ ssa.OpAMD64VPROLVDMasked256,
+ ssa.OpAMD64VPROLVDMasked512,
+ ssa.OpAMD64VPROLVQMasked128,
+ ssa.OpAMD64VPROLVQMasked256,
+ ssa.OpAMD64VPROLVQMasked512,
+ ssa.OpAMD64VPRORVDMasked128,
+ ssa.OpAMD64VPRORVDMasked256,
+ ssa.OpAMD64VPRORVDMasked512,
+ ssa.OpAMD64VPRORVQMasked128,
+ ssa.OpAMD64VPRORVQMasked256,
+ ssa.OpAMD64VPRORVQMasked512,
+ ssa.OpAMD64VPACKSSDWMasked256,
+ ssa.OpAMD64VPACKSSDWMasked512,
+ ssa.OpAMD64VPACKSSDWMasked128,
+ ssa.OpAMD64VPACKUSDWMasked256,
+ ssa.OpAMD64VPACKUSDWMasked512,
+ ssa.OpAMD64VPACKUSDWMasked128,
+ ssa.OpAMD64VSCALEFPSMasked128,
+ ssa.OpAMD64VSCALEFPSMasked256,
+ ssa.OpAMD64VSCALEFPSMasked512,
+ ssa.OpAMD64VSCALEFPDMasked128,
+ ssa.OpAMD64VSCALEFPDMasked256,
+ ssa.OpAMD64VSCALEFPDMasked512,
+ ssa.OpAMD64VPSLLVWMasked128,
+ ssa.OpAMD64VPSLLVWMasked256,
+ ssa.OpAMD64VPSLLVWMasked512,
+ ssa.OpAMD64VPSLLVDMasked128,
+ ssa.OpAMD64VPSLLVDMasked256,
+ ssa.OpAMD64VPSLLVDMasked512,
+ ssa.OpAMD64VPSLLVQMasked128,
+ ssa.OpAMD64VPSLLVQMasked256,
+ ssa.OpAMD64VPSLLVQMasked512,
+ ssa.OpAMD64VPSRAVWMasked128,
+ ssa.OpAMD64VPSRAVWMasked256,
+ ssa.OpAMD64VPSRAVWMasked512,
+ ssa.OpAMD64VPSRAVDMasked128,
+ ssa.OpAMD64VPSRAVDMasked256,
+ ssa.OpAMD64VPSRAVDMasked512,
+ ssa.OpAMD64VPSRAVQMasked128,
+ ssa.OpAMD64VPSRAVQMasked256,
+ ssa.OpAMD64VPSRAVQMasked512,
+ ssa.OpAMD64VPSRLVWMasked128,
+ ssa.OpAMD64VPSRLVWMasked256,
+ ssa.OpAMD64VPSRLVWMasked512,
+ ssa.OpAMD64VPSRLVDMasked128,
+ ssa.OpAMD64VPSRLVDMasked256,
+ ssa.OpAMD64VPSRLVDMasked512,
+ ssa.OpAMD64VPSRLVQMasked128,
+ ssa.OpAMD64VPSRLVQMasked256,
+ ssa.OpAMD64VPSRLVQMasked512,
+ ssa.OpAMD64VSUBPSMasked128,
+ ssa.OpAMD64VSUBPSMasked256,
+ ssa.OpAMD64VSUBPSMasked512,
+ ssa.OpAMD64VSUBPDMasked128,
+ ssa.OpAMD64VSUBPDMasked256,
+ ssa.OpAMD64VSUBPDMasked512,
+ ssa.OpAMD64VPSUBBMasked128,
+ ssa.OpAMD64VPSUBBMasked256,
+ ssa.OpAMD64VPSUBBMasked512,
+ ssa.OpAMD64VPSUBWMasked128,
+ ssa.OpAMD64VPSUBWMasked256,
+ ssa.OpAMD64VPSUBWMasked512,
+ ssa.OpAMD64VPSUBDMasked128,
+ ssa.OpAMD64VPSUBDMasked256,
+ ssa.OpAMD64VPSUBDMasked512,
+ ssa.OpAMD64VPSUBQMasked128,
+ ssa.OpAMD64VPSUBQMasked256,
+ ssa.OpAMD64VPSUBQMasked512,
+ ssa.OpAMD64VPSUBSBMasked128,
+ ssa.OpAMD64VPSUBSBMasked256,
+ ssa.OpAMD64VPSUBSBMasked512,
+ ssa.OpAMD64VPSUBSWMasked128,
+ ssa.OpAMD64VPSUBSWMasked256,
+ ssa.OpAMD64VPSUBSWMasked512,
+ ssa.OpAMD64VPSUBUSBMasked128,
+ ssa.OpAMD64VPSUBUSBMasked256,
+ ssa.OpAMD64VPSUBUSBMasked512,
+ ssa.OpAMD64VPSUBUSWMasked128,
+ ssa.OpAMD64VPSUBUSWMasked256,
+ ssa.OpAMD64VPSUBUSWMasked512,
+ ssa.OpAMD64VPXORDMasked128,
+ ssa.OpAMD64VPXORDMasked256,
+ ssa.OpAMD64VPXORDMasked512,
+ ssa.OpAMD64VPXORQMasked128,
+ ssa.OpAMD64VPXORQMasked256,
+ ssa.OpAMD64VPXORQMasked512,
+ ssa.OpAMD64VPBLENDMBMasked512,
+ ssa.OpAMD64VPBLENDMWMasked512,
+ ssa.OpAMD64VPBLENDMDMasked512,
+ ssa.OpAMD64VPBLENDMQMasked512:
+ p = simdV2kv(s, v)
+
+ case ssa.OpAMD64VPABSBMasked128,
+ ssa.OpAMD64VPABSBMasked256,
+ ssa.OpAMD64VPABSBMasked512,
+ ssa.OpAMD64VPABSWMasked128,
+ ssa.OpAMD64VPABSWMasked256,
+ ssa.OpAMD64VPABSWMasked512,
+ ssa.OpAMD64VPABSDMasked128,
+ ssa.OpAMD64VPABSDMasked256,
+ ssa.OpAMD64VPABSDMasked512,
+ ssa.OpAMD64VPABSQMasked128,
+ ssa.OpAMD64VPABSQMasked256,
+ ssa.OpAMD64VPABSQMasked512,
+ ssa.OpAMD64VCOMPRESSPSMasked128,
+ ssa.OpAMD64VCOMPRESSPSMasked256,
+ ssa.OpAMD64VCOMPRESSPSMasked512,
+ ssa.OpAMD64VCOMPRESSPDMasked128,
+ ssa.OpAMD64VCOMPRESSPDMasked256,
+ ssa.OpAMD64VCOMPRESSPDMasked512,
+ ssa.OpAMD64VPCOMPRESSBMasked128,
+ ssa.OpAMD64VPCOMPRESSBMasked256,
+ ssa.OpAMD64VPCOMPRESSBMasked512,
+ ssa.OpAMD64VPCOMPRESSWMasked128,
+ ssa.OpAMD64VPCOMPRESSWMasked256,
+ ssa.OpAMD64VPCOMPRESSWMasked512,
+ ssa.OpAMD64VPCOMPRESSDMasked128,
+ ssa.OpAMD64VPCOMPRESSDMasked256,
+ ssa.OpAMD64VPCOMPRESSDMasked512,
+ ssa.OpAMD64VPCOMPRESSQMasked128,
+ ssa.OpAMD64VPCOMPRESSQMasked256,
+ ssa.OpAMD64VPCOMPRESSQMasked512,
+ ssa.OpAMD64VCVTPD2PSXMasked128,
+ ssa.OpAMD64VCVTPD2PSYMasked128,
+ ssa.OpAMD64VCVTPD2PSMasked256,
+ ssa.OpAMD64VCVTDQ2PSMasked128,
+ ssa.OpAMD64VCVTDQ2PSMasked256,
+ ssa.OpAMD64VCVTDQ2PSMasked512,
+ ssa.OpAMD64VCVTQQ2PSXMasked128,
+ ssa.OpAMD64VCVTQQ2PSYMasked128,
+ ssa.OpAMD64VCVTQQ2PSMasked256,
+ ssa.OpAMD64VCVTUDQ2PSMasked128,
+ ssa.OpAMD64VCVTUDQ2PSMasked256,
+ ssa.OpAMD64VCVTUDQ2PSMasked512,
+ ssa.OpAMD64VCVTUQQ2PSXMasked128,
+ ssa.OpAMD64VCVTUQQ2PSYMasked128,
+ ssa.OpAMD64VCVTUQQ2PSMasked256,
+ ssa.OpAMD64VCVTPS2PDMasked256,
+ ssa.OpAMD64VCVTPS2PDMasked512,
+ ssa.OpAMD64VCVTDQ2PDMasked256,
+ ssa.OpAMD64VCVTDQ2PDMasked512,
+ ssa.OpAMD64VCVTQQ2PDMasked128,
+ ssa.OpAMD64VCVTQQ2PDMasked256,
+ ssa.OpAMD64VCVTQQ2PDMasked512,
+ ssa.OpAMD64VCVTUDQ2PDMasked256,
+ ssa.OpAMD64VCVTUDQ2PDMasked512,
+ ssa.OpAMD64VCVTUQQ2PDMasked128,
+ ssa.OpAMD64VCVTUQQ2PDMasked256,
+ ssa.OpAMD64VCVTUQQ2PDMasked512,
+ ssa.OpAMD64VCVTTPS2DQMasked128,
+ ssa.OpAMD64VCVTTPS2DQMasked256,
+ ssa.OpAMD64VCVTTPS2DQMasked512,
+ ssa.OpAMD64VCVTTPD2DQXMasked128,
+ ssa.OpAMD64VCVTTPD2DQYMasked128,
+ ssa.OpAMD64VCVTTPD2DQMasked256,
+ ssa.OpAMD64VCVTTPS2QQMasked256,
+ ssa.OpAMD64VCVTTPS2QQMasked512,
+ ssa.OpAMD64VCVTTPD2QQMasked128,
+ ssa.OpAMD64VCVTTPD2QQMasked256,
+ ssa.OpAMD64VCVTTPD2QQMasked512,
+ ssa.OpAMD64VCVTTPS2UDQMasked128,
+ ssa.OpAMD64VCVTTPS2UDQMasked256,
+ ssa.OpAMD64VCVTTPS2UDQMasked512,
+ ssa.OpAMD64VCVTTPD2UDQXMasked128,
+ ssa.OpAMD64VCVTTPD2UDQYMasked128,
+ ssa.OpAMD64VCVTTPD2UDQMasked256,
+ ssa.OpAMD64VCVTTPS2UQQMasked256,
+ ssa.OpAMD64VCVTTPS2UQQMasked512,
+ ssa.OpAMD64VCVTTPD2UQQMasked128,
+ ssa.OpAMD64VCVTTPD2UQQMasked256,
+ ssa.OpAMD64VCVTTPD2UQQMasked512,
+ ssa.OpAMD64VEXPANDPSMasked128,
+ ssa.OpAMD64VEXPANDPSMasked256,
+ ssa.OpAMD64VEXPANDPSMasked512,
+ ssa.OpAMD64VEXPANDPDMasked128,
+ ssa.OpAMD64VEXPANDPDMasked256,
+ ssa.OpAMD64VEXPANDPDMasked512,
+ ssa.OpAMD64VPEXPANDBMasked128,
+ ssa.OpAMD64VPEXPANDBMasked256,
+ ssa.OpAMD64VPEXPANDBMasked512,
+ ssa.OpAMD64VPEXPANDWMasked128,
+ ssa.OpAMD64VPEXPANDWMasked256,
+ ssa.OpAMD64VPEXPANDWMasked512,
+ ssa.OpAMD64VPEXPANDDMasked128,
+ ssa.OpAMD64VPEXPANDDMasked256,
+ ssa.OpAMD64VPEXPANDDMasked512,
+ ssa.OpAMD64VPEXPANDQMasked128,
+ ssa.OpAMD64VPEXPANDQMasked256,
+ ssa.OpAMD64VPEXPANDQMasked512,
+ ssa.OpAMD64VPMOVSXBQMasked128,
+ ssa.OpAMD64VPMOVSXWQMasked128,
+ ssa.OpAMD64VPMOVSXDQMasked128,
+ ssa.OpAMD64VPMOVZXBQMasked128,
+ ssa.OpAMD64VPMOVZXWQMasked128,
+ ssa.OpAMD64VPMOVZXDQMasked128,
+ ssa.OpAMD64VPMOVSXBDMasked128,
+ ssa.OpAMD64VPMOVSXWDMasked128,
+ ssa.OpAMD64VPMOVSXBQMasked256,
+ ssa.OpAMD64VPMOVSXWQMasked256,
+ ssa.OpAMD64VPMOVZXBDMasked128,
+ ssa.OpAMD64VPMOVZXWDMasked128,
+ ssa.OpAMD64VPMOVZXBQMasked256,
+ ssa.OpAMD64VPMOVZXWQMasked256,
+ ssa.OpAMD64VPMOVSXBWMasked128,
+ ssa.OpAMD64VPMOVSXBDMasked256,
+ ssa.OpAMD64VPMOVSXBQMasked512,
+ ssa.OpAMD64VPMOVZXBWMasked128,
+ ssa.OpAMD64VPMOVZXBDMasked256,
+ ssa.OpAMD64VPMOVZXBQMasked512,
+ ssa.OpAMD64VPMOVSXBWMasked256,
+ ssa.OpAMD64VPMOVSXBWMasked512,
+ ssa.OpAMD64VPMOVSXBDMasked512,
+ ssa.OpAMD64VPMOVSXWDMasked256,
+ ssa.OpAMD64VPMOVSXWDMasked512,
+ ssa.OpAMD64VPMOVSXWQMasked512,
+ ssa.OpAMD64VPMOVSXDQMasked256,
+ ssa.OpAMD64VPMOVSXDQMasked512,
+ ssa.OpAMD64VPMOVZXBWMasked256,
+ ssa.OpAMD64VPMOVZXBWMasked512,
+ ssa.OpAMD64VPMOVZXBDMasked512,
+ ssa.OpAMD64VPMOVZXWDMasked256,
+ ssa.OpAMD64VPMOVZXWDMasked512,
+ ssa.OpAMD64VPMOVZXWQMasked512,
+ ssa.OpAMD64VPMOVZXDQMasked256,
+ ssa.OpAMD64VPMOVZXDQMasked512,
+ ssa.OpAMD64VPLZCNTDMasked128,
+ ssa.OpAMD64VPLZCNTDMasked256,
+ ssa.OpAMD64VPLZCNTDMasked512,
+ ssa.OpAMD64VPLZCNTQMasked128,
+ ssa.OpAMD64VPLZCNTQMasked256,
+ ssa.OpAMD64VPLZCNTQMasked512,
+ ssa.OpAMD64VPOPCNTBMasked128,
+ ssa.OpAMD64VPOPCNTBMasked256,
+ ssa.OpAMD64VPOPCNTBMasked512,
+ ssa.OpAMD64VPOPCNTWMasked128,
+ ssa.OpAMD64VPOPCNTWMasked256,
+ ssa.OpAMD64VPOPCNTWMasked512,
+ ssa.OpAMD64VPOPCNTDMasked128,
+ ssa.OpAMD64VPOPCNTDMasked256,
+ ssa.OpAMD64VPOPCNTDMasked512,
+ ssa.OpAMD64VPOPCNTQMasked128,
+ ssa.OpAMD64VPOPCNTQMasked256,
+ ssa.OpAMD64VPOPCNTQMasked512,
+ ssa.OpAMD64VRCP14PSMasked128,
+ ssa.OpAMD64VRCP14PSMasked256,
+ ssa.OpAMD64VRCP14PSMasked512,
+ ssa.OpAMD64VRCP14PDMasked128,
+ ssa.OpAMD64VRCP14PDMasked256,
+ ssa.OpAMD64VRCP14PDMasked512,
+ ssa.OpAMD64VRSQRT14PSMasked128,
+ ssa.OpAMD64VRSQRT14PSMasked256,
+ ssa.OpAMD64VRSQRT14PSMasked512,
+ ssa.OpAMD64VRSQRT14PDMasked128,
+ ssa.OpAMD64VRSQRT14PDMasked256,
+ ssa.OpAMD64VRSQRT14PDMasked512,
+ ssa.OpAMD64VPMOVSWBMasked128_128,
+ ssa.OpAMD64VPMOVSWBMasked128_256,
+ ssa.OpAMD64VPMOVSWBMasked256,
+ ssa.OpAMD64VPMOVSDBMasked128_128,
+ ssa.OpAMD64VPMOVSDBMasked128_256,
+ ssa.OpAMD64VPMOVSDBMasked128_512,
+ ssa.OpAMD64VPMOVSQBMasked128_128,
+ ssa.OpAMD64VPMOVSQBMasked128_256,
+ ssa.OpAMD64VPMOVSQBMasked128_512,
+ ssa.OpAMD64VPMOVSDWMasked128_128,
+ ssa.OpAMD64VPMOVSDWMasked128_256,
+ ssa.OpAMD64VPMOVSDWMasked256,
+ ssa.OpAMD64VPMOVSQWMasked128_128,
+ ssa.OpAMD64VPMOVSQWMasked128_256,
+ ssa.OpAMD64VPMOVSQWMasked128_512,
+ ssa.OpAMD64VPMOVSQDMasked128_128,
+ ssa.OpAMD64VPMOVSQDMasked128_256,
+ ssa.OpAMD64VPMOVSQDMasked256,
+ ssa.OpAMD64VPMOVUSWBMasked128_128,
+ ssa.OpAMD64VPMOVUSWBMasked128_256,
+ ssa.OpAMD64VPMOVUSWBMasked256,
+ ssa.OpAMD64VPMOVUSDBMasked128_128,
+ ssa.OpAMD64VPMOVUSDBMasked128_256,
+ ssa.OpAMD64VPMOVUSDBMasked128_512,
+ ssa.OpAMD64VPMOVUSQBMasked128_128,
+ ssa.OpAMD64VPMOVUSQBMasked128_256,
+ ssa.OpAMD64VPMOVUSQBMasked128_512,
+ ssa.OpAMD64VPMOVUSDWMasked128_128,
+ ssa.OpAMD64VPMOVUSDWMasked128_256,
+ ssa.OpAMD64VPMOVUSDWMasked256,
+ ssa.OpAMD64VPMOVUSQWMasked128_128,
+ ssa.OpAMD64VPMOVUSQWMasked128_256,
+ ssa.OpAMD64VPMOVUSQWMasked128_512,
+ ssa.OpAMD64VPMOVUSQDMasked128_128,
+ ssa.OpAMD64VPMOVUSQDMasked128_256,
+ ssa.OpAMD64VPMOVUSQDMasked256,
+ ssa.OpAMD64VSQRTPSMasked128,
+ ssa.OpAMD64VSQRTPSMasked256,
+ ssa.OpAMD64VSQRTPSMasked512,
+ ssa.OpAMD64VSQRTPDMasked128,
+ ssa.OpAMD64VSQRTPDMasked256,
+ ssa.OpAMD64VSQRTPDMasked512,
+ ssa.OpAMD64VPMOVWBMasked128_128,
+ ssa.OpAMD64VPMOVWBMasked128_256,
+ ssa.OpAMD64VPMOVWBMasked256,
+ ssa.OpAMD64VPMOVDBMasked128_128,
+ ssa.OpAMD64VPMOVDBMasked128_256,
+ ssa.OpAMD64VPMOVDBMasked128_512,
+ ssa.OpAMD64VPMOVQBMasked128_128,
+ ssa.OpAMD64VPMOVQBMasked128_256,
+ ssa.OpAMD64VPMOVQBMasked128_512,
+ ssa.OpAMD64VPMOVDWMasked128_128,
+ ssa.OpAMD64VPMOVDWMasked128_256,
+ ssa.OpAMD64VPMOVDWMasked256,
+ ssa.OpAMD64VPMOVQWMasked128_128,
+ ssa.OpAMD64VPMOVQWMasked128_256,
+ ssa.OpAMD64VPMOVQWMasked128_512,
+ ssa.OpAMD64VPMOVQDMasked128_128,
+ ssa.OpAMD64VPMOVQDMasked128_256,
+ ssa.OpAMD64VPMOVQDMasked256,
+ ssa.OpAMD64VPBROADCASTQMasked128,
+ ssa.OpAMD64VBROADCASTSSMasked128,
+ ssa.OpAMD64VBROADCASTSDMasked256,
+ ssa.OpAMD64VPBROADCASTDMasked128,
+ ssa.OpAMD64VPBROADCASTQMasked256,
+ ssa.OpAMD64VBROADCASTSSMasked256,
+ ssa.OpAMD64VBROADCASTSDMasked512,
+ ssa.OpAMD64VPBROADCASTWMasked128,
+ ssa.OpAMD64VPBROADCASTDMasked256,
+ ssa.OpAMD64VPBROADCASTQMasked512,
+ ssa.OpAMD64VBROADCASTSSMasked512,
+ ssa.OpAMD64VPBROADCASTBMasked128,
+ ssa.OpAMD64VPBROADCASTWMasked256,
+ ssa.OpAMD64VPBROADCASTDMasked512,
+ ssa.OpAMD64VPBROADCASTBMasked256,
+ ssa.OpAMD64VPBROADCASTWMasked512,
+ ssa.OpAMD64VPBROADCASTBMasked512,
+ ssa.OpAMD64VMOVDQU8Masked128,
+ ssa.OpAMD64VMOVDQU8Masked256,
+ ssa.OpAMD64VMOVDQU8Masked512,
+ ssa.OpAMD64VMOVDQU16Masked128,
+ ssa.OpAMD64VMOVDQU16Masked256,
+ ssa.OpAMD64VMOVDQU16Masked512,
+ ssa.OpAMD64VMOVDQU32Masked128,
+ ssa.OpAMD64VMOVDQU32Masked256,
+ ssa.OpAMD64VMOVDQU32Masked512,
+ ssa.OpAMD64VMOVDQU64Masked128,
+ ssa.OpAMD64VMOVDQU64Masked256,
+ ssa.OpAMD64VMOVDQU64Masked512:
+ p = simdVkv(s, v)
+
+ case ssa.OpAMD64VPBLENDVB128,
+ ssa.OpAMD64VPBLENDVB256:
+ p = simdV31(s, v)
+
+ case ssa.OpAMD64VAESKEYGENASSIST128,
+ ssa.OpAMD64VROUNDPS128,
+ ssa.OpAMD64VROUNDPS256,
+ ssa.OpAMD64VROUNDPD128,
+ ssa.OpAMD64VROUNDPD256,
+ ssa.OpAMD64VRNDSCALEPS128,
+ ssa.OpAMD64VRNDSCALEPS256,
+ ssa.OpAMD64VRNDSCALEPS512,
+ ssa.OpAMD64VRNDSCALEPD128,
+ ssa.OpAMD64VRNDSCALEPD256,
+ ssa.OpAMD64VRNDSCALEPD512,
+ ssa.OpAMD64VREDUCEPS128,
+ ssa.OpAMD64VREDUCEPS256,
+ ssa.OpAMD64VREDUCEPS512,
+ ssa.OpAMD64VREDUCEPD128,
+ ssa.OpAMD64VREDUCEPD256,
+ ssa.OpAMD64VREDUCEPD512,
+ ssa.OpAMD64VEXTRACTF128128,
+ ssa.OpAMD64VEXTRACTF64X4256,
+ ssa.OpAMD64VEXTRACTI128128,
+ ssa.OpAMD64VEXTRACTI64X4256,
+ ssa.OpAMD64VPSHUFD128,
+ ssa.OpAMD64VPSHUFD256,
+ ssa.OpAMD64VPSHUFD512,
+ ssa.OpAMD64VPSHUFHW128,
+ ssa.OpAMD64VPSHUFHW256,
+ ssa.OpAMD64VPSHUFHW512,
+ ssa.OpAMD64VPSHUFLW128,
+ ssa.OpAMD64VPSHUFLW256,
+ ssa.OpAMD64VPSHUFLW512,
+ ssa.OpAMD64VPROLD128,
+ ssa.OpAMD64VPROLD256,
+ ssa.OpAMD64VPROLD512,
+ ssa.OpAMD64VPROLQ128,
+ ssa.OpAMD64VPROLQ256,
+ ssa.OpAMD64VPROLQ512,
+ ssa.OpAMD64VPRORD128,
+ ssa.OpAMD64VPRORD256,
+ ssa.OpAMD64VPRORD512,
+ ssa.OpAMD64VPRORQ128,
+ ssa.OpAMD64VPRORQ256,
+ ssa.OpAMD64VPRORQ512,
+ ssa.OpAMD64VPSLLW128const,
+ ssa.OpAMD64VPSLLW256const,
+ ssa.OpAMD64VPSLLW512const,
+ ssa.OpAMD64VPSLLD128const,
+ ssa.OpAMD64VPSLLD256const,
+ ssa.OpAMD64VPSLLD512const,
+ ssa.OpAMD64VPSLLQ128const,
+ ssa.OpAMD64VPSLLQ256const,
+ ssa.OpAMD64VPSLLQ512const,
+ ssa.OpAMD64VPSRAW128const,
+ ssa.OpAMD64VPSRAW256const,
+ ssa.OpAMD64VPSRAW512const,
+ ssa.OpAMD64VPSRAD128const,
+ ssa.OpAMD64VPSRAD256const,
+ ssa.OpAMD64VPSRAD512const,
+ ssa.OpAMD64VPSRAQ128const,
+ ssa.OpAMD64VPSRAQ256const,
+ ssa.OpAMD64VPSRAQ512const,
+ ssa.OpAMD64VPSRLW128const,
+ ssa.OpAMD64VPSRLW256const,
+ ssa.OpAMD64VPSRLW512const,
+ ssa.OpAMD64VPSRLD128const,
+ ssa.OpAMD64VPSRLD256const,
+ ssa.OpAMD64VPSRLD512const,
+ ssa.OpAMD64VPSRLQ128const,
+ ssa.OpAMD64VPSRLQ256const,
+ ssa.OpAMD64VPSRLQ512const:
+ p = simdV11Imm8(s, v)
+
+ case ssa.OpAMD64VRNDSCALEPSMasked128,
+ ssa.OpAMD64VRNDSCALEPSMasked256,
+ ssa.OpAMD64VRNDSCALEPSMasked512,
+ ssa.OpAMD64VRNDSCALEPDMasked128,
+ ssa.OpAMD64VRNDSCALEPDMasked256,
+ ssa.OpAMD64VRNDSCALEPDMasked512,
+ ssa.OpAMD64VREDUCEPSMasked128,
+ ssa.OpAMD64VREDUCEPSMasked256,
+ ssa.OpAMD64VREDUCEPSMasked512,
+ ssa.OpAMD64VREDUCEPDMasked128,
+ ssa.OpAMD64VREDUCEPDMasked256,
+ ssa.OpAMD64VREDUCEPDMasked512,
+ ssa.OpAMD64VPSHUFDMasked256,
+ ssa.OpAMD64VPSHUFDMasked512,
+ ssa.OpAMD64VPSHUFHWMasked256,
+ ssa.OpAMD64VPSHUFHWMasked512,
+ ssa.OpAMD64VPSHUFHWMasked128,
+ ssa.OpAMD64VPSHUFLWMasked256,
+ ssa.OpAMD64VPSHUFLWMasked512,
+ ssa.OpAMD64VPSHUFLWMasked128,
+ ssa.OpAMD64VPSHUFDMasked128,
+ ssa.OpAMD64VPROLDMasked128,
+ ssa.OpAMD64VPROLDMasked256,
+ ssa.OpAMD64VPROLDMasked512,
+ ssa.OpAMD64VPROLQMasked128,
+ ssa.OpAMD64VPROLQMasked256,
+ ssa.OpAMD64VPROLQMasked512,
+ ssa.OpAMD64VPRORDMasked128,
+ ssa.OpAMD64VPRORDMasked256,
+ ssa.OpAMD64VPRORDMasked512,
+ ssa.OpAMD64VPRORQMasked128,
+ ssa.OpAMD64VPRORQMasked256,
+ ssa.OpAMD64VPRORQMasked512,
+ ssa.OpAMD64VPSLLWMasked128const,
+ ssa.OpAMD64VPSLLWMasked256const,
+ ssa.OpAMD64VPSLLWMasked512const,
+ ssa.OpAMD64VPSLLDMasked128const,
+ ssa.OpAMD64VPSLLDMasked256const,
+ ssa.OpAMD64VPSLLDMasked512const,
+ ssa.OpAMD64VPSLLQMasked128const,
+ ssa.OpAMD64VPSLLQMasked256const,
+ ssa.OpAMD64VPSLLQMasked512const,
+ ssa.OpAMD64VPSRAWMasked128const,
+ ssa.OpAMD64VPSRAWMasked256const,
+ ssa.OpAMD64VPSRAWMasked512const,
+ ssa.OpAMD64VPSRADMasked128const,
+ ssa.OpAMD64VPSRADMasked256const,
+ ssa.OpAMD64VPSRADMasked512const,
+ ssa.OpAMD64VPSRAQMasked128const,
+ ssa.OpAMD64VPSRAQMasked256const,
+ ssa.OpAMD64VPSRAQMasked512const,
+ ssa.OpAMD64VPSRLWMasked128const,
+ ssa.OpAMD64VPSRLWMasked256const,
+ ssa.OpAMD64VPSRLWMasked512const,
+ ssa.OpAMD64VPSRLDMasked128const,
+ ssa.OpAMD64VPSRLDMasked256const,
+ ssa.OpAMD64VPSRLDMasked512const,
+ ssa.OpAMD64VPSRLQMasked128const,
+ ssa.OpAMD64VPSRLQMasked256const,
+ ssa.OpAMD64VPSRLQMasked512const:
+ p = simdVkvImm8(s, v)
+
+ case ssa.OpAMD64VPERM2F128256,
+ ssa.OpAMD64VPERM2I128256,
+ ssa.OpAMD64VPALIGNR128,
+ ssa.OpAMD64VPALIGNR256,
+ ssa.OpAMD64VPALIGNR512,
+ ssa.OpAMD64VCMPPS128,
+ ssa.OpAMD64VCMPPS256,
+ ssa.OpAMD64VCMPPD128,
+ ssa.OpAMD64VCMPPD256,
+ ssa.OpAMD64VGF2P8AFFINEQB128,
+ ssa.OpAMD64VGF2P8AFFINEQB256,
+ ssa.OpAMD64VGF2P8AFFINEQB512,
+ ssa.OpAMD64VGF2P8AFFINEINVQB128,
+ ssa.OpAMD64VGF2P8AFFINEINVQB256,
+ ssa.OpAMD64VGF2P8AFFINEINVQB512,
+ ssa.OpAMD64VINSERTF128256,
+ ssa.OpAMD64VINSERTF64X4512,
+ ssa.OpAMD64VINSERTI128256,
+ ssa.OpAMD64VINSERTI64X4512,
+ ssa.OpAMD64VPSHLDW128,
+ ssa.OpAMD64VPSHLDW256,
+ ssa.OpAMD64VPSHLDW512,
+ ssa.OpAMD64VPSHLDD128,
+ ssa.OpAMD64VPSHLDD256,
+ ssa.OpAMD64VPSHLDD512,
+ ssa.OpAMD64VPSHLDQ128,
+ ssa.OpAMD64VPSHLDQ256,
+ ssa.OpAMD64VPSHLDQ512,
+ ssa.OpAMD64VPSHRDW128,
+ ssa.OpAMD64VPSHRDW256,
+ ssa.OpAMD64VPSHRDW512,
+ ssa.OpAMD64VPSHRDD128,
+ ssa.OpAMD64VPSHRDD256,
+ ssa.OpAMD64VPSHRDD512,
+ ssa.OpAMD64VPSHRDQ128,
+ ssa.OpAMD64VPSHRDQ256,
+ ssa.OpAMD64VPSHRDQ512,
+ ssa.OpAMD64VPCLMULQDQ128,
+ ssa.OpAMD64VPCLMULQDQ256,
+ ssa.OpAMD64VPCLMULQDQ512,
+ ssa.OpAMD64VSHUFPS128,
+ ssa.OpAMD64VSHUFPD128,
+ ssa.OpAMD64VSHUFPS256,
+ ssa.OpAMD64VSHUFPS512,
+ ssa.OpAMD64VSHUFPD256,
+ ssa.OpAMD64VSHUFPD512:
+ p = simdV21Imm8(s, v)
+
+ case ssa.OpAMD64VCMPPS512,
+ ssa.OpAMD64VCMPPD512,
+ ssa.OpAMD64VPCMPUB512,
+ ssa.OpAMD64VPCMPUW512,
+ ssa.OpAMD64VPCMPUD512,
+ ssa.OpAMD64VPCMPUQ512,
+ ssa.OpAMD64VPCMPB512,
+ ssa.OpAMD64VPCMPW512,
+ ssa.OpAMD64VPCMPD512,
+ ssa.OpAMD64VPCMPQ512:
+ p = simdV2kImm8(s, v)
+
+ case ssa.OpAMD64VCMPPSMasked128,
+ ssa.OpAMD64VCMPPSMasked256,
+ ssa.OpAMD64VCMPPSMasked512,
+ ssa.OpAMD64VCMPPDMasked128,
+ ssa.OpAMD64VCMPPDMasked256,
+ ssa.OpAMD64VCMPPDMasked512,
+ ssa.OpAMD64VPCMPBMasked128,
+ ssa.OpAMD64VPCMPBMasked256,
+ ssa.OpAMD64VPCMPBMasked512,
+ ssa.OpAMD64VPCMPWMasked128,
+ ssa.OpAMD64VPCMPWMasked256,
+ ssa.OpAMD64VPCMPWMasked512,
+ ssa.OpAMD64VPCMPDMasked128,
+ ssa.OpAMD64VPCMPDMasked256,
+ ssa.OpAMD64VPCMPDMasked512,
+ ssa.OpAMD64VPCMPQMasked128,
+ ssa.OpAMD64VPCMPQMasked256,
+ ssa.OpAMD64VPCMPQMasked512,
+ ssa.OpAMD64VPCMPUBMasked128,
+ ssa.OpAMD64VPCMPUBMasked256,
+ ssa.OpAMD64VPCMPUBMasked512,
+ ssa.OpAMD64VPCMPUWMasked128,
+ ssa.OpAMD64VPCMPUWMasked256,
+ ssa.OpAMD64VPCMPUWMasked512,
+ ssa.OpAMD64VPCMPUDMasked128,
+ ssa.OpAMD64VPCMPUDMasked256,
+ ssa.OpAMD64VPCMPUDMasked512,
+ ssa.OpAMD64VPCMPUQMasked128,
+ ssa.OpAMD64VPCMPUQMasked256,
+ ssa.OpAMD64VPCMPUQMasked512:
+ p = simdV2kkImm8(s, v)
+
+ case ssa.OpAMD64VPDPWSSD128,
+ ssa.OpAMD64VPDPWSSD256,
+ ssa.OpAMD64VPDPWSSD512,
+ ssa.OpAMD64VPERMI2B128,
+ ssa.OpAMD64VPERMI2B256,
+ ssa.OpAMD64VPERMI2B512,
+ ssa.OpAMD64VPERMI2W128,
+ ssa.OpAMD64VPERMI2W256,
+ ssa.OpAMD64VPERMI2W512,
+ ssa.OpAMD64VPERMI2PS128,
+ ssa.OpAMD64VPERMI2D128,
+ ssa.OpAMD64VPERMI2PS256,
+ ssa.OpAMD64VPERMI2D256,
+ ssa.OpAMD64VPERMI2PS512,
+ ssa.OpAMD64VPERMI2D512,
+ ssa.OpAMD64VPERMI2PD128,
+ ssa.OpAMD64VPERMI2Q128,
+ ssa.OpAMD64VPERMI2PD256,
+ ssa.OpAMD64VPERMI2Q256,
+ ssa.OpAMD64VPERMI2PD512,
+ ssa.OpAMD64VPERMI2Q512,
+ ssa.OpAMD64VFMADD213PS128,
+ ssa.OpAMD64VFMADD213PS256,
+ ssa.OpAMD64VFMADD213PS512,
+ ssa.OpAMD64VFMADD213PD128,
+ ssa.OpAMD64VFMADD213PD256,
+ ssa.OpAMD64VFMADD213PD512,
+ ssa.OpAMD64VFMADDSUB213PS128,
+ ssa.OpAMD64VFMADDSUB213PS256,
+ ssa.OpAMD64VFMADDSUB213PS512,
+ ssa.OpAMD64VFMADDSUB213PD128,
+ ssa.OpAMD64VFMADDSUB213PD256,
+ ssa.OpAMD64VFMADDSUB213PD512,
+ ssa.OpAMD64VFMSUBADD213PS128,
+ ssa.OpAMD64VFMSUBADD213PS256,
+ ssa.OpAMD64VFMSUBADD213PS512,
+ ssa.OpAMD64VFMSUBADD213PD128,
+ ssa.OpAMD64VFMSUBADD213PD256,
+ ssa.OpAMD64VFMSUBADD213PD512,
+ ssa.OpAMD64VPSHLDVW128,
+ ssa.OpAMD64VPSHLDVW256,
+ ssa.OpAMD64VPSHLDVW512,
+ ssa.OpAMD64VPSHLDVD128,
+ ssa.OpAMD64VPSHLDVD256,
+ ssa.OpAMD64VPSHLDVD512,
+ ssa.OpAMD64VPSHLDVQ128,
+ ssa.OpAMD64VPSHLDVQ256,
+ ssa.OpAMD64VPSHLDVQ512,
+ ssa.OpAMD64VPSHRDVW128,
+ ssa.OpAMD64VPSHRDVW256,
+ ssa.OpAMD64VPSHRDVW512,
+ ssa.OpAMD64VPSHRDVD128,
+ ssa.OpAMD64VPSHRDVD256,
+ ssa.OpAMD64VPSHRDVD512,
+ ssa.OpAMD64VPSHRDVQ128,
+ ssa.OpAMD64VPSHRDVQ256,
+ ssa.OpAMD64VPSHRDVQ512:
+ p = simdV31ResultInArg0(s, v)
+
+ case ssa.OpAMD64VPDPWSSDMasked128,
+ ssa.OpAMD64VPDPWSSDMasked256,
+ ssa.OpAMD64VPDPWSSDMasked512,
+ ssa.OpAMD64VADDPSMasked128Merging,
+ ssa.OpAMD64VADDPSMasked256Merging,
+ ssa.OpAMD64VADDPSMasked512Merging,
+ ssa.OpAMD64VADDPDMasked128Merging,
+ ssa.OpAMD64VADDPDMasked256Merging,
+ ssa.OpAMD64VADDPDMasked512Merging,
+ ssa.OpAMD64VPADDBMasked128Merging,
+ ssa.OpAMD64VPADDBMasked256Merging,
+ ssa.OpAMD64VPADDBMasked512Merging,
+ ssa.OpAMD64VPADDWMasked128Merging,
+ ssa.OpAMD64VPADDWMasked256Merging,
+ ssa.OpAMD64VPADDWMasked512Merging,
+ ssa.OpAMD64VPADDDMasked128Merging,
+ ssa.OpAMD64VPADDDMasked256Merging,
+ ssa.OpAMD64VPADDDMasked512Merging,
+ ssa.OpAMD64VPADDQMasked128Merging,
+ ssa.OpAMD64VPADDQMasked256Merging,
+ ssa.OpAMD64VPADDQMasked512Merging,
+ ssa.OpAMD64VPADDSBMasked128Merging,
+ ssa.OpAMD64VPADDSBMasked256Merging,
+ ssa.OpAMD64VPADDSBMasked512Merging,
+ ssa.OpAMD64VPADDSWMasked128Merging,
+ ssa.OpAMD64VPADDSWMasked256Merging,
+ ssa.OpAMD64VPADDSWMasked512Merging,
+ ssa.OpAMD64VPADDUSBMasked128Merging,
+ ssa.OpAMD64VPADDUSBMasked256Merging,
+ ssa.OpAMD64VPADDUSBMasked512Merging,
+ ssa.OpAMD64VPADDUSWMasked128Merging,
+ ssa.OpAMD64VPADDUSWMasked256Merging,
+ ssa.OpAMD64VPADDUSWMasked512Merging,
+ ssa.OpAMD64VPANDDMasked128Merging,
+ ssa.OpAMD64VPANDDMasked256Merging,
+ ssa.OpAMD64VPANDDMasked512Merging,
+ ssa.OpAMD64VPANDQMasked128Merging,
+ ssa.OpAMD64VPANDQMasked256Merging,
+ ssa.OpAMD64VPANDQMasked512Merging,
+ ssa.OpAMD64VPAVGBMasked128Merging,
+ ssa.OpAMD64VPAVGBMasked256Merging,
+ ssa.OpAMD64VPAVGBMasked512Merging,
+ ssa.OpAMD64VPAVGWMasked128Merging,
+ ssa.OpAMD64VPAVGWMasked256Merging,
+ ssa.OpAMD64VPAVGWMasked512Merging,
+ ssa.OpAMD64VPERMI2BMasked128,
+ ssa.OpAMD64VPERMI2BMasked256,
+ ssa.OpAMD64VPERMI2BMasked512,
+ ssa.OpAMD64VPERMI2WMasked128,
+ ssa.OpAMD64VPERMI2WMasked256,
+ ssa.OpAMD64VPERMI2WMasked512,
+ ssa.OpAMD64VPERMI2PSMasked128,
+ ssa.OpAMD64VPERMI2DMasked128,
+ ssa.OpAMD64VPERMI2PSMasked256,
+ ssa.OpAMD64VPERMI2DMasked256,
+ ssa.OpAMD64VPERMI2PSMasked512,
+ ssa.OpAMD64VPERMI2DMasked512,
+ ssa.OpAMD64VPERMI2PDMasked128,
+ ssa.OpAMD64VPERMI2QMasked128,
+ ssa.OpAMD64VPERMI2PDMasked256,
+ ssa.OpAMD64VPERMI2QMasked256,
+ ssa.OpAMD64VPERMI2PDMasked512,
+ ssa.OpAMD64VPERMI2QMasked512,
+ ssa.OpAMD64VDIVPSMasked128Merging,
+ ssa.OpAMD64VDIVPSMasked256Merging,
+ ssa.OpAMD64VDIVPSMasked512Merging,
+ ssa.OpAMD64VDIVPDMasked128Merging,
+ ssa.OpAMD64VDIVPDMasked256Merging,
+ ssa.OpAMD64VDIVPDMasked512Merging,
+ ssa.OpAMD64VPMADDWDMasked128Merging,
+ ssa.OpAMD64VPMADDWDMasked256Merging,
+ ssa.OpAMD64VPMADDWDMasked512Merging,
+ ssa.OpAMD64VPMADDUBSWMasked128Merging,
+ ssa.OpAMD64VPMADDUBSWMasked256Merging,
+ ssa.OpAMD64VPMADDUBSWMasked512Merging,
+ ssa.OpAMD64VGF2P8MULBMasked128Merging,
+ ssa.OpAMD64VGF2P8MULBMasked256Merging,
+ ssa.OpAMD64VGF2P8MULBMasked512Merging,
+ ssa.OpAMD64VMAXPSMasked128Merging,
+ ssa.OpAMD64VMAXPSMasked256Merging,
+ ssa.OpAMD64VMAXPSMasked512Merging,
+ ssa.OpAMD64VMAXPDMasked128Merging,
+ ssa.OpAMD64VMAXPDMasked256Merging,
+ ssa.OpAMD64VMAXPDMasked512Merging,
+ ssa.OpAMD64VPMAXSBMasked128Merging,
+ ssa.OpAMD64VPMAXSBMasked256Merging,
+ ssa.OpAMD64VPMAXSBMasked512Merging,
+ ssa.OpAMD64VPMAXSWMasked128Merging,
+ ssa.OpAMD64VPMAXSWMasked256Merging,
+ ssa.OpAMD64VPMAXSWMasked512Merging,
+ ssa.OpAMD64VPMAXSDMasked128Merging,
+ ssa.OpAMD64VPMAXSDMasked256Merging,
+ ssa.OpAMD64VPMAXSDMasked512Merging,
+ ssa.OpAMD64VPMAXSQMasked128Merging,
+ ssa.OpAMD64VPMAXSQMasked256Merging,
+ ssa.OpAMD64VPMAXSQMasked512Merging,
+ ssa.OpAMD64VPMAXUBMasked128Merging,
+ ssa.OpAMD64VPMAXUBMasked256Merging,
+ ssa.OpAMD64VPMAXUBMasked512Merging,
+ ssa.OpAMD64VPMAXUWMasked128Merging,
+ ssa.OpAMD64VPMAXUWMasked256Merging,
+ ssa.OpAMD64VPMAXUWMasked512Merging,
+ ssa.OpAMD64VPMAXUDMasked128Merging,
+ ssa.OpAMD64VPMAXUDMasked256Merging,
+ ssa.OpAMD64VPMAXUDMasked512Merging,
+ ssa.OpAMD64VPMAXUQMasked128Merging,
+ ssa.OpAMD64VPMAXUQMasked256Merging,
+ ssa.OpAMD64VPMAXUQMasked512Merging,
+ ssa.OpAMD64VMINPSMasked128Merging,
+ ssa.OpAMD64VMINPSMasked256Merging,
+ ssa.OpAMD64VMINPSMasked512Merging,
+ ssa.OpAMD64VMINPDMasked128Merging,
+ ssa.OpAMD64VMINPDMasked256Merging,
+ ssa.OpAMD64VMINPDMasked512Merging,
+ ssa.OpAMD64VPMINSBMasked128Merging,
+ ssa.OpAMD64VPMINSBMasked256Merging,
+ ssa.OpAMD64VPMINSBMasked512Merging,
+ ssa.OpAMD64VPMINSWMasked128Merging,
+ ssa.OpAMD64VPMINSWMasked256Merging,
+ ssa.OpAMD64VPMINSWMasked512Merging,
+ ssa.OpAMD64VPMINSDMasked128Merging,
+ ssa.OpAMD64VPMINSDMasked256Merging,
+ ssa.OpAMD64VPMINSDMasked512Merging,
+ ssa.OpAMD64VPMINSQMasked128Merging,
+ ssa.OpAMD64VPMINSQMasked256Merging,
+ ssa.OpAMD64VPMINSQMasked512Merging,
+ ssa.OpAMD64VPMINUBMasked128Merging,
+ ssa.OpAMD64VPMINUBMasked256Merging,
+ ssa.OpAMD64VPMINUBMasked512Merging,
+ ssa.OpAMD64VPMINUWMasked128Merging,
+ ssa.OpAMD64VPMINUWMasked256Merging,
+ ssa.OpAMD64VPMINUWMasked512Merging,
+ ssa.OpAMD64VPMINUDMasked128Merging,
+ ssa.OpAMD64VPMINUDMasked256Merging,
+ ssa.OpAMD64VPMINUDMasked512Merging,
+ ssa.OpAMD64VPMINUQMasked128Merging,
+ ssa.OpAMD64VPMINUQMasked256Merging,
+ ssa.OpAMD64VPMINUQMasked512Merging,
+ ssa.OpAMD64VFMADDSUB213PSMasked128,
+ ssa.OpAMD64VFMADDSUB213PSMasked256,
+ ssa.OpAMD64VFMADDSUB213PSMasked512,
+ ssa.OpAMD64VFMADDSUB213PDMasked128,
+ ssa.OpAMD64VFMADDSUB213PDMasked256,
+ ssa.OpAMD64VFMADDSUB213PDMasked512,
+ ssa.OpAMD64VFMADD213PSMasked128,
+ ssa.OpAMD64VFMADD213PSMasked256,
+ ssa.OpAMD64VFMADD213PSMasked512,
+ ssa.OpAMD64VFMADD213PDMasked128,
+ ssa.OpAMD64VFMADD213PDMasked256,
+ ssa.OpAMD64VFMADD213PDMasked512,
+ ssa.OpAMD64VFMSUBADD213PSMasked128,
+ ssa.OpAMD64VFMSUBADD213PSMasked256,
+ ssa.OpAMD64VFMSUBADD213PSMasked512,
+ ssa.OpAMD64VFMSUBADD213PDMasked128,
+ ssa.OpAMD64VFMSUBADD213PDMasked256,
+ ssa.OpAMD64VFMSUBADD213PDMasked512,
+ ssa.OpAMD64VPMULHWMasked128Merging,
+ ssa.OpAMD64VPMULHWMasked256Merging,
+ ssa.OpAMD64VPMULHWMasked512Merging,
+ ssa.OpAMD64VPMULHUWMasked128Merging,
+ ssa.OpAMD64VPMULHUWMasked256Merging,
+ ssa.OpAMD64VPMULHUWMasked512Merging,
+ ssa.OpAMD64VMULPSMasked128Merging,
+ ssa.OpAMD64VMULPSMasked256Merging,
+ ssa.OpAMD64VMULPSMasked512Merging,
+ ssa.OpAMD64VMULPDMasked128Merging,
+ ssa.OpAMD64VMULPDMasked256Merging,
+ ssa.OpAMD64VMULPDMasked512Merging,
+ ssa.OpAMD64VPMULLWMasked128Merging,
+ ssa.OpAMD64VPMULLWMasked256Merging,
+ ssa.OpAMD64VPMULLWMasked512Merging,
+ ssa.OpAMD64VPMULLDMasked128Merging,
+ ssa.OpAMD64VPMULLDMasked256Merging,
+ ssa.OpAMD64VPMULLDMasked512Merging,
+ ssa.OpAMD64VPMULLQMasked128Merging,
+ ssa.OpAMD64VPMULLQMasked256Merging,
+ ssa.OpAMD64VPMULLQMasked512Merging,
+ ssa.OpAMD64VPORDMasked128Merging,
+ ssa.OpAMD64VPORDMasked256Merging,
+ ssa.OpAMD64VPORDMasked512Merging,
+ ssa.OpAMD64VPORQMasked128Merging,
+ ssa.OpAMD64VPORQMasked256Merging,
+ ssa.OpAMD64VPORQMasked512Merging,
+ ssa.OpAMD64VPSHUFBMasked256Merging,
+ ssa.OpAMD64VPSHUFBMasked512Merging,
+ ssa.OpAMD64VPSHUFBMasked128Merging,
+ ssa.OpAMD64VPROLVDMasked128Merging,
+ ssa.OpAMD64VPROLVDMasked256Merging,
+ ssa.OpAMD64VPROLVDMasked512Merging,
+ ssa.OpAMD64VPROLVQMasked128Merging,
+ ssa.OpAMD64VPROLVQMasked256Merging,
+ ssa.OpAMD64VPROLVQMasked512Merging,
+ ssa.OpAMD64VPRORVDMasked128Merging,
+ ssa.OpAMD64VPRORVDMasked256Merging,
+ ssa.OpAMD64VPRORVDMasked512Merging,
+ ssa.OpAMD64VPRORVQMasked128Merging,
+ ssa.OpAMD64VPRORVQMasked256Merging,
+ ssa.OpAMD64VPRORVQMasked512Merging,
+ ssa.OpAMD64VPACKSSDWMasked256Merging,
+ ssa.OpAMD64VPACKSSDWMasked512Merging,
+ ssa.OpAMD64VPACKSSDWMasked128Merging,
+ ssa.OpAMD64VPACKUSDWMasked256Merging,
+ ssa.OpAMD64VPACKUSDWMasked512Merging,
+ ssa.OpAMD64VPACKUSDWMasked128Merging,
+ ssa.OpAMD64VSCALEFPSMasked128Merging,
+ ssa.OpAMD64VSCALEFPSMasked256Merging,
+ ssa.OpAMD64VSCALEFPSMasked512Merging,
+ ssa.OpAMD64VSCALEFPDMasked128Merging,
+ ssa.OpAMD64VSCALEFPDMasked256Merging,
+ ssa.OpAMD64VSCALEFPDMasked512Merging,
+ ssa.OpAMD64VPSHLDVWMasked128,
+ ssa.OpAMD64VPSHLDVWMasked256,
+ ssa.OpAMD64VPSHLDVWMasked512,
+ ssa.OpAMD64VPSHLDVDMasked128,
+ ssa.OpAMD64VPSHLDVDMasked256,
+ ssa.OpAMD64VPSHLDVDMasked512,
+ ssa.OpAMD64VPSHLDVQMasked128,
+ ssa.OpAMD64VPSHLDVQMasked256,
+ ssa.OpAMD64VPSHLDVQMasked512,
+ ssa.OpAMD64VPSLLVWMasked128Merging,
+ ssa.OpAMD64VPSLLVWMasked256Merging,
+ ssa.OpAMD64VPSLLVWMasked512Merging,
+ ssa.OpAMD64VPSLLVDMasked128Merging,
+ ssa.OpAMD64VPSLLVDMasked256Merging,
+ ssa.OpAMD64VPSLLVDMasked512Merging,
+ ssa.OpAMD64VPSLLVQMasked128Merging,
+ ssa.OpAMD64VPSLLVQMasked256Merging,
+ ssa.OpAMD64VPSLLVQMasked512Merging,
+ ssa.OpAMD64VPSHRDVWMasked128,
+ ssa.OpAMD64VPSHRDVWMasked256,
+ ssa.OpAMD64VPSHRDVWMasked512,
+ ssa.OpAMD64VPSHRDVDMasked128,
+ ssa.OpAMD64VPSHRDVDMasked256,
+ ssa.OpAMD64VPSHRDVDMasked512,
+ ssa.OpAMD64VPSHRDVQMasked128,
+ ssa.OpAMD64VPSHRDVQMasked256,
+ ssa.OpAMD64VPSHRDVQMasked512,
+ ssa.OpAMD64VPSRAVWMasked128Merging,
+ ssa.OpAMD64VPSRAVWMasked256Merging,
+ ssa.OpAMD64VPSRAVWMasked512Merging,
+ ssa.OpAMD64VPSRAVDMasked128Merging,
+ ssa.OpAMD64VPSRAVDMasked256Merging,
+ ssa.OpAMD64VPSRAVDMasked512Merging,
+ ssa.OpAMD64VPSRAVQMasked128Merging,
+ ssa.OpAMD64VPSRAVQMasked256Merging,
+ ssa.OpAMD64VPSRAVQMasked512Merging,
+ ssa.OpAMD64VPSRLVWMasked128Merging,
+ ssa.OpAMD64VPSRLVWMasked256Merging,
+ ssa.OpAMD64VPSRLVWMasked512Merging,
+ ssa.OpAMD64VPSRLVDMasked128Merging,
+ ssa.OpAMD64VPSRLVDMasked256Merging,
+ ssa.OpAMD64VPSRLVDMasked512Merging,
+ ssa.OpAMD64VPSRLVQMasked128Merging,
+ ssa.OpAMD64VPSRLVQMasked256Merging,
+ ssa.OpAMD64VPSRLVQMasked512Merging,
+ ssa.OpAMD64VSUBPSMasked128Merging,
+ ssa.OpAMD64VSUBPSMasked256Merging,
+ ssa.OpAMD64VSUBPSMasked512Merging,
+ ssa.OpAMD64VSUBPDMasked128Merging,
+ ssa.OpAMD64VSUBPDMasked256Merging,
+ ssa.OpAMD64VSUBPDMasked512Merging,
+ ssa.OpAMD64VPSUBBMasked128Merging,
+ ssa.OpAMD64VPSUBBMasked256Merging,
+ ssa.OpAMD64VPSUBBMasked512Merging,
+ ssa.OpAMD64VPSUBWMasked128Merging,
+ ssa.OpAMD64VPSUBWMasked256Merging,
+ ssa.OpAMD64VPSUBWMasked512Merging,
+ ssa.OpAMD64VPSUBDMasked128Merging,
+ ssa.OpAMD64VPSUBDMasked256Merging,
+ ssa.OpAMD64VPSUBDMasked512Merging,
+ ssa.OpAMD64VPSUBQMasked128Merging,
+ ssa.OpAMD64VPSUBQMasked256Merging,
+ ssa.OpAMD64VPSUBQMasked512Merging,
+ ssa.OpAMD64VPSUBSBMasked128Merging,
+ ssa.OpAMD64VPSUBSBMasked256Merging,
+ ssa.OpAMD64VPSUBSBMasked512Merging,
+ ssa.OpAMD64VPSUBSWMasked128Merging,
+ ssa.OpAMD64VPSUBSWMasked256Merging,
+ ssa.OpAMD64VPSUBSWMasked512Merging,
+ ssa.OpAMD64VPSUBUSBMasked128Merging,
+ ssa.OpAMD64VPSUBUSBMasked256Merging,
+ ssa.OpAMD64VPSUBUSBMasked512Merging,
+ ssa.OpAMD64VPSUBUSWMasked128Merging,
+ ssa.OpAMD64VPSUBUSWMasked256Merging,
+ ssa.OpAMD64VPSUBUSWMasked512Merging,
+ ssa.OpAMD64VPXORDMasked128Merging,
+ ssa.OpAMD64VPXORDMasked256Merging,
+ ssa.OpAMD64VPXORDMasked512Merging,
+ ssa.OpAMD64VPXORQMasked128Merging,
+ ssa.OpAMD64VPXORQMasked256Merging,
+ ssa.OpAMD64VPXORQMasked512Merging:
+ p = simdV3kvResultInArg0(s, v)
+
+ case ssa.OpAMD64VPSLLW128,
+ ssa.OpAMD64VPSLLW256,
+ ssa.OpAMD64VPSLLW512,
+ ssa.OpAMD64VPSLLD128,
+ ssa.OpAMD64VPSLLD256,
+ ssa.OpAMD64VPSLLD512,
+ ssa.OpAMD64VPSLLQ128,
+ ssa.OpAMD64VPSLLQ256,
+ ssa.OpAMD64VPSLLQ512,
+ ssa.OpAMD64VPSRAW128,
+ ssa.OpAMD64VPSRAW256,
+ ssa.OpAMD64VPSRAW512,
+ ssa.OpAMD64VPSRAD128,
+ ssa.OpAMD64VPSRAD256,
+ ssa.OpAMD64VPSRAD512,
+ ssa.OpAMD64VPSRAQ128,
+ ssa.OpAMD64VPSRAQ256,
+ ssa.OpAMD64VPSRAQ512,
+ ssa.OpAMD64VPSRLW128,
+ ssa.OpAMD64VPSRLW256,
+ ssa.OpAMD64VPSRLW512,
+ ssa.OpAMD64VPSRLD128,
+ ssa.OpAMD64VPSRLD256,
+ ssa.OpAMD64VPSRLD512,
+ ssa.OpAMD64VPSRLQ128,
+ ssa.OpAMD64VPSRLQ256,
+ ssa.OpAMD64VPSRLQ512:
+ p = simdVfpv(s, v)
+
+ case ssa.OpAMD64VPSLLWMasked128,
+ ssa.OpAMD64VPSLLWMasked256,
+ ssa.OpAMD64VPSLLWMasked512,
+ ssa.OpAMD64VPSLLDMasked128,
+ ssa.OpAMD64VPSLLDMasked256,
+ ssa.OpAMD64VPSLLDMasked512,
+ ssa.OpAMD64VPSLLQMasked128,
+ ssa.OpAMD64VPSLLQMasked256,
+ ssa.OpAMD64VPSLLQMasked512,
+ ssa.OpAMD64VPSRAWMasked128,
+ ssa.OpAMD64VPSRAWMasked256,
+ ssa.OpAMD64VPSRAWMasked512,
+ ssa.OpAMD64VPSRADMasked128,
+ ssa.OpAMD64VPSRADMasked256,
+ ssa.OpAMD64VPSRADMasked512,
+ ssa.OpAMD64VPSRAQMasked128,
+ ssa.OpAMD64VPSRAQMasked256,
+ ssa.OpAMD64VPSRAQMasked512,
+ ssa.OpAMD64VPSRLWMasked128,
+ ssa.OpAMD64VPSRLWMasked256,
+ ssa.OpAMD64VPSRLWMasked512,
+ ssa.OpAMD64VPSRLDMasked128,
+ ssa.OpAMD64VPSRLDMasked256,
+ ssa.OpAMD64VPSRLDMasked512,
+ ssa.OpAMD64VPSRLQMasked128,
+ ssa.OpAMD64VPSRLQMasked256,
+ ssa.OpAMD64VPSRLQMasked512:
+ p = simdVfpkv(s, v)
+
+ case ssa.OpAMD64VPINSRD128,
+ ssa.OpAMD64VPINSRQ128,
+ ssa.OpAMD64VPINSRB128,
+ ssa.OpAMD64VPINSRW128:
+ p = simdVgpvImm(s, v)
+
+ case ssa.OpAMD64VPEXTRD128,
+ ssa.OpAMD64VPEXTRQ128,
+ ssa.OpAMD64VPEXTRB128,
+ ssa.OpAMD64VPEXTRW128:
+ p = simdVgpImm(s, v)
+
+ case ssa.OpAMD64VPALIGNRMasked256,
+ ssa.OpAMD64VPALIGNRMasked512,
+ ssa.OpAMD64VPALIGNRMasked128,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked128,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked256,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked512,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked128,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked256,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked512,
+ ssa.OpAMD64VPSHLDWMasked128,
+ ssa.OpAMD64VPSHLDWMasked256,
+ ssa.OpAMD64VPSHLDWMasked512,
+ ssa.OpAMD64VPSHLDDMasked128,
+ ssa.OpAMD64VPSHLDDMasked256,
+ ssa.OpAMD64VPSHLDDMasked512,
+ ssa.OpAMD64VPSHLDQMasked128,
+ ssa.OpAMD64VPSHLDQMasked256,
+ ssa.OpAMD64VPSHLDQMasked512,
+ ssa.OpAMD64VPSHRDWMasked128,
+ ssa.OpAMD64VPSHRDWMasked256,
+ ssa.OpAMD64VPSHRDWMasked512,
+ ssa.OpAMD64VPSHRDDMasked128,
+ ssa.OpAMD64VPSHRDDMasked256,
+ ssa.OpAMD64VPSHRDDMasked512,
+ ssa.OpAMD64VPSHRDQMasked128,
+ ssa.OpAMD64VPSHRDQMasked256,
+ ssa.OpAMD64VPSHRDQMasked512:
+ p = simdV2kvImm8(s, v)
+
+ case ssa.OpAMD64VPABSBMasked128load,
+ ssa.OpAMD64VPABSBMasked256load,
+ ssa.OpAMD64VPABSWMasked128load,
+ ssa.OpAMD64VPABSWMasked256load,
+ ssa.OpAMD64VPABSDMasked128load,
+ ssa.OpAMD64VPABSDMasked256load,
+ ssa.OpAMD64VPABSDMasked512load,
+ ssa.OpAMD64VPABSQMasked128load,
+ ssa.OpAMD64VPABSQMasked256load,
+ ssa.OpAMD64VPABSQMasked512load,
+ ssa.OpAMD64VCVTPD2PSXMasked128load,
+ ssa.OpAMD64VCVTPD2PSYMasked128load,
+ ssa.OpAMD64VCVTPD2PSMasked256load,
+ ssa.OpAMD64VCVTDQ2PSMasked128load,
+ ssa.OpAMD64VCVTDQ2PSMasked256load,
+ ssa.OpAMD64VCVTDQ2PSMasked512load,
+ ssa.OpAMD64VCVTQQ2PSXMasked128load,
+ ssa.OpAMD64VCVTQQ2PSYMasked128load,
+ ssa.OpAMD64VCVTQQ2PSMasked256load,
+ ssa.OpAMD64VCVTUDQ2PSMasked128load,
+ ssa.OpAMD64VCVTUDQ2PSMasked256load,
+ ssa.OpAMD64VCVTUDQ2PSMasked512load,
+ ssa.OpAMD64VCVTUQQ2PSXMasked128load,
+ ssa.OpAMD64VCVTUQQ2PSYMasked128load,
+ ssa.OpAMD64VCVTUQQ2PSMasked256load,
+ ssa.OpAMD64VCVTPS2PDMasked256load,
+ ssa.OpAMD64VCVTPS2PDMasked512load,
+ ssa.OpAMD64VCVTDQ2PDMasked256load,
+ ssa.OpAMD64VCVTDQ2PDMasked512load,
+ ssa.OpAMD64VCVTQQ2PDMasked128load,
+ ssa.OpAMD64VCVTQQ2PDMasked256load,
+ ssa.OpAMD64VCVTQQ2PDMasked512load,
+ ssa.OpAMD64VCVTUDQ2PDMasked256load,
+ ssa.OpAMD64VCVTUDQ2PDMasked512load,
+ ssa.OpAMD64VCVTUQQ2PDMasked128load,
+ ssa.OpAMD64VCVTUQQ2PDMasked256load,
+ ssa.OpAMD64VCVTUQQ2PDMasked512load,
+ ssa.OpAMD64VCVTTPS2DQMasked128load,
+ ssa.OpAMD64VCVTTPS2DQMasked256load,
+ ssa.OpAMD64VCVTTPS2DQMasked512load,
+ ssa.OpAMD64VCVTTPD2DQXMasked128load,
+ ssa.OpAMD64VCVTTPD2DQYMasked128load,
+ ssa.OpAMD64VCVTTPD2DQMasked256load,
+ ssa.OpAMD64VCVTTPS2QQMasked256load,
+ ssa.OpAMD64VCVTTPS2QQMasked512load,
+ ssa.OpAMD64VCVTTPD2QQMasked128load,
+ ssa.OpAMD64VCVTTPD2QQMasked256load,
+ ssa.OpAMD64VCVTTPD2QQMasked512load,
+ ssa.OpAMD64VCVTTPS2UDQMasked128load,
+ ssa.OpAMD64VCVTTPS2UDQMasked256load,
+ ssa.OpAMD64VCVTTPS2UDQMasked512load,
+ ssa.OpAMD64VCVTTPD2UDQXMasked128load,
+ ssa.OpAMD64VCVTTPD2UDQYMasked128load,
+ ssa.OpAMD64VCVTTPD2UDQMasked256load,
+ ssa.OpAMD64VCVTTPS2UQQMasked256load,
+ ssa.OpAMD64VCVTTPS2UQQMasked512load,
+ ssa.OpAMD64VCVTTPD2UQQMasked128load,
+ ssa.OpAMD64VCVTTPD2UQQMasked256load,
+ ssa.OpAMD64VCVTTPD2UQQMasked512load,
+ ssa.OpAMD64VEXPANDPSMasked128load,
+ ssa.OpAMD64VEXPANDPSMasked256load,
+ ssa.OpAMD64VEXPANDPDMasked128load,
+ ssa.OpAMD64VEXPANDPDMasked256load,
+ ssa.OpAMD64VPEXPANDBMasked128load,
+ ssa.OpAMD64VPEXPANDBMasked256load,
+ ssa.OpAMD64VPEXPANDWMasked128load,
+ ssa.OpAMD64VPEXPANDWMasked256load,
+ ssa.OpAMD64VPEXPANDDMasked128load,
+ ssa.OpAMD64VPEXPANDDMasked256load,
+ ssa.OpAMD64VPEXPANDQMasked128load,
+ ssa.OpAMD64VPEXPANDQMasked256load,
+ ssa.OpAMD64VPMOVSXWQMasked128load,
+ ssa.OpAMD64VPMOVSXDQMasked128load,
+ ssa.OpAMD64VPMOVZXWQMasked128load,
+ ssa.OpAMD64VPMOVZXDQMasked128load,
+ ssa.OpAMD64VPMOVSXBDMasked128load,
+ ssa.OpAMD64VPMOVSXWDMasked128load,
+ ssa.OpAMD64VPMOVSXBQMasked256load,
+ ssa.OpAMD64VPMOVSXWQMasked256load,
+ ssa.OpAMD64VPMOVZXBDMasked128load,
+ ssa.OpAMD64VPMOVZXWDMasked128load,
+ ssa.OpAMD64VPMOVZXBQMasked256load,
+ ssa.OpAMD64VPMOVZXWQMasked256load,
+ ssa.OpAMD64VPMOVSXBWMasked128load,
+ ssa.OpAMD64VPMOVSXBDMasked256load,
+ ssa.OpAMD64VPMOVSXBQMasked512load,
+ ssa.OpAMD64VPMOVZXBWMasked128load,
+ ssa.OpAMD64VPMOVZXBDMasked256load,
+ ssa.OpAMD64VPMOVZXBQMasked512load,
+ ssa.OpAMD64VPMOVSXBWMasked256load,
+ ssa.OpAMD64VPMOVSXBWMasked512load,
+ ssa.OpAMD64VPMOVSXBDMasked512load,
+ ssa.OpAMD64VPMOVSXWDMasked256load,
+ ssa.OpAMD64VPMOVSXWDMasked512load,
+ ssa.OpAMD64VPMOVSXWQMasked512load,
+ ssa.OpAMD64VPMOVSXDQMasked256load,
+ ssa.OpAMD64VPMOVSXDQMasked512load,
+ ssa.OpAMD64VPMOVZXBWMasked256load,
+ ssa.OpAMD64VPMOVZXBWMasked512load,
+ ssa.OpAMD64VPMOVZXBDMasked512load,
+ ssa.OpAMD64VPMOVZXWDMasked256load,
+ ssa.OpAMD64VPMOVZXWDMasked512load,
+ ssa.OpAMD64VPMOVZXWQMasked512load,
+ ssa.OpAMD64VPMOVZXDQMasked256load,
+ ssa.OpAMD64VPMOVZXDQMasked512load,
+ ssa.OpAMD64VPLZCNTDMasked128load,
+ ssa.OpAMD64VPLZCNTDMasked256load,
+ ssa.OpAMD64VPLZCNTDMasked512load,
+ ssa.OpAMD64VPLZCNTQMasked128load,
+ ssa.OpAMD64VPLZCNTQMasked256load,
+ ssa.OpAMD64VPLZCNTQMasked512load,
+ ssa.OpAMD64VPOPCNTBMasked128load,
+ ssa.OpAMD64VPOPCNTBMasked256load,
+ ssa.OpAMD64VPOPCNTWMasked128load,
+ ssa.OpAMD64VPOPCNTWMasked256load,
+ ssa.OpAMD64VPOPCNTDMasked128load,
+ ssa.OpAMD64VPOPCNTDMasked256load,
+ ssa.OpAMD64VPOPCNTDMasked512load,
+ ssa.OpAMD64VPOPCNTQMasked128load,
+ ssa.OpAMD64VPOPCNTQMasked256load,
+ ssa.OpAMD64VPOPCNTQMasked512load,
+ ssa.OpAMD64VRCP14PSMasked128load,
+ ssa.OpAMD64VRCP14PSMasked256load,
+ ssa.OpAMD64VRCP14PSMasked512load,
+ ssa.OpAMD64VRCP14PDMasked128load,
+ ssa.OpAMD64VRCP14PDMasked256load,
+ ssa.OpAMD64VRCP14PDMasked512load,
+ ssa.OpAMD64VRSQRT14PSMasked128load,
+ ssa.OpAMD64VRSQRT14PSMasked256load,
+ ssa.OpAMD64VRSQRT14PSMasked512load,
+ ssa.OpAMD64VRSQRT14PDMasked128load,
+ ssa.OpAMD64VRSQRT14PDMasked256load,
+ ssa.OpAMD64VRSQRT14PDMasked512load,
+ ssa.OpAMD64VSQRTPSMasked128load,
+ ssa.OpAMD64VSQRTPSMasked256load,
+ ssa.OpAMD64VSQRTPSMasked512load,
+ ssa.OpAMD64VSQRTPDMasked128load,
+ ssa.OpAMD64VSQRTPDMasked256load,
+ ssa.OpAMD64VSQRTPDMasked512load,
+ ssa.OpAMD64VPBROADCASTQMasked128load,
+ ssa.OpAMD64VBROADCASTSSMasked128load,
+ ssa.OpAMD64VBROADCASTSDMasked256load,
+ ssa.OpAMD64VPBROADCASTDMasked128load,
+ ssa.OpAMD64VPBROADCASTQMasked256load,
+ ssa.OpAMD64VBROADCASTSSMasked256load,
+ ssa.OpAMD64VBROADCASTSDMasked512load,
+ ssa.OpAMD64VPBROADCASTDMasked256load,
+ ssa.OpAMD64VPBROADCASTQMasked512load,
+ ssa.OpAMD64VBROADCASTSSMasked512load,
+ ssa.OpAMD64VPBROADCASTBMasked128load,
+ ssa.OpAMD64VPBROADCASTDMasked512load,
+ ssa.OpAMD64VPBROADCASTBMasked256load,
+ ssa.OpAMD64VPBROADCASTBMasked512load:
+ p = simdVkvload(s, v)
+
+ case ssa.OpAMD64VAESDECLAST128load,
+ ssa.OpAMD64VAESDECLAST256load,
+ ssa.OpAMD64VAESDEC128load,
+ ssa.OpAMD64VAESDEC256load,
+ ssa.OpAMD64VAESENCLAST128load,
+ ssa.OpAMD64VAESENCLAST256load,
+ ssa.OpAMD64VAESENC128load,
+ ssa.OpAMD64VAESENC256load,
+ ssa.OpAMD64VADDPS128load,
+ ssa.OpAMD64VADDPS256load,
+ ssa.OpAMD64VADDPS512load,
+ ssa.OpAMD64VADDPD128load,
+ ssa.OpAMD64VADDPD256load,
+ ssa.OpAMD64VADDPD512load,
+ ssa.OpAMD64VPADDB128load,
+ ssa.OpAMD64VPADDB256load,
+ ssa.OpAMD64VPADDW128load,
+ ssa.OpAMD64VPADDW256load,
+ ssa.OpAMD64VPADDD128load,
+ ssa.OpAMD64VPADDD256load,
+ ssa.OpAMD64VPADDD512load,
+ ssa.OpAMD64VPADDQ128load,
+ ssa.OpAMD64VPADDQ256load,
+ ssa.OpAMD64VPADDQ512load,
+ ssa.OpAMD64VADDSUBPS128load,
+ ssa.OpAMD64VADDSUBPS256load,
+ ssa.OpAMD64VADDSUBPD128load,
+ ssa.OpAMD64VADDSUBPD256load,
+ ssa.OpAMD64VPADDSB128load,
+ ssa.OpAMD64VPADDSB256load,
+ ssa.OpAMD64VPADDSW128load,
+ ssa.OpAMD64VPADDSW256load,
+ ssa.OpAMD64VPADDUSB128load,
+ ssa.OpAMD64VPADDUSB256load,
+ ssa.OpAMD64VPADDUSW128load,
+ ssa.OpAMD64VPADDUSW256load,
+ ssa.OpAMD64VPAND128load,
+ ssa.OpAMD64VPAND256load,
+ ssa.OpAMD64VPANDD512load,
+ ssa.OpAMD64VPANDQ512load,
+ ssa.OpAMD64VPANDN128load,
+ ssa.OpAMD64VPANDN256load,
+ ssa.OpAMD64VPANDND512load,
+ ssa.OpAMD64VPANDNQ512load,
+ ssa.OpAMD64VPAVGB128load,
+ ssa.OpAMD64VPAVGB256load,
+ ssa.OpAMD64VPAVGW128load,
+ ssa.OpAMD64VPAVGW256load,
+ ssa.OpAMD64VHADDPS128load,
+ ssa.OpAMD64VHADDPD128load,
+ ssa.OpAMD64VPHADDW128load,
+ ssa.OpAMD64VPHADDD128load,
+ ssa.OpAMD64VHADDPS256load,
+ ssa.OpAMD64VHADDPD256load,
+ ssa.OpAMD64VPHADDW256load,
+ ssa.OpAMD64VPHADDD256load,
+ ssa.OpAMD64VPHADDSW128load,
+ ssa.OpAMD64VPHADDSW256load,
+ ssa.OpAMD64VHSUBPS128load,
+ ssa.OpAMD64VHSUBPD128load,
+ ssa.OpAMD64VPHSUBW128load,
+ ssa.OpAMD64VPHSUBD128load,
+ ssa.OpAMD64VHSUBPS256load,
+ ssa.OpAMD64VHSUBPD256load,
+ ssa.OpAMD64VPHSUBW256load,
+ ssa.OpAMD64VPHSUBD256load,
+ ssa.OpAMD64VPHSUBSW128load,
+ ssa.OpAMD64VPHSUBSW256load,
+ ssa.OpAMD64VDIVPS128load,
+ ssa.OpAMD64VDIVPS256load,
+ ssa.OpAMD64VDIVPS512load,
+ ssa.OpAMD64VDIVPD128load,
+ ssa.OpAMD64VDIVPD256load,
+ ssa.OpAMD64VDIVPD512load,
+ ssa.OpAMD64VPMADDWD128load,
+ ssa.OpAMD64VPMADDWD256load,
+ ssa.OpAMD64VPMADDUBSW128load,
+ ssa.OpAMD64VPMADDUBSW256load,
+ ssa.OpAMD64VPCMPEQB128load,
+ ssa.OpAMD64VPCMPEQB256load,
+ ssa.OpAMD64VPCMPEQW128load,
+ ssa.OpAMD64VPCMPEQW256load,
+ ssa.OpAMD64VPCMPEQD128load,
+ ssa.OpAMD64VPCMPEQD256load,
+ ssa.OpAMD64VPCMPEQQ128load,
+ ssa.OpAMD64VPCMPEQQ256load,
+ ssa.OpAMD64VGF2P8MULB128load,
+ ssa.OpAMD64VGF2P8MULB256load,
+ ssa.OpAMD64VPCMPGTB128load,
+ ssa.OpAMD64VPCMPGTB256load,
+ ssa.OpAMD64VPCMPGTW128load,
+ ssa.OpAMD64VPCMPGTW256load,
+ ssa.OpAMD64VPCMPGTD128load,
+ ssa.OpAMD64VPCMPGTD256load,
+ ssa.OpAMD64VPCMPGTQ128load,
+ ssa.OpAMD64VPCMPGTQ256load,
+ ssa.OpAMD64VPUNPCKHWD128load,
+ ssa.OpAMD64VPUNPCKHDQ128load,
+ ssa.OpAMD64VPUNPCKHQDQ128load,
+ ssa.OpAMD64VPUNPCKHWD256load,
+ ssa.OpAMD64VPUNPCKHDQ256load,
+ ssa.OpAMD64VPUNPCKHDQ512load,
+ ssa.OpAMD64VPUNPCKHQDQ256load,
+ ssa.OpAMD64VPUNPCKHQDQ512load,
+ ssa.OpAMD64VPUNPCKLWD128load,
+ ssa.OpAMD64VPUNPCKLDQ128load,
+ ssa.OpAMD64VPUNPCKLQDQ128load,
+ ssa.OpAMD64VPUNPCKLWD256load,
+ ssa.OpAMD64VPUNPCKLDQ256load,
+ ssa.OpAMD64VPUNPCKLDQ512load,
+ ssa.OpAMD64VPUNPCKLQDQ256load,
+ ssa.OpAMD64VPUNPCKLQDQ512load,
+ ssa.OpAMD64VMAXPS128load,
+ ssa.OpAMD64VMAXPS256load,
+ ssa.OpAMD64VMAXPS512load,
+ ssa.OpAMD64VMAXPD128load,
+ ssa.OpAMD64VMAXPD256load,
+ ssa.OpAMD64VMAXPD512load,
+ ssa.OpAMD64VPMAXSB128load,
+ ssa.OpAMD64VPMAXSB256load,
+ ssa.OpAMD64VPMAXSW128load,
+ ssa.OpAMD64VPMAXSW256load,
+ ssa.OpAMD64VPMAXSD128load,
+ ssa.OpAMD64VPMAXSD256load,
+ ssa.OpAMD64VPMAXSD512load,
+ ssa.OpAMD64VPMAXSQ128load,
+ ssa.OpAMD64VPMAXSQ256load,
+ ssa.OpAMD64VPMAXSQ512load,
+ ssa.OpAMD64VPMAXUB128load,
+ ssa.OpAMD64VPMAXUB256load,
+ ssa.OpAMD64VPMAXUW128load,
+ ssa.OpAMD64VPMAXUW256load,
+ ssa.OpAMD64VPMAXUD128load,
+ ssa.OpAMD64VPMAXUD256load,
+ ssa.OpAMD64VPMAXUD512load,
+ ssa.OpAMD64VPMAXUQ128load,
+ ssa.OpAMD64VPMAXUQ256load,
+ ssa.OpAMD64VPMAXUQ512load,
+ ssa.OpAMD64VMINPS128load,
+ ssa.OpAMD64VMINPS256load,
+ ssa.OpAMD64VMINPS512load,
+ ssa.OpAMD64VMINPD128load,
+ ssa.OpAMD64VMINPD256load,
+ ssa.OpAMD64VMINPD512load,
+ ssa.OpAMD64VPMINSB128load,
+ ssa.OpAMD64VPMINSB256load,
+ ssa.OpAMD64VPMINSW128load,
+ ssa.OpAMD64VPMINSW256load,
+ ssa.OpAMD64VPMINSD128load,
+ ssa.OpAMD64VPMINSD256load,
+ ssa.OpAMD64VPMINSD512load,
+ ssa.OpAMD64VPMINSQ128load,
+ ssa.OpAMD64VPMINSQ256load,
+ ssa.OpAMD64VPMINSQ512load,
+ ssa.OpAMD64VPMINUB128load,
+ ssa.OpAMD64VPMINUB256load,
+ ssa.OpAMD64VPMINUW128load,
+ ssa.OpAMD64VPMINUW256load,
+ ssa.OpAMD64VPMINUD128load,
+ ssa.OpAMD64VPMINUD256load,
+ ssa.OpAMD64VPMINUD512load,
+ ssa.OpAMD64VPMINUQ128load,
+ ssa.OpAMD64VPMINUQ256load,
+ ssa.OpAMD64VPMINUQ512load,
+ ssa.OpAMD64VMULPS128load,
+ ssa.OpAMD64VMULPS256load,
+ ssa.OpAMD64VMULPS512load,
+ ssa.OpAMD64VMULPD128load,
+ ssa.OpAMD64VMULPD256load,
+ ssa.OpAMD64VMULPD512load,
+ ssa.OpAMD64VPMULLW128load,
+ ssa.OpAMD64VPMULLW256load,
+ ssa.OpAMD64VPMULLD128load,
+ ssa.OpAMD64VPMULLD256load,
+ ssa.OpAMD64VPMULLD512load,
+ ssa.OpAMD64VPMULLQ128load,
+ ssa.OpAMD64VPMULLQ256load,
+ ssa.OpAMD64VPMULLQ512load,
+ ssa.OpAMD64VPMULHW128load,
+ ssa.OpAMD64VPMULHW256load,
+ ssa.OpAMD64VPMULHUW128load,
+ ssa.OpAMD64VPMULHUW256load,
+ ssa.OpAMD64VPSIGNB128load,
+ ssa.OpAMD64VPSIGNB256load,
+ ssa.OpAMD64VPSIGNW128load,
+ ssa.OpAMD64VPSIGNW256load,
+ ssa.OpAMD64VPSIGND128load,
+ ssa.OpAMD64VPSIGND256load,
+ ssa.OpAMD64VPMULDQ128load,
+ ssa.OpAMD64VPMULDQ256load,
+ ssa.OpAMD64VPMULUDQ128load,
+ ssa.OpAMD64VPMULUDQ256load,
+ ssa.OpAMD64VPOR128load,
+ ssa.OpAMD64VPOR256load,
+ ssa.OpAMD64VPORD512load,
+ ssa.OpAMD64VPORQ512load,
+ ssa.OpAMD64VPERMB128load,
+ ssa.OpAMD64VPERMB256load,
+ ssa.OpAMD64VPERMW128load,
+ ssa.OpAMD64VPERMW256load,
+ ssa.OpAMD64VPERMPS256load,
+ ssa.OpAMD64VPERMD256load,
+ ssa.OpAMD64VPERMPS512load,
+ ssa.OpAMD64VPERMD512load,
+ ssa.OpAMD64VPERMPD256load,
+ ssa.OpAMD64VPERMQ256load,
+ ssa.OpAMD64VPERMPD512load,
+ ssa.OpAMD64VPERMQ512load,
+ ssa.OpAMD64VPSHUFB128load,
+ ssa.OpAMD64VPSHUFB256load,
+ ssa.OpAMD64VPROLVD128load,
+ ssa.OpAMD64VPROLVD256load,
+ ssa.OpAMD64VPROLVD512load,
+ ssa.OpAMD64VPROLVQ128load,
+ ssa.OpAMD64VPROLVQ256load,
+ ssa.OpAMD64VPROLVQ512load,
+ ssa.OpAMD64VPRORVD128load,
+ ssa.OpAMD64VPRORVD256load,
+ ssa.OpAMD64VPRORVD512load,
+ ssa.OpAMD64VPRORVQ128load,
+ ssa.OpAMD64VPRORVQ256load,
+ ssa.OpAMD64VPRORVQ512load,
+ ssa.OpAMD64VPACKSSDW128load,
+ ssa.OpAMD64VPACKSSDW256load,
+ ssa.OpAMD64VPACKSSDW512load,
+ ssa.OpAMD64VPACKUSDW128load,
+ ssa.OpAMD64VPACKUSDW256load,
+ ssa.OpAMD64VPACKUSDW512load,
+ ssa.OpAMD64VSCALEFPS128load,
+ ssa.OpAMD64VSCALEFPS256load,
+ ssa.OpAMD64VSCALEFPS512load,
+ ssa.OpAMD64VSCALEFPD128load,
+ ssa.OpAMD64VSCALEFPD256load,
+ ssa.OpAMD64VSCALEFPD512load,
+ ssa.OpAMD64VPSLLVW128load,
+ ssa.OpAMD64VPSLLVW256load,
+ ssa.OpAMD64VPSLLVD128load,
+ ssa.OpAMD64VPSLLVD256load,
+ ssa.OpAMD64VPSLLVD512load,
+ ssa.OpAMD64VPSLLVQ128load,
+ ssa.OpAMD64VPSLLVQ256load,
+ ssa.OpAMD64VPSLLVQ512load,
+ ssa.OpAMD64VPSRAVW128load,
+ ssa.OpAMD64VPSRAVW256load,
+ ssa.OpAMD64VPSRAVD128load,
+ ssa.OpAMD64VPSRAVD256load,
+ ssa.OpAMD64VPSRAVD512load,
+ ssa.OpAMD64VPSRAVQ128load,
+ ssa.OpAMD64VPSRAVQ256load,
+ ssa.OpAMD64VPSRAVQ512load,
+ ssa.OpAMD64VPSRLVW128load,
+ ssa.OpAMD64VPSRLVW256load,
+ ssa.OpAMD64VPSRLVD128load,
+ ssa.OpAMD64VPSRLVD256load,
+ ssa.OpAMD64VPSRLVD512load,
+ ssa.OpAMD64VPSRLVQ128load,
+ ssa.OpAMD64VPSRLVQ256load,
+ ssa.OpAMD64VPSRLVQ512load,
+ ssa.OpAMD64VSUBPS128load,
+ ssa.OpAMD64VSUBPS256load,
+ ssa.OpAMD64VSUBPS512load,
+ ssa.OpAMD64VSUBPD128load,
+ ssa.OpAMD64VSUBPD256load,
+ ssa.OpAMD64VSUBPD512load,
+ ssa.OpAMD64VPSUBB128load,
+ ssa.OpAMD64VPSUBB256load,
+ ssa.OpAMD64VPSUBW128load,
+ ssa.OpAMD64VPSUBW256load,
+ ssa.OpAMD64VPSUBD128load,
+ ssa.OpAMD64VPSUBD256load,
+ ssa.OpAMD64VPSUBD512load,
+ ssa.OpAMD64VPSUBQ128load,
+ ssa.OpAMD64VPSUBQ256load,
+ ssa.OpAMD64VPSUBQ512load,
+ ssa.OpAMD64VPSUBSB128load,
+ ssa.OpAMD64VPSUBSB256load,
+ ssa.OpAMD64VPSUBSW128load,
+ ssa.OpAMD64VPSUBSW256load,
+ ssa.OpAMD64VPSUBUSB128load,
+ ssa.OpAMD64VPSUBUSB256load,
+ ssa.OpAMD64VPSUBUSW128load,
+ ssa.OpAMD64VPSUBUSW256load,
+ ssa.OpAMD64VPSADBW128load,
+ ssa.OpAMD64VPSADBW256load,
+ ssa.OpAMD64VPXOR128load,
+ ssa.OpAMD64VPXOR256load,
+ ssa.OpAMD64VPXORD512load,
+ ssa.OpAMD64VPXORQ512load:
+ p = simdV21load(s, v)
+
+ case ssa.OpAMD64VPDPWSSD128load,
+ ssa.OpAMD64VPDPWSSD256load,
+ ssa.OpAMD64VPDPWSSD512load,
+ ssa.OpAMD64VPERMI2B128load,
+ ssa.OpAMD64VPERMI2B256load,
+ ssa.OpAMD64VPERMI2W128load,
+ ssa.OpAMD64VPERMI2W256load,
+ ssa.OpAMD64VPERMI2PS128load,
+ ssa.OpAMD64VPERMI2D128load,
+ ssa.OpAMD64VPERMI2PS256load,
+ ssa.OpAMD64VPERMI2D256load,
+ ssa.OpAMD64VPERMI2PS512load,
+ ssa.OpAMD64VPERMI2D512load,
+ ssa.OpAMD64VPERMI2PD128load,
+ ssa.OpAMD64VPERMI2Q128load,
+ ssa.OpAMD64VPERMI2PD256load,
+ ssa.OpAMD64VPERMI2Q256load,
+ ssa.OpAMD64VPERMI2PD512load,
+ ssa.OpAMD64VPERMI2Q512load,
+ ssa.OpAMD64VFMADD213PS128load,
+ ssa.OpAMD64VFMADD213PS256load,
+ ssa.OpAMD64VFMADD213PS512load,
+ ssa.OpAMD64VFMADD213PD128load,
+ ssa.OpAMD64VFMADD213PD256load,
+ ssa.OpAMD64VFMADD213PD512load,
+ ssa.OpAMD64VFMADDSUB213PS128load,
+ ssa.OpAMD64VFMADDSUB213PS256load,
+ ssa.OpAMD64VFMADDSUB213PS512load,
+ ssa.OpAMD64VFMADDSUB213PD128load,
+ ssa.OpAMD64VFMADDSUB213PD256load,
+ ssa.OpAMD64VFMADDSUB213PD512load,
+ ssa.OpAMD64VFMSUBADD213PS128load,
+ ssa.OpAMD64VFMSUBADD213PS256load,
+ ssa.OpAMD64VFMSUBADD213PS512load,
+ ssa.OpAMD64VFMSUBADD213PD128load,
+ ssa.OpAMD64VFMSUBADD213PD256load,
+ ssa.OpAMD64VFMSUBADD213PD512load,
+ ssa.OpAMD64VPSHLDVW128load,
+ ssa.OpAMD64VPSHLDVW256load,
+ ssa.OpAMD64VPSHLDVD128load,
+ ssa.OpAMD64VPSHLDVD256load,
+ ssa.OpAMD64VPSHLDVD512load,
+ ssa.OpAMD64VPSHLDVQ128load,
+ ssa.OpAMD64VPSHLDVQ256load,
+ ssa.OpAMD64VPSHLDVQ512load,
+ ssa.OpAMD64VPSHRDVW128load,
+ ssa.OpAMD64VPSHRDVW256load,
+ ssa.OpAMD64VPSHRDVD128load,
+ ssa.OpAMD64VPSHRDVD256load,
+ ssa.OpAMD64VPSHRDVD512load,
+ ssa.OpAMD64VPSHRDVQ128load,
+ ssa.OpAMD64VPSHRDVQ256load,
+ ssa.OpAMD64VPSHRDVQ512load:
+ p = simdV31loadResultInArg0(s, v)
+
+ case ssa.OpAMD64VPDPWSSDMasked128load,
+ ssa.OpAMD64VPDPWSSDMasked256load,
+ ssa.OpAMD64VPDPWSSDMasked512load,
+ ssa.OpAMD64VPERMI2BMasked128load,
+ ssa.OpAMD64VPERMI2BMasked256load,
+ ssa.OpAMD64VPERMI2WMasked128load,
+ ssa.OpAMD64VPERMI2WMasked256load,
+ ssa.OpAMD64VPERMI2PSMasked128load,
+ ssa.OpAMD64VPERMI2DMasked128load,
+ ssa.OpAMD64VPERMI2PSMasked256load,
+ ssa.OpAMD64VPERMI2DMasked256load,
+ ssa.OpAMD64VPERMI2PSMasked512load,
+ ssa.OpAMD64VPERMI2DMasked512load,
+ ssa.OpAMD64VPERMI2PDMasked128load,
+ ssa.OpAMD64VPERMI2QMasked128load,
+ ssa.OpAMD64VPERMI2PDMasked256load,
+ ssa.OpAMD64VPERMI2QMasked256load,
+ ssa.OpAMD64VPERMI2PDMasked512load,
+ ssa.OpAMD64VPERMI2QMasked512load,
+ ssa.OpAMD64VFMADDSUB213PSMasked128load,
+ ssa.OpAMD64VFMADDSUB213PSMasked256load,
+ ssa.OpAMD64VFMADDSUB213PSMasked512load,
+ ssa.OpAMD64VFMADDSUB213PDMasked128load,
+ ssa.OpAMD64VFMADDSUB213PDMasked256load,
+ ssa.OpAMD64VFMADDSUB213PDMasked512load,
+ ssa.OpAMD64VFMADD213PSMasked128load,
+ ssa.OpAMD64VFMADD213PSMasked256load,
+ ssa.OpAMD64VFMADD213PSMasked512load,
+ ssa.OpAMD64VFMADD213PDMasked128load,
+ ssa.OpAMD64VFMADD213PDMasked256load,
+ ssa.OpAMD64VFMADD213PDMasked512load,
+ ssa.OpAMD64VFMSUBADD213PSMasked128load,
+ ssa.OpAMD64VFMSUBADD213PSMasked256load,
+ ssa.OpAMD64VFMSUBADD213PSMasked512load,
+ ssa.OpAMD64VFMSUBADD213PDMasked128load,
+ ssa.OpAMD64VFMSUBADD213PDMasked256load,
+ ssa.OpAMD64VFMSUBADD213PDMasked512load,
+ ssa.OpAMD64VPSHLDVWMasked128load,
+ ssa.OpAMD64VPSHLDVWMasked256load,
+ ssa.OpAMD64VPSHLDVDMasked128load,
+ ssa.OpAMD64VPSHLDVDMasked256load,
+ ssa.OpAMD64VPSHLDVDMasked512load,
+ ssa.OpAMD64VPSHLDVQMasked128load,
+ ssa.OpAMD64VPSHLDVQMasked256load,
+ ssa.OpAMD64VPSHLDVQMasked512load,
+ ssa.OpAMD64VPSHRDVWMasked128load,
+ ssa.OpAMD64VPSHRDVWMasked256load,
+ ssa.OpAMD64VPSHRDVDMasked128load,
+ ssa.OpAMD64VPSHRDVDMasked256load,
+ ssa.OpAMD64VPSHRDVDMasked512load,
+ ssa.OpAMD64VPSHRDVQMasked128load,
+ ssa.OpAMD64VPSHRDVQMasked256load,
+ ssa.OpAMD64VPSHRDVQMasked512load:
+ p = simdV3kvloadResultInArg0(s, v)
+
+ case ssa.OpAMD64VADDPSMasked128load,
+ ssa.OpAMD64VADDPSMasked256load,
+ ssa.OpAMD64VADDPSMasked512load,
+ ssa.OpAMD64VADDPDMasked128load,
+ ssa.OpAMD64VADDPDMasked256load,
+ ssa.OpAMD64VADDPDMasked512load,
+ ssa.OpAMD64VPADDBMasked128load,
+ ssa.OpAMD64VPADDBMasked256load,
+ ssa.OpAMD64VPADDWMasked128load,
+ ssa.OpAMD64VPADDWMasked256load,
+ ssa.OpAMD64VPADDDMasked128load,
+ ssa.OpAMD64VPADDDMasked256load,
+ ssa.OpAMD64VPADDDMasked512load,
+ ssa.OpAMD64VPADDQMasked128load,
+ ssa.OpAMD64VPADDQMasked256load,
+ ssa.OpAMD64VPADDQMasked512load,
+ ssa.OpAMD64VPADDSBMasked128load,
+ ssa.OpAMD64VPADDSBMasked256load,
+ ssa.OpAMD64VPADDSWMasked128load,
+ ssa.OpAMD64VPADDSWMasked256load,
+ ssa.OpAMD64VPADDUSBMasked128load,
+ ssa.OpAMD64VPADDUSBMasked256load,
+ ssa.OpAMD64VPADDUSWMasked128load,
+ ssa.OpAMD64VPADDUSWMasked256load,
+ ssa.OpAMD64VPANDDMasked128load,
+ ssa.OpAMD64VPANDDMasked256load,
+ ssa.OpAMD64VPANDDMasked512load,
+ ssa.OpAMD64VPANDQMasked128load,
+ ssa.OpAMD64VPANDQMasked256load,
+ ssa.OpAMD64VPANDQMasked512load,
+ ssa.OpAMD64VPANDNDMasked128load,
+ ssa.OpAMD64VPANDNDMasked256load,
+ ssa.OpAMD64VPANDNDMasked512load,
+ ssa.OpAMD64VPANDNQMasked128load,
+ ssa.OpAMD64VPANDNQMasked256load,
+ ssa.OpAMD64VPANDNQMasked512load,
+ ssa.OpAMD64VPAVGBMasked128load,
+ ssa.OpAMD64VPAVGBMasked256load,
+ ssa.OpAMD64VPAVGWMasked128load,
+ ssa.OpAMD64VPAVGWMasked256load,
+ ssa.OpAMD64VDIVPSMasked128load,
+ ssa.OpAMD64VDIVPSMasked256load,
+ ssa.OpAMD64VDIVPSMasked512load,
+ ssa.OpAMD64VDIVPDMasked128load,
+ ssa.OpAMD64VDIVPDMasked256load,
+ ssa.OpAMD64VDIVPDMasked512load,
+ ssa.OpAMD64VPMADDWDMasked128load,
+ ssa.OpAMD64VPMADDWDMasked256load,
+ ssa.OpAMD64VPMADDUBSWMasked128load,
+ ssa.OpAMD64VPMADDUBSWMasked256load,
+ ssa.OpAMD64VGF2P8MULBMasked128load,
+ ssa.OpAMD64VGF2P8MULBMasked256load,
+ ssa.OpAMD64VMAXPSMasked128load,
+ ssa.OpAMD64VMAXPSMasked256load,
+ ssa.OpAMD64VMAXPSMasked512load,
+ ssa.OpAMD64VMAXPDMasked128load,
+ ssa.OpAMD64VMAXPDMasked256load,
+ ssa.OpAMD64VMAXPDMasked512load,
+ ssa.OpAMD64VPMAXSBMasked128load,
+ ssa.OpAMD64VPMAXSBMasked256load,
+ ssa.OpAMD64VPMAXSWMasked128load,
+ ssa.OpAMD64VPMAXSWMasked256load,
+ ssa.OpAMD64VPMAXSDMasked128load,
+ ssa.OpAMD64VPMAXSDMasked256load,
+ ssa.OpAMD64VPMAXSDMasked512load,
+ ssa.OpAMD64VPMAXSQMasked128load,
+ ssa.OpAMD64VPMAXSQMasked256load,
+ ssa.OpAMD64VPMAXSQMasked512load,
+ ssa.OpAMD64VPMAXUBMasked128load,
+ ssa.OpAMD64VPMAXUBMasked256load,
+ ssa.OpAMD64VPMAXUWMasked128load,
+ ssa.OpAMD64VPMAXUWMasked256load,
+ ssa.OpAMD64VPMAXUDMasked128load,
+ ssa.OpAMD64VPMAXUDMasked256load,
+ ssa.OpAMD64VPMAXUDMasked512load,
+ ssa.OpAMD64VPMAXUQMasked128load,
+ ssa.OpAMD64VPMAXUQMasked256load,
+ ssa.OpAMD64VPMAXUQMasked512load,
+ ssa.OpAMD64VMINPSMasked128load,
+ ssa.OpAMD64VMINPSMasked256load,
+ ssa.OpAMD64VMINPSMasked512load,
+ ssa.OpAMD64VMINPDMasked128load,
+ ssa.OpAMD64VMINPDMasked256load,
+ ssa.OpAMD64VMINPDMasked512load,
+ ssa.OpAMD64VPMINSBMasked128load,
+ ssa.OpAMD64VPMINSBMasked256load,
+ ssa.OpAMD64VPMINSWMasked128load,
+ ssa.OpAMD64VPMINSWMasked256load,
+ ssa.OpAMD64VPMINSDMasked128load,
+ ssa.OpAMD64VPMINSDMasked256load,
+ ssa.OpAMD64VPMINSDMasked512load,
+ ssa.OpAMD64VPMINSQMasked128load,
+ ssa.OpAMD64VPMINSQMasked256load,
+ ssa.OpAMD64VPMINSQMasked512load,
+ ssa.OpAMD64VPMINUBMasked128load,
+ ssa.OpAMD64VPMINUBMasked256load,
+ ssa.OpAMD64VPMINUWMasked128load,
+ ssa.OpAMD64VPMINUWMasked256load,
+ ssa.OpAMD64VPMINUDMasked128load,
+ ssa.OpAMD64VPMINUDMasked256load,
+ ssa.OpAMD64VPMINUDMasked512load,
+ ssa.OpAMD64VPMINUQMasked128load,
+ ssa.OpAMD64VPMINUQMasked256load,
+ ssa.OpAMD64VPMINUQMasked512load,
+ ssa.OpAMD64VPMULHWMasked128load,
+ ssa.OpAMD64VPMULHWMasked256load,
+ ssa.OpAMD64VPMULHUWMasked128load,
+ ssa.OpAMD64VPMULHUWMasked256load,
+ ssa.OpAMD64VMULPSMasked128load,
+ ssa.OpAMD64VMULPSMasked256load,
+ ssa.OpAMD64VMULPSMasked512load,
+ ssa.OpAMD64VMULPDMasked128load,
+ ssa.OpAMD64VMULPDMasked256load,
+ ssa.OpAMD64VMULPDMasked512load,
+ ssa.OpAMD64VPMULLWMasked128load,
+ ssa.OpAMD64VPMULLWMasked256load,
+ ssa.OpAMD64VPMULLDMasked128load,
+ ssa.OpAMD64VPMULLDMasked256load,
+ ssa.OpAMD64VPMULLDMasked512load,
+ ssa.OpAMD64VPMULLQMasked128load,
+ ssa.OpAMD64VPMULLQMasked256load,
+ ssa.OpAMD64VPMULLQMasked512load,
+ ssa.OpAMD64VPORDMasked128load,
+ ssa.OpAMD64VPORDMasked256load,
+ ssa.OpAMD64VPORDMasked512load,
+ ssa.OpAMD64VPORQMasked128load,
+ ssa.OpAMD64VPORQMasked256load,
+ ssa.OpAMD64VPORQMasked512load,
+ ssa.OpAMD64VPERMBMasked128load,
+ ssa.OpAMD64VPERMBMasked256load,
+ ssa.OpAMD64VPERMWMasked128load,
+ ssa.OpAMD64VPERMWMasked256load,
+ ssa.OpAMD64VPERMPSMasked256load,
+ ssa.OpAMD64VPERMDMasked256load,
+ ssa.OpAMD64VPERMPSMasked512load,
+ ssa.OpAMD64VPERMDMasked512load,
+ ssa.OpAMD64VPERMPDMasked256load,
+ ssa.OpAMD64VPERMQMasked256load,
+ ssa.OpAMD64VPERMPDMasked512load,
+ ssa.OpAMD64VPERMQMasked512load,
+ ssa.OpAMD64VPSHUFBMasked256load,
+ ssa.OpAMD64VPSHUFBMasked128load,
+ ssa.OpAMD64VPROLVDMasked128load,
+ ssa.OpAMD64VPROLVDMasked256load,
+ ssa.OpAMD64VPROLVDMasked512load,
+ ssa.OpAMD64VPROLVQMasked128load,
+ ssa.OpAMD64VPROLVQMasked256load,
+ ssa.OpAMD64VPROLVQMasked512load,
+ ssa.OpAMD64VPRORVDMasked128load,
+ ssa.OpAMD64VPRORVDMasked256load,
+ ssa.OpAMD64VPRORVDMasked512load,
+ ssa.OpAMD64VPRORVQMasked128load,
+ ssa.OpAMD64VPRORVQMasked256load,
+ ssa.OpAMD64VPRORVQMasked512load,
+ ssa.OpAMD64VPACKSSDWMasked256load,
+ ssa.OpAMD64VPACKSSDWMasked512load,
+ ssa.OpAMD64VPACKSSDWMasked128load,
+ ssa.OpAMD64VPACKUSDWMasked256load,
+ ssa.OpAMD64VPACKUSDWMasked512load,
+ ssa.OpAMD64VPACKUSDWMasked128load,
+ ssa.OpAMD64VSCALEFPSMasked128load,
+ ssa.OpAMD64VSCALEFPSMasked256load,
+ ssa.OpAMD64VSCALEFPSMasked512load,
+ ssa.OpAMD64VSCALEFPDMasked128load,
+ ssa.OpAMD64VSCALEFPDMasked256load,
+ ssa.OpAMD64VSCALEFPDMasked512load,
+ ssa.OpAMD64VPSLLVWMasked128load,
+ ssa.OpAMD64VPSLLVWMasked256load,
+ ssa.OpAMD64VPSLLVDMasked128load,
+ ssa.OpAMD64VPSLLVDMasked256load,
+ ssa.OpAMD64VPSLLVDMasked512load,
+ ssa.OpAMD64VPSLLVQMasked128load,
+ ssa.OpAMD64VPSLLVQMasked256load,
+ ssa.OpAMD64VPSLLVQMasked512load,
+ ssa.OpAMD64VPSRAVWMasked128load,
+ ssa.OpAMD64VPSRAVWMasked256load,
+ ssa.OpAMD64VPSRAVDMasked128load,
+ ssa.OpAMD64VPSRAVDMasked256load,
+ ssa.OpAMD64VPSRAVDMasked512load,
+ ssa.OpAMD64VPSRAVQMasked128load,
+ ssa.OpAMD64VPSRAVQMasked256load,
+ ssa.OpAMD64VPSRAVQMasked512load,
+ ssa.OpAMD64VPSRLVWMasked128load,
+ ssa.OpAMD64VPSRLVWMasked256load,
+ ssa.OpAMD64VPSRLVDMasked128load,
+ ssa.OpAMD64VPSRLVDMasked256load,
+ ssa.OpAMD64VPSRLVDMasked512load,
+ ssa.OpAMD64VPSRLVQMasked128load,
+ ssa.OpAMD64VPSRLVQMasked256load,
+ ssa.OpAMD64VPSRLVQMasked512load,
+ ssa.OpAMD64VSUBPSMasked128load,
+ ssa.OpAMD64VSUBPSMasked256load,
+ ssa.OpAMD64VSUBPSMasked512load,
+ ssa.OpAMD64VSUBPDMasked128load,
+ ssa.OpAMD64VSUBPDMasked256load,
+ ssa.OpAMD64VSUBPDMasked512load,
+ ssa.OpAMD64VPSUBBMasked128load,
+ ssa.OpAMD64VPSUBBMasked256load,
+ ssa.OpAMD64VPSUBWMasked128load,
+ ssa.OpAMD64VPSUBWMasked256load,
+ ssa.OpAMD64VPSUBDMasked128load,
+ ssa.OpAMD64VPSUBDMasked256load,
+ ssa.OpAMD64VPSUBDMasked512load,
+ ssa.OpAMD64VPSUBQMasked128load,
+ ssa.OpAMD64VPSUBQMasked256load,
+ ssa.OpAMD64VPSUBQMasked512load,
+ ssa.OpAMD64VPSUBSBMasked128load,
+ ssa.OpAMD64VPSUBSBMasked256load,
+ ssa.OpAMD64VPSUBSWMasked128load,
+ ssa.OpAMD64VPSUBSWMasked256load,
+ ssa.OpAMD64VPSUBUSBMasked128load,
+ ssa.OpAMD64VPSUBUSBMasked256load,
+ ssa.OpAMD64VPSUBUSWMasked128load,
+ ssa.OpAMD64VPSUBUSWMasked256load,
+ ssa.OpAMD64VPXORDMasked128load,
+ ssa.OpAMD64VPXORDMasked256load,
+ ssa.OpAMD64VPXORDMasked512load,
+ ssa.OpAMD64VPXORQMasked128load,
+ ssa.OpAMD64VPXORQMasked256load,
+ ssa.OpAMD64VPXORQMasked512load,
+ ssa.OpAMD64VPBLENDMDMasked512load,
+ ssa.OpAMD64VPBLENDMQMasked512load:
+ p = simdV2kvload(s, v)
+
+ case ssa.OpAMD64VPCMPEQD512load,
+ ssa.OpAMD64VPCMPEQQ512load,
+ ssa.OpAMD64VPCMPGTD512load,
+ ssa.OpAMD64VPCMPGTQ512load:
+ p = simdV2kload(s, v)
+
+ case ssa.OpAMD64VAESIMC128load,
+ ssa.OpAMD64VPABSB128load,
+ ssa.OpAMD64VPABSB256load,
+ ssa.OpAMD64VPABSW128load,
+ ssa.OpAMD64VPABSW256load,
+ ssa.OpAMD64VPABSD128load,
+ ssa.OpAMD64VPABSD256load,
+ ssa.OpAMD64VPABSD512load,
+ ssa.OpAMD64VPABSQ128load,
+ ssa.OpAMD64VPABSQ256load,
+ ssa.OpAMD64VPABSQ512load,
+ ssa.OpAMD64VCVTPD2PSX128load,
+ ssa.OpAMD64VCVTPD2PSY128load,
+ ssa.OpAMD64VCVTPD2PS256load,
+ ssa.OpAMD64VCVTDQ2PS128load,
+ ssa.OpAMD64VCVTDQ2PS256load,
+ ssa.OpAMD64VCVTDQ2PS512load,
+ ssa.OpAMD64VCVTQQ2PSX128load,
+ ssa.OpAMD64VCVTQQ2PSY128load,
+ ssa.OpAMD64VCVTQQ2PS256load,
+ ssa.OpAMD64VCVTUDQ2PS128load,
+ ssa.OpAMD64VCVTUDQ2PS256load,
+ ssa.OpAMD64VCVTUDQ2PS512load,
+ ssa.OpAMD64VCVTUQQ2PSX128load,
+ ssa.OpAMD64VCVTUQQ2PSY128load,
+ ssa.OpAMD64VCVTUQQ2PS256load,
+ ssa.OpAMD64VCVTPS2PD256load,
+ ssa.OpAMD64VCVTPS2PD512load,
+ ssa.OpAMD64VCVTDQ2PD256load,
+ ssa.OpAMD64VCVTDQ2PD512load,
+ ssa.OpAMD64VCVTQQ2PD128load,
+ ssa.OpAMD64VCVTQQ2PD256load,
+ ssa.OpAMD64VCVTQQ2PD512load,
+ ssa.OpAMD64VCVTUDQ2PD256load,
+ ssa.OpAMD64VCVTUDQ2PD512load,
+ ssa.OpAMD64VCVTUQQ2PD128load,
+ ssa.OpAMD64VCVTUQQ2PD256load,
+ ssa.OpAMD64VCVTUQQ2PD512load,
+ ssa.OpAMD64VCVTTPS2DQ128load,
+ ssa.OpAMD64VCVTTPS2DQ256load,
+ ssa.OpAMD64VCVTTPS2DQ512load,
+ ssa.OpAMD64VCVTTPD2DQX128load,
+ ssa.OpAMD64VCVTTPD2DQY128load,
+ ssa.OpAMD64VCVTTPD2DQ256load,
+ ssa.OpAMD64VCVTTPS2QQ256load,
+ ssa.OpAMD64VCVTTPS2QQ512load,
+ ssa.OpAMD64VCVTTPD2QQ128load,
+ ssa.OpAMD64VCVTTPD2QQ256load,
+ ssa.OpAMD64VCVTTPD2QQ512load,
+ ssa.OpAMD64VCVTTPS2UDQ128load,
+ ssa.OpAMD64VCVTTPS2UDQ256load,
+ ssa.OpAMD64VCVTTPS2UDQ512load,
+ ssa.OpAMD64VCVTTPD2UDQX128load,
+ ssa.OpAMD64VCVTTPD2UDQY128load,
+ ssa.OpAMD64VCVTTPD2UDQ256load,
+ ssa.OpAMD64VCVTTPS2UQQ256load,
+ ssa.OpAMD64VCVTTPS2UQQ512load,
+ ssa.OpAMD64VCVTTPD2UQQ128load,
+ ssa.OpAMD64VCVTTPD2UQQ256load,
+ ssa.OpAMD64VCVTTPD2UQQ512load,
+ ssa.OpAMD64VPMOVSXBQ128load,
+ ssa.OpAMD64VPMOVSXWQ128load,
+ ssa.OpAMD64VPMOVSXDQ128load,
+ ssa.OpAMD64VPMOVZXBQ128load,
+ ssa.OpAMD64VPMOVZXWQ128load,
+ ssa.OpAMD64VPMOVZXDQ128load,
+ ssa.OpAMD64VPMOVSXBD128load,
+ ssa.OpAMD64VPMOVSXWD128load,
+ ssa.OpAMD64VPMOVSXBQ256load,
+ ssa.OpAMD64VPMOVSXWQ256load,
+ ssa.OpAMD64VPMOVZXBD128load,
+ ssa.OpAMD64VPMOVZXWD128load,
+ ssa.OpAMD64VPMOVZXBQ256load,
+ ssa.OpAMD64VPMOVZXWQ256load,
+ ssa.OpAMD64VPMOVSXBW128load,
+ ssa.OpAMD64VPMOVSXBD256load,
+ ssa.OpAMD64VPMOVSXBQ512load,
+ ssa.OpAMD64VPMOVZXBW128load,
+ ssa.OpAMD64VPMOVZXBD256load,
+ ssa.OpAMD64VPMOVZXBQ512load,
+ ssa.OpAMD64VPMOVSXBW256load,
+ ssa.OpAMD64VPMOVSXBW512load,
+ ssa.OpAMD64VPMOVSXBD512load,
+ ssa.OpAMD64VPMOVSXWD256load,
+ ssa.OpAMD64VPMOVSXWD512load,
+ ssa.OpAMD64VPMOVSXWQ512load,
+ ssa.OpAMD64VPMOVSXDQ256load,
+ ssa.OpAMD64VPMOVSXDQ512load,
+ ssa.OpAMD64VPMOVZXBW256load,
+ ssa.OpAMD64VPMOVZXBW512load,
+ ssa.OpAMD64VPMOVZXBD512load,
+ ssa.OpAMD64VPMOVZXWD256load,
+ ssa.OpAMD64VPMOVZXWD512load,
+ ssa.OpAMD64VPMOVZXWQ512load,
+ ssa.OpAMD64VPMOVZXDQ256load,
+ ssa.OpAMD64VPMOVZXDQ512load,
+ ssa.OpAMD64VPLZCNTD128load,
+ ssa.OpAMD64VPLZCNTD256load,
+ ssa.OpAMD64VPLZCNTD512load,
+ ssa.OpAMD64VPLZCNTQ128load,
+ ssa.OpAMD64VPLZCNTQ256load,
+ ssa.OpAMD64VPLZCNTQ512load,
+ ssa.OpAMD64VPOPCNTB128load,
+ ssa.OpAMD64VPOPCNTB256load,
+ ssa.OpAMD64VPOPCNTW128load,
+ ssa.OpAMD64VPOPCNTW256load,
+ ssa.OpAMD64VPOPCNTD128load,
+ ssa.OpAMD64VPOPCNTD256load,
+ ssa.OpAMD64VPOPCNTD512load,
+ ssa.OpAMD64VPOPCNTQ128load,
+ ssa.OpAMD64VPOPCNTQ256load,
+ ssa.OpAMD64VPOPCNTQ512load,
+ ssa.OpAMD64VRCPPS128load,
+ ssa.OpAMD64VRCPPS256load,
+ ssa.OpAMD64VRCP14PS512load,
+ ssa.OpAMD64VRCP14PD128load,
+ ssa.OpAMD64VRCP14PD256load,
+ ssa.OpAMD64VRCP14PD512load,
+ ssa.OpAMD64VRSQRTPS128load,
+ ssa.OpAMD64VRSQRTPS256load,
+ ssa.OpAMD64VRSQRT14PS512load,
+ ssa.OpAMD64VRSQRT14PD128load,
+ ssa.OpAMD64VRSQRT14PD256load,
+ ssa.OpAMD64VRSQRT14PD512load,
+ ssa.OpAMD64VSQRTPS128load,
+ ssa.OpAMD64VSQRTPS256load,
+ ssa.OpAMD64VSQRTPS512load,
+ ssa.OpAMD64VSQRTPD128load,
+ ssa.OpAMD64VSQRTPD256load,
+ ssa.OpAMD64VSQRTPD512load,
+ ssa.OpAMD64VPBROADCASTQ128load,
+ ssa.OpAMD64VBROADCASTSS128load,
+ ssa.OpAMD64VBROADCASTSD256load,
+ ssa.OpAMD64VPBROADCASTD128load,
+ ssa.OpAMD64VPBROADCASTQ256load,
+ ssa.OpAMD64VBROADCASTSS256load,
+ ssa.OpAMD64VBROADCASTSD512load,
+ ssa.OpAMD64VPBROADCASTW128load,
+ ssa.OpAMD64VPBROADCASTD256load,
+ ssa.OpAMD64VPBROADCASTQ512load,
+ ssa.OpAMD64VBROADCASTSS512load,
+ ssa.OpAMD64VPBROADCASTB128load,
+ ssa.OpAMD64VPBROADCASTW256load,
+ ssa.OpAMD64VPBROADCASTD512load,
+ ssa.OpAMD64VPBROADCASTB256load,
+ ssa.OpAMD64VPBROADCASTB512load:
+ p = simdV11load(s, v)
+
+ case ssa.OpAMD64VAESKEYGENASSIST128load,
+ ssa.OpAMD64VROUNDPS128load,
+ ssa.OpAMD64VROUNDPS256load,
+ ssa.OpAMD64VROUNDPD128load,
+ ssa.OpAMD64VROUNDPD256load,
+ ssa.OpAMD64VRNDSCALEPS128load,
+ ssa.OpAMD64VRNDSCALEPS256load,
+ ssa.OpAMD64VRNDSCALEPS512load,
+ ssa.OpAMD64VRNDSCALEPD128load,
+ ssa.OpAMD64VRNDSCALEPD256load,
+ ssa.OpAMD64VRNDSCALEPD512load,
+ ssa.OpAMD64VREDUCEPS128load,
+ ssa.OpAMD64VREDUCEPS256load,
+ ssa.OpAMD64VREDUCEPS512load,
+ ssa.OpAMD64VREDUCEPD128load,
+ ssa.OpAMD64VREDUCEPD256load,
+ ssa.OpAMD64VREDUCEPD512load,
+ ssa.OpAMD64VPSHUFD128load,
+ ssa.OpAMD64VPSHUFD256load,
+ ssa.OpAMD64VPSHUFD512load,
+ ssa.OpAMD64VPSHUFHW128load,
+ ssa.OpAMD64VPSHUFHW256load,
+ ssa.OpAMD64VPSHUFLW128load,
+ ssa.OpAMD64VPSHUFLW256load,
+ ssa.OpAMD64VPROLD128load,
+ ssa.OpAMD64VPROLD256load,
+ ssa.OpAMD64VPROLD512load,
+ ssa.OpAMD64VPROLQ128load,
+ ssa.OpAMD64VPROLQ256load,
+ ssa.OpAMD64VPROLQ512load,
+ ssa.OpAMD64VPRORD128load,
+ ssa.OpAMD64VPRORD256load,
+ ssa.OpAMD64VPRORD512load,
+ ssa.OpAMD64VPRORQ128load,
+ ssa.OpAMD64VPRORQ256load,
+ ssa.OpAMD64VPRORQ512load,
+ ssa.OpAMD64VPSLLD512constload,
+ ssa.OpAMD64VPSLLQ512constload,
+ ssa.OpAMD64VPSRAD512constload,
+ ssa.OpAMD64VPSRAQ128constload,
+ ssa.OpAMD64VPSRAQ256constload,
+ ssa.OpAMD64VPSRAQ512constload,
+ ssa.OpAMD64VPSRLD512constload,
+ ssa.OpAMD64VPSRLQ512constload:
+ p = simdV11loadImm8(s, v)
+
+ case ssa.OpAMD64VRNDSCALEPSMasked128load,
+ ssa.OpAMD64VRNDSCALEPSMasked256load,
+ ssa.OpAMD64VRNDSCALEPSMasked512load,
+ ssa.OpAMD64VRNDSCALEPDMasked128load,
+ ssa.OpAMD64VRNDSCALEPDMasked256load,
+ ssa.OpAMD64VRNDSCALEPDMasked512load,
+ ssa.OpAMD64VREDUCEPSMasked128load,
+ ssa.OpAMD64VREDUCEPSMasked256load,
+ ssa.OpAMD64VREDUCEPSMasked512load,
+ ssa.OpAMD64VREDUCEPDMasked128load,
+ ssa.OpAMD64VREDUCEPDMasked256load,
+ ssa.OpAMD64VREDUCEPDMasked512load,
+ ssa.OpAMD64VPSHUFDMasked256load,
+ ssa.OpAMD64VPSHUFDMasked512load,
+ ssa.OpAMD64VPSHUFHWMasked256load,
+ ssa.OpAMD64VPSHUFHWMasked128load,
+ ssa.OpAMD64VPSHUFLWMasked256load,
+ ssa.OpAMD64VPSHUFLWMasked128load,
+ ssa.OpAMD64VPSHUFDMasked128load,
+ ssa.OpAMD64VPROLDMasked128load,
+ ssa.OpAMD64VPROLDMasked256load,
+ ssa.OpAMD64VPROLDMasked512load,
+ ssa.OpAMD64VPROLQMasked128load,
+ ssa.OpAMD64VPROLQMasked256load,
+ ssa.OpAMD64VPROLQMasked512load,
+ ssa.OpAMD64VPRORDMasked128load,
+ ssa.OpAMD64VPRORDMasked256load,
+ ssa.OpAMD64VPRORDMasked512load,
+ ssa.OpAMD64VPRORQMasked128load,
+ ssa.OpAMD64VPRORQMasked256load,
+ ssa.OpAMD64VPRORQMasked512load,
+ ssa.OpAMD64VPSLLWMasked128constload,
+ ssa.OpAMD64VPSLLWMasked256constload,
+ ssa.OpAMD64VPSLLDMasked128constload,
+ ssa.OpAMD64VPSLLDMasked256constload,
+ ssa.OpAMD64VPSLLDMasked512constload,
+ ssa.OpAMD64VPSLLQMasked128constload,
+ ssa.OpAMD64VPSLLQMasked256constload,
+ ssa.OpAMD64VPSLLQMasked512constload,
+ ssa.OpAMD64VPSRAWMasked128constload,
+ ssa.OpAMD64VPSRAWMasked256constload,
+ ssa.OpAMD64VPSRADMasked128constload,
+ ssa.OpAMD64VPSRADMasked256constload,
+ ssa.OpAMD64VPSRADMasked512constload,
+ ssa.OpAMD64VPSRAQMasked128constload,
+ ssa.OpAMD64VPSRAQMasked256constload,
+ ssa.OpAMD64VPSRAQMasked512constload,
+ ssa.OpAMD64VPSRLWMasked128constload,
+ ssa.OpAMD64VPSRLWMasked256constload,
+ ssa.OpAMD64VPSRLDMasked128constload,
+ ssa.OpAMD64VPSRLDMasked256constload,
+ ssa.OpAMD64VPSRLDMasked512constload,
+ ssa.OpAMD64VPSRLQMasked128constload,
+ ssa.OpAMD64VPSRLQMasked256constload,
+ ssa.OpAMD64VPSRLQMasked512constload:
+ p = simdVkvloadImm8(s, v)
+
+ case ssa.OpAMD64VPERM2F128256load,
+ ssa.OpAMD64VPERM2I128256load,
+ ssa.OpAMD64VPALIGNR128load,
+ ssa.OpAMD64VPALIGNR256load,
+ ssa.OpAMD64VCMPPS128load,
+ ssa.OpAMD64VCMPPS256load,
+ ssa.OpAMD64VCMPPD128load,
+ ssa.OpAMD64VCMPPD256load,
+ ssa.OpAMD64VGF2P8AFFINEQB128load,
+ ssa.OpAMD64VGF2P8AFFINEQB256load,
+ ssa.OpAMD64VGF2P8AFFINEQB512load,
+ ssa.OpAMD64VGF2P8AFFINEINVQB128load,
+ ssa.OpAMD64VGF2P8AFFINEINVQB256load,
+ ssa.OpAMD64VGF2P8AFFINEINVQB512load,
+ ssa.OpAMD64VINSERTF128256load,
+ ssa.OpAMD64VINSERTF64X4512load,
+ ssa.OpAMD64VINSERTI128256load,
+ ssa.OpAMD64VINSERTI64X4512load,
+ ssa.OpAMD64VPSHLDW128load,
+ ssa.OpAMD64VPSHLDW256load,
+ ssa.OpAMD64VPSHLDD128load,
+ ssa.OpAMD64VPSHLDD256load,
+ ssa.OpAMD64VPSHLDD512load,
+ ssa.OpAMD64VPSHLDQ128load,
+ ssa.OpAMD64VPSHLDQ256load,
+ ssa.OpAMD64VPSHLDQ512load,
+ ssa.OpAMD64VPSHRDW128load,
+ ssa.OpAMD64VPSHRDW256load,
+ ssa.OpAMD64VPSHRDD128load,
+ ssa.OpAMD64VPSHRDD256load,
+ ssa.OpAMD64VPSHRDD512load,
+ ssa.OpAMD64VPSHRDQ128load,
+ ssa.OpAMD64VPSHRDQ256load,
+ ssa.OpAMD64VPSHRDQ512load,
+ ssa.OpAMD64VPCLMULQDQ128load,
+ ssa.OpAMD64VPCLMULQDQ256load,
+ ssa.OpAMD64VSHUFPS128load,
+ ssa.OpAMD64VSHUFPD128load,
+ ssa.OpAMD64VSHUFPS256load,
+ ssa.OpAMD64VSHUFPS512load,
+ ssa.OpAMD64VSHUFPD256load,
+ ssa.OpAMD64VSHUFPD512load:
+ p = simdV21loadImm8(s, v)
+
+ case ssa.OpAMD64VCMPPS512load,
+ ssa.OpAMD64VCMPPD512load,
+ ssa.OpAMD64VPCMPUD512load,
+ ssa.OpAMD64VPCMPUQ512load,
+ ssa.OpAMD64VPCMPD512load,
+ ssa.OpAMD64VPCMPQ512load:
+ p = simdV2kloadImm8(s, v)
+
+ case ssa.OpAMD64VCMPPSMasked128load,
+ ssa.OpAMD64VCMPPSMasked256load,
+ ssa.OpAMD64VCMPPSMasked512load,
+ ssa.OpAMD64VCMPPDMasked128load,
+ ssa.OpAMD64VCMPPDMasked256load,
+ ssa.OpAMD64VCMPPDMasked512load,
+ ssa.OpAMD64VPCMPBMasked128load,
+ ssa.OpAMD64VPCMPBMasked256load,
+ ssa.OpAMD64VPCMPWMasked128load,
+ ssa.OpAMD64VPCMPWMasked256load,
+ ssa.OpAMD64VPCMPDMasked128load,
+ ssa.OpAMD64VPCMPDMasked256load,
+ ssa.OpAMD64VPCMPDMasked512load,
+ ssa.OpAMD64VPCMPQMasked128load,
+ ssa.OpAMD64VPCMPQMasked256load,
+ ssa.OpAMD64VPCMPQMasked512load,
+ ssa.OpAMD64VPCMPUBMasked128load,
+ ssa.OpAMD64VPCMPUBMasked256load,
+ ssa.OpAMD64VPCMPUWMasked128load,
+ ssa.OpAMD64VPCMPUWMasked256load,
+ ssa.OpAMD64VPCMPUDMasked128load,
+ ssa.OpAMD64VPCMPUDMasked256load,
+ ssa.OpAMD64VPCMPUDMasked512load,
+ ssa.OpAMD64VPCMPUQMasked128load,
+ ssa.OpAMD64VPCMPUQMasked256load,
+ ssa.OpAMD64VPCMPUQMasked512load:
+ p = simdV2kkloadImm8(s, v)
+
+ case ssa.OpAMD64VPALIGNRMasked256load,
+ ssa.OpAMD64VPALIGNRMasked128load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked128load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked256load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked512load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked128load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked256load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked512load,
+ ssa.OpAMD64VPSHLDWMasked128load,
+ ssa.OpAMD64VPSHLDWMasked256load,
+ ssa.OpAMD64VPSHLDDMasked128load,
+ ssa.OpAMD64VPSHLDDMasked256load,
+ ssa.OpAMD64VPSHLDDMasked512load,
+ ssa.OpAMD64VPSHLDQMasked128load,
+ ssa.OpAMD64VPSHLDQMasked256load,
+ ssa.OpAMD64VPSHLDQMasked512load,
+ ssa.OpAMD64VPSHRDWMasked128load,
+ ssa.OpAMD64VPSHRDWMasked256load,
+ ssa.OpAMD64VPSHRDDMasked128load,
+ ssa.OpAMD64VPSHRDDMasked256load,
+ ssa.OpAMD64VPSHRDDMasked512load,
+ ssa.OpAMD64VPSHRDQMasked128load,
+ ssa.OpAMD64VPSHRDQMasked256load,
+ ssa.OpAMD64VPSHRDQMasked512load:
+ p = simdV2kvloadImm8(s, v)
+
+ case ssa.OpAMD64VPTERNLOGD128,
+ ssa.OpAMD64VPTERNLOGD256,
+ ssa.OpAMD64VPTERNLOGD512,
+ ssa.OpAMD64VPTERNLOGQ128,
+ ssa.OpAMD64VPTERNLOGQ256,
+ ssa.OpAMD64VPTERNLOGQ512:
+ p = simdV31ResultInArg0Imm8(s, v)
+
+ case ssa.OpAMD64VPTERNLOGD128load,
+ ssa.OpAMD64VPTERNLOGD256load,
+ ssa.OpAMD64VPTERNLOGD512load,
+ ssa.OpAMD64VPTERNLOGQ128load,
+ ssa.OpAMD64VPTERNLOGQ256load,
+ ssa.OpAMD64VPTERNLOGQ512load:
+ p = simdV31loadResultInArg0Imm8(s, v)
+
+ case ssa.OpAMD64SHA1MSG1128,
+ ssa.OpAMD64SHA1MSG2128,
+ ssa.OpAMD64SHA1NEXTE128,
+ ssa.OpAMD64SHA256MSG1128,
+ ssa.OpAMD64SHA256MSG2128:
+ p = simdV21ResultInArg0(s, v)
+
+ case ssa.OpAMD64SHA1RNDS4128:
+ p = simdV21ResultInArg0Imm8(s, v)
+
+ case ssa.OpAMD64SHA256RNDS2128:
+ p = simdV31x0AtIn2ResultInArg0(s, v)
+
+ case ssa.OpAMD64VPABSBMasked128Merging,
+ ssa.OpAMD64VPABSBMasked256Merging,
+ ssa.OpAMD64VPABSBMasked512Merging,
+ ssa.OpAMD64VPABSWMasked128Merging,
+ ssa.OpAMD64VPABSWMasked256Merging,
+ ssa.OpAMD64VPABSWMasked512Merging,
+ ssa.OpAMD64VPABSDMasked128Merging,
+ ssa.OpAMD64VPABSDMasked256Merging,
+ ssa.OpAMD64VPABSDMasked512Merging,
+ ssa.OpAMD64VPABSQMasked128Merging,
+ ssa.OpAMD64VPABSQMasked256Merging,
+ ssa.OpAMD64VPABSQMasked512Merging,
+ ssa.OpAMD64VRNDSCALEPSMasked128Merging,
+ ssa.OpAMD64VRNDSCALEPSMasked256Merging,
+ ssa.OpAMD64VRNDSCALEPSMasked512Merging,
+ ssa.OpAMD64VRNDSCALEPDMasked128Merging,
+ ssa.OpAMD64VRNDSCALEPDMasked256Merging,
+ ssa.OpAMD64VRNDSCALEPDMasked512Merging,
+ ssa.OpAMD64VREDUCEPSMasked128Merging,
+ ssa.OpAMD64VREDUCEPSMasked256Merging,
+ ssa.OpAMD64VREDUCEPSMasked512Merging,
+ ssa.OpAMD64VREDUCEPDMasked128Merging,
+ ssa.OpAMD64VREDUCEPDMasked256Merging,
+ ssa.OpAMD64VREDUCEPDMasked512Merging,
+ ssa.OpAMD64VCVTPD2PSXMasked128Merging,
+ ssa.OpAMD64VCVTPD2PSYMasked128Merging,
+ ssa.OpAMD64VCVTPD2PSMasked256Merging,
+ ssa.OpAMD64VCVTDQ2PSMasked128Merging,
+ ssa.OpAMD64VCVTDQ2PSMasked256Merging,
+ ssa.OpAMD64VCVTDQ2PSMasked512Merging,
+ ssa.OpAMD64VCVTQQ2PSXMasked128Merging,
+ ssa.OpAMD64VCVTQQ2PSYMasked128Merging,
+ ssa.OpAMD64VCVTQQ2PSMasked256Merging,
+ ssa.OpAMD64VCVTUDQ2PSMasked128Merging,
+ ssa.OpAMD64VCVTUDQ2PSMasked256Merging,
+ ssa.OpAMD64VCVTUDQ2PSMasked512Merging,
+ ssa.OpAMD64VCVTUQQ2PSXMasked128Merging,
+ ssa.OpAMD64VCVTUQQ2PSYMasked128Merging,
+ ssa.OpAMD64VCVTUQQ2PSMasked256Merging,
+ ssa.OpAMD64VCVTPS2PDMasked256Merging,
+ ssa.OpAMD64VCVTPS2PDMasked512Merging,
+ ssa.OpAMD64VCVTDQ2PDMasked256Merging,
+ ssa.OpAMD64VCVTDQ2PDMasked512Merging,
+ ssa.OpAMD64VCVTQQ2PDMasked128Merging,
+ ssa.OpAMD64VCVTQQ2PDMasked256Merging,
+ ssa.OpAMD64VCVTQQ2PDMasked512Merging,
+ ssa.OpAMD64VCVTUDQ2PDMasked256Merging,
+ ssa.OpAMD64VCVTUDQ2PDMasked512Merging,
+ ssa.OpAMD64VCVTUQQ2PDMasked128Merging,
+ ssa.OpAMD64VCVTUQQ2PDMasked256Merging,
+ ssa.OpAMD64VCVTUQQ2PDMasked512Merging,
+ ssa.OpAMD64VCVTTPS2DQMasked128Merging,
+ ssa.OpAMD64VCVTTPS2DQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2DQMasked512Merging,
+ ssa.OpAMD64VCVTTPD2DQXMasked128Merging,
+ ssa.OpAMD64VCVTTPD2DQYMasked128Merging,
+ ssa.OpAMD64VCVTTPD2DQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2QQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2QQMasked512Merging,
+ ssa.OpAMD64VCVTTPD2QQMasked128Merging,
+ ssa.OpAMD64VCVTTPD2QQMasked256Merging,
+ ssa.OpAMD64VCVTTPD2QQMasked512Merging,
+ ssa.OpAMD64VCVTTPS2UDQMasked128Merging,
+ ssa.OpAMD64VCVTTPS2UDQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2UDQMasked512Merging,
+ ssa.OpAMD64VCVTTPD2UDQXMasked128Merging,
+ ssa.OpAMD64VCVTTPD2UDQYMasked128Merging,
+ ssa.OpAMD64VCVTTPD2UDQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2UQQMasked256Merging,
+ ssa.OpAMD64VCVTTPS2UQQMasked512Merging,
+ ssa.OpAMD64VCVTTPD2UQQMasked128Merging,
+ ssa.OpAMD64VCVTTPD2UQQMasked256Merging,
+ ssa.OpAMD64VCVTTPD2UQQMasked512Merging,
+ ssa.OpAMD64VPMOVSXBQMasked128Merging,
+ ssa.OpAMD64VPMOVSXWQMasked128Merging,
+ ssa.OpAMD64VPMOVSXDQMasked128Merging,
+ ssa.OpAMD64VPMOVZXBQMasked128Merging,
+ ssa.OpAMD64VPMOVZXWQMasked128Merging,
+ ssa.OpAMD64VPMOVZXDQMasked128Merging,
+ ssa.OpAMD64VPMOVSXBDMasked128Merging,
+ ssa.OpAMD64VPMOVSXWDMasked128Merging,
+ ssa.OpAMD64VPMOVSXBQMasked256Merging,
+ ssa.OpAMD64VPMOVSXWQMasked256Merging,
+ ssa.OpAMD64VPMOVZXBDMasked128Merging,
+ ssa.OpAMD64VPMOVZXWDMasked128Merging,
+ ssa.OpAMD64VPMOVZXBQMasked256Merging,
+ ssa.OpAMD64VPMOVZXWQMasked256Merging,
+ ssa.OpAMD64VPMOVSXBWMasked128Merging,
+ ssa.OpAMD64VPMOVSXBDMasked256Merging,
+ ssa.OpAMD64VPMOVSXBQMasked512Merging,
+ ssa.OpAMD64VPMOVZXBWMasked128Merging,
+ ssa.OpAMD64VPMOVZXBDMasked256Merging,
+ ssa.OpAMD64VPMOVZXBQMasked512Merging,
+ ssa.OpAMD64VPMOVSXBWMasked256Merging,
+ ssa.OpAMD64VPMOVSXBWMasked512Merging,
+ ssa.OpAMD64VPMOVSXBDMasked512Merging,
+ ssa.OpAMD64VPMOVSXWDMasked256Merging,
+ ssa.OpAMD64VPMOVSXWDMasked512Merging,
+ ssa.OpAMD64VPMOVSXWQMasked512Merging,
+ ssa.OpAMD64VPMOVSXDQMasked256Merging,
+ ssa.OpAMD64VPMOVSXDQMasked512Merging,
+ ssa.OpAMD64VPMOVZXBWMasked256Merging,
+ ssa.OpAMD64VPMOVZXBWMasked512Merging,
+ ssa.OpAMD64VPMOVZXBDMasked512Merging,
+ ssa.OpAMD64VPMOVZXWDMasked256Merging,
+ ssa.OpAMD64VPMOVZXWDMasked512Merging,
+ ssa.OpAMD64VPMOVZXWQMasked512Merging,
+ ssa.OpAMD64VPMOVZXDQMasked256Merging,
+ ssa.OpAMD64VPMOVZXDQMasked512Merging,
+ ssa.OpAMD64VPLZCNTDMasked128Merging,
+ ssa.OpAMD64VPLZCNTDMasked256Merging,
+ ssa.OpAMD64VPLZCNTDMasked512Merging,
+ ssa.OpAMD64VPLZCNTQMasked128Merging,
+ ssa.OpAMD64VPLZCNTQMasked256Merging,
+ ssa.OpAMD64VPLZCNTQMasked512Merging,
+ ssa.OpAMD64VPOPCNTBMasked128Merging,
+ ssa.OpAMD64VPOPCNTBMasked256Merging,
+ ssa.OpAMD64VPOPCNTBMasked512Merging,
+ ssa.OpAMD64VPOPCNTWMasked128Merging,
+ ssa.OpAMD64VPOPCNTWMasked256Merging,
+ ssa.OpAMD64VPOPCNTWMasked512Merging,
+ ssa.OpAMD64VPOPCNTDMasked128Merging,
+ ssa.OpAMD64VPOPCNTDMasked256Merging,
+ ssa.OpAMD64VPOPCNTDMasked512Merging,
+ ssa.OpAMD64VPOPCNTQMasked128Merging,
+ ssa.OpAMD64VPOPCNTQMasked256Merging,
+ ssa.OpAMD64VPOPCNTQMasked512Merging,
+ ssa.OpAMD64VRCP14PSMasked128Merging,
+ ssa.OpAMD64VRCP14PSMasked256Merging,
+ ssa.OpAMD64VRCP14PSMasked512Merging,
+ ssa.OpAMD64VRCP14PDMasked128Merging,
+ ssa.OpAMD64VRCP14PDMasked256Merging,
+ ssa.OpAMD64VRCP14PDMasked512Merging,
+ ssa.OpAMD64VRSQRT14PSMasked128Merging,
+ ssa.OpAMD64VRSQRT14PSMasked256Merging,
+ ssa.OpAMD64VRSQRT14PSMasked512Merging,
+ ssa.OpAMD64VRSQRT14PDMasked128Merging,
+ ssa.OpAMD64VRSQRT14PDMasked256Merging,
+ ssa.OpAMD64VRSQRT14PDMasked512Merging,
+ ssa.OpAMD64VPMOVSWBMasked128_128Merging,
+ ssa.OpAMD64VPMOVSWBMasked128_256Merging,
+ ssa.OpAMD64VPMOVSWBMasked256Merging,
+ ssa.OpAMD64VPMOVSDBMasked128_128Merging,
+ ssa.OpAMD64VPMOVSDBMasked128_256Merging,
+ ssa.OpAMD64VPMOVSDBMasked128_512Merging,
+ ssa.OpAMD64VPMOVSQBMasked128_128Merging,
+ ssa.OpAMD64VPMOVSQBMasked128_256Merging,
+ ssa.OpAMD64VPMOVSQBMasked128_512Merging,
+ ssa.OpAMD64VPMOVSDWMasked128_128Merging,
+ ssa.OpAMD64VPMOVSDWMasked128_256Merging,
+ ssa.OpAMD64VPMOVSDWMasked256Merging,
+ ssa.OpAMD64VPMOVSQWMasked128_128Merging,
+ ssa.OpAMD64VPMOVSQWMasked128_256Merging,
+ ssa.OpAMD64VPMOVSQWMasked128_512Merging,
+ ssa.OpAMD64VPMOVSQDMasked128_128Merging,
+ ssa.OpAMD64VPMOVSQDMasked128_256Merging,
+ ssa.OpAMD64VPMOVSQDMasked256Merging,
+ ssa.OpAMD64VPMOVUSWBMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSWBMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSWBMasked256Merging,
+ ssa.OpAMD64VPMOVUSDBMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSDBMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSDBMasked128_512Merging,
+ ssa.OpAMD64VPMOVUSQBMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSQBMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSQBMasked128_512Merging,
+ ssa.OpAMD64VPMOVUSDWMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSDWMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSDWMasked256Merging,
+ ssa.OpAMD64VPMOVUSQWMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSQWMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSQWMasked128_512Merging,
+ ssa.OpAMD64VPMOVUSQDMasked128_128Merging,
+ ssa.OpAMD64VPMOVUSQDMasked128_256Merging,
+ ssa.OpAMD64VPMOVUSQDMasked256Merging,
+ ssa.OpAMD64VSQRTPSMasked128Merging,
+ ssa.OpAMD64VSQRTPSMasked256Merging,
+ ssa.OpAMD64VSQRTPSMasked512Merging,
+ ssa.OpAMD64VSQRTPDMasked128Merging,
+ ssa.OpAMD64VSQRTPDMasked256Merging,
+ ssa.OpAMD64VSQRTPDMasked512Merging,
+ ssa.OpAMD64VPMOVWBMasked128_128Merging,
+ ssa.OpAMD64VPMOVWBMasked128_256Merging,
+ ssa.OpAMD64VPMOVWBMasked256Merging,
+ ssa.OpAMD64VPMOVDBMasked128_128Merging,
+ ssa.OpAMD64VPMOVDBMasked128_256Merging,
+ ssa.OpAMD64VPMOVDBMasked128_512Merging,
+ ssa.OpAMD64VPMOVQBMasked128_128Merging,
+ ssa.OpAMD64VPMOVQBMasked128_256Merging,
+ ssa.OpAMD64VPMOVQBMasked128_512Merging,
+ ssa.OpAMD64VPMOVDWMasked128_128Merging,
+ ssa.OpAMD64VPMOVDWMasked128_256Merging,
+ ssa.OpAMD64VPMOVDWMasked256Merging,
+ ssa.OpAMD64VPMOVQWMasked128_128Merging,
+ ssa.OpAMD64VPMOVQWMasked128_256Merging,
+ ssa.OpAMD64VPMOVQWMasked128_512Merging,
+ ssa.OpAMD64VPMOVQDMasked128_128Merging,
+ ssa.OpAMD64VPMOVQDMasked128_256Merging,
+ ssa.OpAMD64VPMOVQDMasked256Merging,
+ ssa.OpAMD64VPBROADCASTQMasked128Merging,
+ ssa.OpAMD64VBROADCASTSSMasked128Merging,
+ ssa.OpAMD64VBROADCASTSDMasked256Merging,
+ ssa.OpAMD64VPBROADCASTDMasked128Merging,
+ ssa.OpAMD64VPBROADCASTQMasked256Merging,
+ ssa.OpAMD64VBROADCASTSSMasked256Merging,
+ ssa.OpAMD64VBROADCASTSDMasked512Merging,
+ ssa.OpAMD64VPBROADCASTWMasked128Merging,
+ ssa.OpAMD64VPBROADCASTDMasked256Merging,
+ ssa.OpAMD64VPBROADCASTQMasked512Merging,
+ ssa.OpAMD64VBROADCASTSSMasked512Merging,
+ ssa.OpAMD64VPBROADCASTBMasked128Merging,
+ ssa.OpAMD64VPBROADCASTWMasked256Merging,
+ ssa.OpAMD64VPBROADCASTDMasked512Merging,
+ ssa.OpAMD64VPBROADCASTBMasked256Merging,
+ ssa.OpAMD64VPBROADCASTWMasked512Merging,
+ ssa.OpAMD64VPBROADCASTBMasked512Merging,
+ ssa.OpAMD64VPSHUFDMasked256Merging,
+ ssa.OpAMD64VPSHUFDMasked512Merging,
+ ssa.OpAMD64VPSHUFHWMasked256Merging,
+ ssa.OpAMD64VPSHUFHWMasked512Merging,
+ ssa.OpAMD64VPSHUFHWMasked128Merging,
+ ssa.OpAMD64VPSHUFLWMasked256Merging,
+ ssa.OpAMD64VPSHUFLWMasked512Merging,
+ ssa.OpAMD64VPSHUFLWMasked128Merging,
+ ssa.OpAMD64VPSHUFDMasked128Merging,
+ ssa.OpAMD64VPROLDMasked128Merging,
+ ssa.OpAMD64VPROLDMasked256Merging,
+ ssa.OpAMD64VPROLDMasked512Merging,
+ ssa.OpAMD64VPROLQMasked128Merging,
+ ssa.OpAMD64VPROLQMasked256Merging,
+ ssa.OpAMD64VPROLQMasked512Merging,
+ ssa.OpAMD64VPRORDMasked128Merging,
+ ssa.OpAMD64VPRORDMasked256Merging,
+ ssa.OpAMD64VPRORDMasked512Merging,
+ ssa.OpAMD64VPRORQMasked128Merging,
+ ssa.OpAMD64VPRORQMasked256Merging,
+ ssa.OpAMD64VPRORQMasked512Merging,
+ ssa.OpAMD64VPSLLWMasked128constMerging,
+ ssa.OpAMD64VPSLLWMasked256constMerging,
+ ssa.OpAMD64VPSLLWMasked512constMerging,
+ ssa.OpAMD64VPSLLDMasked128constMerging,
+ ssa.OpAMD64VPSLLDMasked256constMerging,
+ ssa.OpAMD64VPSLLDMasked512constMerging,
+ ssa.OpAMD64VPSLLQMasked128constMerging,
+ ssa.OpAMD64VPSLLQMasked256constMerging,
+ ssa.OpAMD64VPSLLQMasked512constMerging,
+ ssa.OpAMD64VPSRAWMasked128constMerging,
+ ssa.OpAMD64VPSRAWMasked256constMerging,
+ ssa.OpAMD64VPSRAWMasked512constMerging,
+ ssa.OpAMD64VPSRADMasked128constMerging,
+ ssa.OpAMD64VPSRADMasked256constMerging,
+ ssa.OpAMD64VPSRADMasked512constMerging,
+ ssa.OpAMD64VPSRAQMasked128constMerging,
+ ssa.OpAMD64VPSRAQMasked256constMerging,
+ ssa.OpAMD64VPSRAQMasked512constMerging,
+ ssa.OpAMD64VPSRLWMasked128constMerging,
+ ssa.OpAMD64VPSRLWMasked256constMerging,
+ ssa.OpAMD64VPSRLWMasked512constMerging,
+ ssa.OpAMD64VPSRLDMasked128constMerging,
+ ssa.OpAMD64VPSRLDMasked256constMerging,
+ ssa.OpAMD64VPSRLDMasked512constMerging,
+ ssa.OpAMD64VPSRLQMasked128constMerging,
+ ssa.OpAMD64VPSRLQMasked256constMerging,
+ ssa.OpAMD64VPSRLQMasked512constMerging:
+ p = simdV2kvResultInArg0(s, v)
+
+ default:
+ // Unknown reg shape
+ return false
+ }
+
+ // Masked operation are always compiled with zeroing.
+ switch v.Op {
+ case ssa.OpAMD64VPABSBMasked128,
+ ssa.OpAMD64VPABSBMasked128load,
+ ssa.OpAMD64VPABSBMasked256,
+ ssa.OpAMD64VPABSBMasked256load,
+ ssa.OpAMD64VPABSBMasked512,
+ ssa.OpAMD64VPABSWMasked128,
+ ssa.OpAMD64VPABSWMasked128load,
+ ssa.OpAMD64VPABSWMasked256,
+ ssa.OpAMD64VPABSWMasked256load,
+ ssa.OpAMD64VPABSWMasked512,
+ ssa.OpAMD64VPABSDMasked128,
+ ssa.OpAMD64VPABSDMasked128load,
+ ssa.OpAMD64VPABSDMasked256,
+ ssa.OpAMD64VPABSDMasked256load,
+ ssa.OpAMD64VPABSDMasked512,
+ ssa.OpAMD64VPABSDMasked512load,
+ ssa.OpAMD64VPABSQMasked128,
+ ssa.OpAMD64VPABSQMasked128load,
+ ssa.OpAMD64VPABSQMasked256,
+ ssa.OpAMD64VPABSQMasked256load,
+ ssa.OpAMD64VPABSQMasked512,
+ ssa.OpAMD64VPABSQMasked512load,
+ ssa.OpAMD64VPDPWSSDMasked128,
+ ssa.OpAMD64VPDPWSSDMasked128load,
+ ssa.OpAMD64VPDPWSSDMasked256,
+ ssa.OpAMD64VPDPWSSDMasked256load,
+ ssa.OpAMD64VPDPWSSDMasked512,
+ ssa.OpAMD64VPDPWSSDMasked512load,
+ ssa.OpAMD64VADDPSMasked128,
+ ssa.OpAMD64VADDPSMasked128load,
+ ssa.OpAMD64VADDPSMasked256,
+ ssa.OpAMD64VADDPSMasked256load,
+ ssa.OpAMD64VADDPSMasked512,
+ ssa.OpAMD64VADDPSMasked512load,
+ ssa.OpAMD64VADDPDMasked128,
+ ssa.OpAMD64VADDPDMasked128load,
+ ssa.OpAMD64VADDPDMasked256,
+ ssa.OpAMD64VADDPDMasked256load,
+ ssa.OpAMD64VADDPDMasked512,
+ ssa.OpAMD64VADDPDMasked512load,
+ ssa.OpAMD64VPADDBMasked128,
+ ssa.OpAMD64VPADDBMasked128load,
+ ssa.OpAMD64VPADDBMasked256,
+ ssa.OpAMD64VPADDBMasked256load,
+ ssa.OpAMD64VPADDBMasked512,
+ ssa.OpAMD64VPADDWMasked128,
+ ssa.OpAMD64VPADDWMasked128load,
+ ssa.OpAMD64VPADDWMasked256,
+ ssa.OpAMD64VPADDWMasked256load,
+ ssa.OpAMD64VPADDWMasked512,
+ ssa.OpAMD64VPADDDMasked128,
+ ssa.OpAMD64VPADDDMasked128load,
+ ssa.OpAMD64VPADDDMasked256,
+ ssa.OpAMD64VPADDDMasked256load,
+ ssa.OpAMD64VPADDDMasked512,
+ ssa.OpAMD64VPADDDMasked512load,
+ ssa.OpAMD64VPADDQMasked128,
+ ssa.OpAMD64VPADDQMasked128load,
+ ssa.OpAMD64VPADDQMasked256,
+ ssa.OpAMD64VPADDQMasked256load,
+ ssa.OpAMD64VPADDQMasked512,
+ ssa.OpAMD64VPADDQMasked512load,
+ ssa.OpAMD64VPADDSBMasked128,
+ ssa.OpAMD64VPADDSBMasked128load,
+ ssa.OpAMD64VPADDSBMasked256,
+ ssa.OpAMD64VPADDSBMasked256load,
+ ssa.OpAMD64VPADDSBMasked512,
+ ssa.OpAMD64VPADDSWMasked128,
+ ssa.OpAMD64VPADDSWMasked128load,
+ ssa.OpAMD64VPADDSWMasked256,
+ ssa.OpAMD64VPADDSWMasked256load,
+ ssa.OpAMD64VPADDSWMasked512,
+ ssa.OpAMD64VPADDUSBMasked128,
+ ssa.OpAMD64VPADDUSBMasked128load,
+ ssa.OpAMD64VPADDUSBMasked256,
+ ssa.OpAMD64VPADDUSBMasked256load,
+ ssa.OpAMD64VPADDUSBMasked512,
+ ssa.OpAMD64VPADDUSWMasked128,
+ ssa.OpAMD64VPADDUSWMasked128load,
+ ssa.OpAMD64VPADDUSWMasked256,
+ ssa.OpAMD64VPADDUSWMasked256load,
+ ssa.OpAMD64VPADDUSWMasked512,
+ ssa.OpAMD64VPANDDMasked128,
+ ssa.OpAMD64VPANDDMasked128load,
+ ssa.OpAMD64VPANDDMasked256,
+ ssa.OpAMD64VPANDDMasked256load,
+ ssa.OpAMD64VPANDDMasked512,
+ ssa.OpAMD64VPANDDMasked512load,
+ ssa.OpAMD64VPANDQMasked128,
+ ssa.OpAMD64VPANDQMasked128load,
+ ssa.OpAMD64VPANDQMasked256,
+ ssa.OpAMD64VPANDQMasked256load,
+ ssa.OpAMD64VPANDQMasked512,
+ ssa.OpAMD64VPANDQMasked512load,
+ ssa.OpAMD64VPANDNDMasked128,
+ ssa.OpAMD64VPANDNDMasked128load,
+ ssa.OpAMD64VPANDNDMasked256,
+ ssa.OpAMD64VPANDNDMasked256load,
+ ssa.OpAMD64VPANDNDMasked512,
+ ssa.OpAMD64VPANDNDMasked512load,
+ ssa.OpAMD64VPANDNQMasked128,
+ ssa.OpAMD64VPANDNQMasked128load,
+ ssa.OpAMD64VPANDNQMasked256,
+ ssa.OpAMD64VPANDNQMasked256load,
+ ssa.OpAMD64VPANDNQMasked512,
+ ssa.OpAMD64VPANDNQMasked512load,
+ ssa.OpAMD64VPAVGBMasked128,
+ ssa.OpAMD64VPAVGBMasked128load,
+ ssa.OpAMD64VPAVGBMasked256,
+ ssa.OpAMD64VPAVGBMasked256load,
+ ssa.OpAMD64VPAVGBMasked512,
+ ssa.OpAMD64VPAVGWMasked128,
+ ssa.OpAMD64VPAVGWMasked128load,
+ ssa.OpAMD64VPAVGWMasked256,
+ ssa.OpAMD64VPAVGWMasked256load,
+ ssa.OpAMD64VPAVGWMasked512,
+ ssa.OpAMD64VRNDSCALEPSMasked128,
+ ssa.OpAMD64VRNDSCALEPSMasked128load,
+ ssa.OpAMD64VRNDSCALEPSMasked256,
+ ssa.OpAMD64VRNDSCALEPSMasked256load,
+ ssa.OpAMD64VRNDSCALEPSMasked512,
+ ssa.OpAMD64VRNDSCALEPSMasked512load,
+ ssa.OpAMD64VRNDSCALEPDMasked128,
+ ssa.OpAMD64VRNDSCALEPDMasked128load,
+ ssa.OpAMD64VRNDSCALEPDMasked256,
+ ssa.OpAMD64VRNDSCALEPDMasked256load,
+ ssa.OpAMD64VRNDSCALEPDMasked512,
+ ssa.OpAMD64VRNDSCALEPDMasked512load,
+ ssa.OpAMD64VREDUCEPSMasked128,
+ ssa.OpAMD64VREDUCEPSMasked128load,
+ ssa.OpAMD64VREDUCEPSMasked256,
+ ssa.OpAMD64VREDUCEPSMasked256load,
+ ssa.OpAMD64VREDUCEPSMasked512,
+ ssa.OpAMD64VREDUCEPSMasked512load,
+ ssa.OpAMD64VREDUCEPDMasked128,
+ ssa.OpAMD64VREDUCEPDMasked128load,
+ ssa.OpAMD64VREDUCEPDMasked256,
+ ssa.OpAMD64VREDUCEPDMasked256load,
+ ssa.OpAMD64VREDUCEPDMasked512,
+ ssa.OpAMD64VREDUCEPDMasked512load,
+ ssa.OpAMD64VCOMPRESSPSMasked128,
+ ssa.OpAMD64VCOMPRESSPSMasked256,
+ ssa.OpAMD64VCOMPRESSPSMasked512,
+ ssa.OpAMD64VCOMPRESSPDMasked128,
+ ssa.OpAMD64VCOMPRESSPDMasked256,
+ ssa.OpAMD64VCOMPRESSPDMasked512,
+ ssa.OpAMD64VPCOMPRESSBMasked128,
+ ssa.OpAMD64VPCOMPRESSBMasked256,
+ ssa.OpAMD64VPCOMPRESSBMasked512,
+ ssa.OpAMD64VPCOMPRESSWMasked128,
+ ssa.OpAMD64VPCOMPRESSWMasked256,
+ ssa.OpAMD64VPCOMPRESSWMasked512,
+ ssa.OpAMD64VPCOMPRESSDMasked128,
+ ssa.OpAMD64VPCOMPRESSDMasked256,
+ ssa.OpAMD64VPCOMPRESSDMasked512,
+ ssa.OpAMD64VPCOMPRESSQMasked128,
+ ssa.OpAMD64VPCOMPRESSQMasked256,
+ ssa.OpAMD64VPCOMPRESSQMasked512,
+ ssa.OpAMD64VPERMI2BMasked128,
+ ssa.OpAMD64VPERMI2BMasked128load,
+ ssa.OpAMD64VPERMI2BMasked256,
+ ssa.OpAMD64VPERMI2BMasked256load,
+ ssa.OpAMD64VPERMI2BMasked512,
+ ssa.OpAMD64VPERMI2WMasked128,
+ ssa.OpAMD64VPERMI2WMasked128load,
+ ssa.OpAMD64VPERMI2WMasked256,
+ ssa.OpAMD64VPERMI2WMasked256load,
+ ssa.OpAMD64VPERMI2WMasked512,
+ ssa.OpAMD64VPERMI2PSMasked128,
+ ssa.OpAMD64VPERMI2PSMasked128load,
+ ssa.OpAMD64VPERMI2DMasked128,
+ ssa.OpAMD64VPERMI2DMasked128load,
+ ssa.OpAMD64VPERMI2PSMasked256,
+ ssa.OpAMD64VPERMI2PSMasked256load,
+ ssa.OpAMD64VPERMI2DMasked256,
+ ssa.OpAMD64VPERMI2DMasked256load,
+ ssa.OpAMD64VPERMI2PSMasked512,
+ ssa.OpAMD64VPERMI2PSMasked512load,
+ ssa.OpAMD64VPERMI2DMasked512,
+ ssa.OpAMD64VPERMI2DMasked512load,
+ ssa.OpAMD64VPERMI2PDMasked128,
+ ssa.OpAMD64VPERMI2PDMasked128load,
+ ssa.OpAMD64VPERMI2QMasked128,
+ ssa.OpAMD64VPERMI2QMasked128load,
+ ssa.OpAMD64VPERMI2PDMasked256,
+ ssa.OpAMD64VPERMI2PDMasked256load,
+ ssa.OpAMD64VPERMI2QMasked256,
+ ssa.OpAMD64VPERMI2QMasked256load,
+ ssa.OpAMD64VPERMI2PDMasked512,
+ ssa.OpAMD64VPERMI2PDMasked512load,
+ ssa.OpAMD64VPERMI2QMasked512,
+ ssa.OpAMD64VPERMI2QMasked512load,
+ ssa.OpAMD64VPALIGNRMasked256,
+ ssa.OpAMD64VPALIGNRMasked256load,
+ ssa.OpAMD64VPALIGNRMasked512,
+ ssa.OpAMD64VPALIGNRMasked128,
+ ssa.OpAMD64VPALIGNRMasked128load,
+ ssa.OpAMD64VCVTPD2PSXMasked128,
+ ssa.OpAMD64VCVTPD2PSXMasked128load,
+ ssa.OpAMD64VCVTPD2PSYMasked128,
+ ssa.OpAMD64VCVTPD2PSYMasked128load,
+ ssa.OpAMD64VCVTPD2PSMasked256,
+ ssa.OpAMD64VCVTPD2PSMasked256load,
+ ssa.OpAMD64VCVTDQ2PSMasked128,
+ ssa.OpAMD64VCVTDQ2PSMasked128load,
+ ssa.OpAMD64VCVTDQ2PSMasked256,
+ ssa.OpAMD64VCVTDQ2PSMasked256load,
+ ssa.OpAMD64VCVTDQ2PSMasked512,
+ ssa.OpAMD64VCVTDQ2PSMasked512load,
+ ssa.OpAMD64VCVTQQ2PSXMasked128,
+ ssa.OpAMD64VCVTQQ2PSXMasked128load,
+ ssa.OpAMD64VCVTQQ2PSYMasked128,
+ ssa.OpAMD64VCVTQQ2PSYMasked128load,
+ ssa.OpAMD64VCVTQQ2PSMasked256,
+ ssa.OpAMD64VCVTQQ2PSMasked256load,
+ ssa.OpAMD64VCVTUDQ2PSMasked128,
+ ssa.OpAMD64VCVTUDQ2PSMasked128load,
+ ssa.OpAMD64VCVTUDQ2PSMasked256,
+ ssa.OpAMD64VCVTUDQ2PSMasked256load,
+ ssa.OpAMD64VCVTUDQ2PSMasked512,
+ ssa.OpAMD64VCVTUDQ2PSMasked512load,
+ ssa.OpAMD64VCVTUQQ2PSXMasked128,
+ ssa.OpAMD64VCVTUQQ2PSXMasked128load,
+ ssa.OpAMD64VCVTUQQ2PSYMasked128,
+ ssa.OpAMD64VCVTUQQ2PSYMasked128load,
+ ssa.OpAMD64VCVTUQQ2PSMasked256,
+ ssa.OpAMD64VCVTUQQ2PSMasked256load,
+ ssa.OpAMD64VCVTPS2PDMasked256,
+ ssa.OpAMD64VCVTPS2PDMasked256load,
+ ssa.OpAMD64VCVTPS2PDMasked512,
+ ssa.OpAMD64VCVTPS2PDMasked512load,
+ ssa.OpAMD64VCVTDQ2PDMasked256,
+ ssa.OpAMD64VCVTDQ2PDMasked256load,
+ ssa.OpAMD64VCVTDQ2PDMasked512,
+ ssa.OpAMD64VCVTDQ2PDMasked512load,
+ ssa.OpAMD64VCVTQQ2PDMasked128,
+ ssa.OpAMD64VCVTQQ2PDMasked128load,
+ ssa.OpAMD64VCVTQQ2PDMasked256,
+ ssa.OpAMD64VCVTQQ2PDMasked256load,
+ ssa.OpAMD64VCVTQQ2PDMasked512,
+ ssa.OpAMD64VCVTQQ2PDMasked512load,
+ ssa.OpAMD64VCVTUDQ2PDMasked256,
+ ssa.OpAMD64VCVTUDQ2PDMasked256load,
+ ssa.OpAMD64VCVTUDQ2PDMasked512,
+ ssa.OpAMD64VCVTUDQ2PDMasked512load,
+ ssa.OpAMD64VCVTUQQ2PDMasked128,
+ ssa.OpAMD64VCVTUQQ2PDMasked128load,
+ ssa.OpAMD64VCVTUQQ2PDMasked256,
+ ssa.OpAMD64VCVTUQQ2PDMasked256load,
+ ssa.OpAMD64VCVTUQQ2PDMasked512,
+ ssa.OpAMD64VCVTUQQ2PDMasked512load,
+ ssa.OpAMD64VCVTTPS2DQMasked128,
+ ssa.OpAMD64VCVTTPS2DQMasked128load,
+ ssa.OpAMD64VCVTTPS2DQMasked256,
+ ssa.OpAMD64VCVTTPS2DQMasked256load,
+ ssa.OpAMD64VCVTTPS2DQMasked512,
+ ssa.OpAMD64VCVTTPS2DQMasked512load,
+ ssa.OpAMD64VCVTTPD2DQXMasked128,
+ ssa.OpAMD64VCVTTPD2DQXMasked128load,
+ ssa.OpAMD64VCVTTPD2DQYMasked128,
+ ssa.OpAMD64VCVTTPD2DQYMasked128load,
+ ssa.OpAMD64VCVTTPD2DQMasked256,
+ ssa.OpAMD64VCVTTPD2DQMasked256load,
+ ssa.OpAMD64VCVTTPS2QQMasked256,
+ ssa.OpAMD64VCVTTPS2QQMasked256load,
+ ssa.OpAMD64VCVTTPS2QQMasked512,
+ ssa.OpAMD64VCVTTPS2QQMasked512load,
+ ssa.OpAMD64VCVTTPD2QQMasked128,
+ ssa.OpAMD64VCVTTPD2QQMasked128load,
+ ssa.OpAMD64VCVTTPD2QQMasked256,
+ ssa.OpAMD64VCVTTPD2QQMasked256load,
+ ssa.OpAMD64VCVTTPD2QQMasked512,
+ ssa.OpAMD64VCVTTPD2QQMasked512load,
+ ssa.OpAMD64VCVTTPS2UDQMasked128,
+ ssa.OpAMD64VCVTTPS2UDQMasked128load,
+ ssa.OpAMD64VCVTTPS2UDQMasked256,
+ ssa.OpAMD64VCVTTPS2UDQMasked256load,
+ ssa.OpAMD64VCVTTPS2UDQMasked512,
+ ssa.OpAMD64VCVTTPS2UDQMasked512load,
+ ssa.OpAMD64VCVTTPD2UDQXMasked128,
+ ssa.OpAMD64VCVTTPD2UDQXMasked128load,
+ ssa.OpAMD64VCVTTPD2UDQYMasked128,
+ ssa.OpAMD64VCVTTPD2UDQYMasked128load,
+ ssa.OpAMD64VCVTTPD2UDQMasked256,
+ ssa.OpAMD64VCVTTPD2UDQMasked256load,
+ ssa.OpAMD64VCVTTPS2UQQMasked256,
+ ssa.OpAMD64VCVTTPS2UQQMasked256load,
+ ssa.OpAMD64VCVTTPS2UQQMasked512,
+ ssa.OpAMD64VCVTTPS2UQQMasked512load,
+ ssa.OpAMD64VCVTTPD2UQQMasked128,
+ ssa.OpAMD64VCVTTPD2UQQMasked128load,
+ ssa.OpAMD64VCVTTPD2UQQMasked256,
+ ssa.OpAMD64VCVTTPD2UQQMasked256load,
+ ssa.OpAMD64VCVTTPD2UQQMasked512,
+ ssa.OpAMD64VCVTTPD2UQQMasked512load,
+ ssa.OpAMD64VDIVPSMasked128,
+ ssa.OpAMD64VDIVPSMasked128load,
+ ssa.OpAMD64VDIVPSMasked256,
+ ssa.OpAMD64VDIVPSMasked256load,
+ ssa.OpAMD64VDIVPSMasked512,
+ ssa.OpAMD64VDIVPSMasked512load,
+ ssa.OpAMD64VDIVPDMasked128,
+ ssa.OpAMD64VDIVPDMasked128load,
+ ssa.OpAMD64VDIVPDMasked256,
+ ssa.OpAMD64VDIVPDMasked256load,
+ ssa.OpAMD64VDIVPDMasked512,
+ ssa.OpAMD64VDIVPDMasked512load,
+ ssa.OpAMD64VPMADDWDMasked128,
+ ssa.OpAMD64VPMADDWDMasked128load,
+ ssa.OpAMD64VPMADDWDMasked256,
+ ssa.OpAMD64VPMADDWDMasked256load,
+ ssa.OpAMD64VPMADDWDMasked512,
+ ssa.OpAMD64VPMADDUBSWMasked128,
+ ssa.OpAMD64VPMADDUBSWMasked128load,
+ ssa.OpAMD64VPMADDUBSWMasked256,
+ ssa.OpAMD64VPMADDUBSWMasked256load,
+ ssa.OpAMD64VPMADDUBSWMasked512,
+ ssa.OpAMD64VEXPANDPSMasked128,
+ ssa.OpAMD64VEXPANDPSMasked128load,
+ ssa.OpAMD64VEXPANDPSMasked256,
+ ssa.OpAMD64VEXPANDPSMasked256load,
+ ssa.OpAMD64VEXPANDPSMasked512,
+ ssa.OpAMD64VEXPANDPDMasked128,
+ ssa.OpAMD64VEXPANDPDMasked128load,
+ ssa.OpAMD64VEXPANDPDMasked256,
+ ssa.OpAMD64VEXPANDPDMasked256load,
+ ssa.OpAMD64VEXPANDPDMasked512,
+ ssa.OpAMD64VPEXPANDBMasked128,
+ ssa.OpAMD64VPEXPANDBMasked128load,
+ ssa.OpAMD64VPEXPANDBMasked256,
+ ssa.OpAMD64VPEXPANDBMasked256load,
+ ssa.OpAMD64VPEXPANDBMasked512,
+ ssa.OpAMD64VPEXPANDWMasked128,
+ ssa.OpAMD64VPEXPANDWMasked128load,
+ ssa.OpAMD64VPEXPANDWMasked256,
+ ssa.OpAMD64VPEXPANDWMasked256load,
+ ssa.OpAMD64VPEXPANDWMasked512,
+ ssa.OpAMD64VPEXPANDDMasked128,
+ ssa.OpAMD64VPEXPANDDMasked128load,
+ ssa.OpAMD64VPEXPANDDMasked256,
+ ssa.OpAMD64VPEXPANDDMasked256load,
+ ssa.OpAMD64VPEXPANDDMasked512,
+ ssa.OpAMD64VPEXPANDQMasked128,
+ ssa.OpAMD64VPEXPANDQMasked128load,
+ ssa.OpAMD64VPEXPANDQMasked256,
+ ssa.OpAMD64VPEXPANDQMasked256load,
+ ssa.OpAMD64VPEXPANDQMasked512,
+ ssa.OpAMD64VPMOVSXBQMasked128,
+ ssa.OpAMD64VPMOVSXWQMasked128,
+ ssa.OpAMD64VPMOVSXWQMasked128load,
+ ssa.OpAMD64VPMOVSXDQMasked128,
+ ssa.OpAMD64VPMOVSXDQMasked128load,
+ ssa.OpAMD64VPMOVZXBQMasked128,
+ ssa.OpAMD64VPMOVZXWQMasked128,
+ ssa.OpAMD64VPMOVZXWQMasked128load,
+ ssa.OpAMD64VPMOVZXDQMasked128,
+ ssa.OpAMD64VPMOVZXDQMasked128load,
+ ssa.OpAMD64VPMOVSXBDMasked128,
+ ssa.OpAMD64VPMOVSXBDMasked128load,
+ ssa.OpAMD64VPMOVSXWDMasked128,
+ ssa.OpAMD64VPMOVSXWDMasked128load,
+ ssa.OpAMD64VPMOVSXBQMasked256,
+ ssa.OpAMD64VPMOVSXBQMasked256load,
+ ssa.OpAMD64VPMOVSXWQMasked256,
+ ssa.OpAMD64VPMOVSXWQMasked256load,
+ ssa.OpAMD64VPMOVZXBDMasked128,
+ ssa.OpAMD64VPMOVZXBDMasked128load,
+ ssa.OpAMD64VPMOVZXWDMasked128,
+ ssa.OpAMD64VPMOVZXWDMasked128load,
+ ssa.OpAMD64VPMOVZXBQMasked256,
+ ssa.OpAMD64VPMOVZXBQMasked256load,
+ ssa.OpAMD64VPMOVZXWQMasked256,
+ ssa.OpAMD64VPMOVZXWQMasked256load,
+ ssa.OpAMD64VPMOVSXBWMasked128,
+ ssa.OpAMD64VPMOVSXBWMasked128load,
+ ssa.OpAMD64VPMOVSXBDMasked256,
+ ssa.OpAMD64VPMOVSXBDMasked256load,
+ ssa.OpAMD64VPMOVSXBQMasked512,
+ ssa.OpAMD64VPMOVSXBQMasked512load,
+ ssa.OpAMD64VPMOVZXBWMasked128,
+ ssa.OpAMD64VPMOVZXBWMasked128load,
+ ssa.OpAMD64VPMOVZXBDMasked256,
+ ssa.OpAMD64VPMOVZXBDMasked256load,
+ ssa.OpAMD64VPMOVZXBQMasked512,
+ ssa.OpAMD64VPMOVZXBQMasked512load,
+ ssa.OpAMD64VPMOVSXBWMasked256,
+ ssa.OpAMD64VPMOVSXBWMasked256load,
+ ssa.OpAMD64VPMOVSXBWMasked512,
+ ssa.OpAMD64VPMOVSXBWMasked512load,
+ ssa.OpAMD64VPMOVSXBDMasked512,
+ ssa.OpAMD64VPMOVSXBDMasked512load,
+ ssa.OpAMD64VPMOVSXWDMasked256,
+ ssa.OpAMD64VPMOVSXWDMasked256load,
+ ssa.OpAMD64VPMOVSXWDMasked512,
+ ssa.OpAMD64VPMOVSXWDMasked512load,
+ ssa.OpAMD64VPMOVSXWQMasked512,
+ ssa.OpAMD64VPMOVSXWQMasked512load,
+ ssa.OpAMD64VPMOVSXDQMasked256,
+ ssa.OpAMD64VPMOVSXDQMasked256load,
+ ssa.OpAMD64VPMOVSXDQMasked512,
+ ssa.OpAMD64VPMOVSXDQMasked512load,
+ ssa.OpAMD64VPMOVZXBWMasked256,
+ ssa.OpAMD64VPMOVZXBWMasked256load,
+ ssa.OpAMD64VPMOVZXBWMasked512,
+ ssa.OpAMD64VPMOVZXBWMasked512load,
+ ssa.OpAMD64VPMOVZXBDMasked512,
+ ssa.OpAMD64VPMOVZXBDMasked512load,
+ ssa.OpAMD64VPMOVZXWDMasked256,
+ ssa.OpAMD64VPMOVZXWDMasked256load,
+ ssa.OpAMD64VPMOVZXWDMasked512,
+ ssa.OpAMD64VPMOVZXWDMasked512load,
+ ssa.OpAMD64VPMOVZXWQMasked512,
+ ssa.OpAMD64VPMOVZXWQMasked512load,
+ ssa.OpAMD64VPMOVZXDQMasked256,
+ ssa.OpAMD64VPMOVZXDQMasked256load,
+ ssa.OpAMD64VPMOVZXDQMasked512,
+ ssa.OpAMD64VPMOVZXDQMasked512load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked128,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked128load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked256,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked256load,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked512,
+ ssa.OpAMD64VGF2P8AFFINEINVQBMasked512load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked128,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked128load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked256,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked256load,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked512,
+ ssa.OpAMD64VGF2P8AFFINEQBMasked512load,
+ ssa.OpAMD64VGF2P8MULBMasked128,
+ ssa.OpAMD64VGF2P8MULBMasked128load,
+ ssa.OpAMD64VGF2P8MULBMasked256,
+ ssa.OpAMD64VGF2P8MULBMasked256load,
+ ssa.OpAMD64VGF2P8MULBMasked512,
+ ssa.OpAMD64VPLZCNTDMasked128,
+ ssa.OpAMD64VPLZCNTDMasked128load,
+ ssa.OpAMD64VPLZCNTDMasked256,
+ ssa.OpAMD64VPLZCNTDMasked256load,
+ ssa.OpAMD64VPLZCNTDMasked512,
+ ssa.OpAMD64VPLZCNTDMasked512load,
+ ssa.OpAMD64VPLZCNTQMasked128,
+ ssa.OpAMD64VPLZCNTQMasked128load,
+ ssa.OpAMD64VPLZCNTQMasked256,
+ ssa.OpAMD64VPLZCNTQMasked256load,
+ ssa.OpAMD64VPLZCNTQMasked512,
+ ssa.OpAMD64VPLZCNTQMasked512load,
+ ssa.OpAMD64VMAXPSMasked128,
+ ssa.OpAMD64VMAXPSMasked128load,
+ ssa.OpAMD64VMAXPSMasked256,
+ ssa.OpAMD64VMAXPSMasked256load,
+ ssa.OpAMD64VMAXPSMasked512,
+ ssa.OpAMD64VMAXPSMasked512load,
+ ssa.OpAMD64VMAXPDMasked128,
+ ssa.OpAMD64VMAXPDMasked128load,
+ ssa.OpAMD64VMAXPDMasked256,
+ ssa.OpAMD64VMAXPDMasked256load,
+ ssa.OpAMD64VMAXPDMasked512,
+ ssa.OpAMD64VMAXPDMasked512load,
+ ssa.OpAMD64VPMAXSBMasked128,
+ ssa.OpAMD64VPMAXSBMasked128load,
+ ssa.OpAMD64VPMAXSBMasked256,
+ ssa.OpAMD64VPMAXSBMasked256load,
+ ssa.OpAMD64VPMAXSBMasked512,
+ ssa.OpAMD64VPMAXSWMasked128,
+ ssa.OpAMD64VPMAXSWMasked128load,
+ ssa.OpAMD64VPMAXSWMasked256,
+ ssa.OpAMD64VPMAXSWMasked256load,
+ ssa.OpAMD64VPMAXSWMasked512,
+ ssa.OpAMD64VPMAXSDMasked128,
+ ssa.OpAMD64VPMAXSDMasked128load,
+ ssa.OpAMD64VPMAXSDMasked256,
+ ssa.OpAMD64VPMAXSDMasked256load,
+ ssa.OpAMD64VPMAXSDMasked512,
+ ssa.OpAMD64VPMAXSDMasked512load,
+ ssa.OpAMD64VPMAXSQMasked128,
+ ssa.OpAMD64VPMAXSQMasked128load,
+ ssa.OpAMD64VPMAXSQMasked256,
+ ssa.OpAMD64VPMAXSQMasked256load,
+ ssa.OpAMD64VPMAXSQMasked512,
+ ssa.OpAMD64VPMAXSQMasked512load,
+ ssa.OpAMD64VPMAXUBMasked128,
+ ssa.OpAMD64VPMAXUBMasked128load,
+ ssa.OpAMD64VPMAXUBMasked256,
+ ssa.OpAMD64VPMAXUBMasked256load,
+ ssa.OpAMD64VPMAXUBMasked512,
+ ssa.OpAMD64VPMAXUWMasked128,
+ ssa.OpAMD64VPMAXUWMasked128load,
+ ssa.OpAMD64VPMAXUWMasked256,
+ ssa.OpAMD64VPMAXUWMasked256load,
+ ssa.OpAMD64VPMAXUWMasked512,
+ ssa.OpAMD64VPMAXUDMasked128,
+ ssa.OpAMD64VPMAXUDMasked128load,
+ ssa.OpAMD64VPMAXUDMasked256,
+ ssa.OpAMD64VPMAXUDMasked256load,
+ ssa.OpAMD64VPMAXUDMasked512,
+ ssa.OpAMD64VPMAXUDMasked512load,
+ ssa.OpAMD64VPMAXUQMasked128,
+ ssa.OpAMD64VPMAXUQMasked128load,
+ ssa.OpAMD64VPMAXUQMasked256,
+ ssa.OpAMD64VPMAXUQMasked256load,
+ ssa.OpAMD64VPMAXUQMasked512,
+ ssa.OpAMD64VPMAXUQMasked512load,
+ ssa.OpAMD64VMINPSMasked128,
+ ssa.OpAMD64VMINPSMasked128load,
+ ssa.OpAMD64VMINPSMasked256,
+ ssa.OpAMD64VMINPSMasked256load,
+ ssa.OpAMD64VMINPSMasked512,
+ ssa.OpAMD64VMINPSMasked512load,
+ ssa.OpAMD64VMINPDMasked128,
+ ssa.OpAMD64VMINPDMasked128load,
+ ssa.OpAMD64VMINPDMasked256,
+ ssa.OpAMD64VMINPDMasked256load,
+ ssa.OpAMD64VMINPDMasked512,
+ ssa.OpAMD64VMINPDMasked512load,
+ ssa.OpAMD64VPMINSBMasked128,
+ ssa.OpAMD64VPMINSBMasked128load,
+ ssa.OpAMD64VPMINSBMasked256,
+ ssa.OpAMD64VPMINSBMasked256load,
+ ssa.OpAMD64VPMINSBMasked512,
+ ssa.OpAMD64VPMINSWMasked128,
+ ssa.OpAMD64VPMINSWMasked128load,
+ ssa.OpAMD64VPMINSWMasked256,
+ ssa.OpAMD64VPMINSWMasked256load,
+ ssa.OpAMD64VPMINSWMasked512,
+ ssa.OpAMD64VPMINSDMasked128,
+ ssa.OpAMD64VPMINSDMasked128load,
+ ssa.OpAMD64VPMINSDMasked256,
+ ssa.OpAMD64VPMINSDMasked256load,
+ ssa.OpAMD64VPMINSDMasked512,
+ ssa.OpAMD64VPMINSDMasked512load,
+ ssa.OpAMD64VPMINSQMasked128,
+ ssa.OpAMD64VPMINSQMasked128load,
+ ssa.OpAMD64VPMINSQMasked256,
+ ssa.OpAMD64VPMINSQMasked256load,
+ ssa.OpAMD64VPMINSQMasked512,
+ ssa.OpAMD64VPMINSQMasked512load,
+ ssa.OpAMD64VPMINUBMasked128,
+ ssa.OpAMD64VPMINUBMasked128load,
+ ssa.OpAMD64VPMINUBMasked256,
+ ssa.OpAMD64VPMINUBMasked256load,
+ ssa.OpAMD64VPMINUBMasked512,
+ ssa.OpAMD64VPMINUWMasked128,
+ ssa.OpAMD64VPMINUWMasked128load,
+ ssa.OpAMD64VPMINUWMasked256,
+ ssa.OpAMD64VPMINUWMasked256load,
+ ssa.OpAMD64VPMINUWMasked512,
+ ssa.OpAMD64VPMINUDMasked128,
+ ssa.OpAMD64VPMINUDMasked128load,
+ ssa.OpAMD64VPMINUDMasked256,
+ ssa.OpAMD64VPMINUDMasked256load,
+ ssa.OpAMD64VPMINUDMasked512,
+ ssa.OpAMD64VPMINUDMasked512load,
+ ssa.OpAMD64VPMINUQMasked128,
+ ssa.OpAMD64VPMINUQMasked128load,
+ ssa.OpAMD64VPMINUQMasked256,
+ ssa.OpAMD64VPMINUQMasked256load,
+ ssa.OpAMD64VPMINUQMasked512,
+ ssa.OpAMD64VPMINUQMasked512load,
+ ssa.OpAMD64VFMADDSUB213PSMasked128,
+ ssa.OpAMD64VFMADDSUB213PSMasked128load,
+ ssa.OpAMD64VFMADDSUB213PSMasked256,
+ ssa.OpAMD64VFMADDSUB213PSMasked256load,
+ ssa.OpAMD64VFMADDSUB213PSMasked512,
+ ssa.OpAMD64VFMADDSUB213PSMasked512load,
+ ssa.OpAMD64VFMADDSUB213PDMasked128,
+ ssa.OpAMD64VFMADDSUB213PDMasked128load,
+ ssa.OpAMD64VFMADDSUB213PDMasked256,
+ ssa.OpAMD64VFMADDSUB213PDMasked256load,
+ ssa.OpAMD64VFMADDSUB213PDMasked512,
+ ssa.OpAMD64VFMADDSUB213PDMasked512load,
+ ssa.OpAMD64VFMADD213PSMasked128,
+ ssa.OpAMD64VFMADD213PSMasked128load,
+ ssa.OpAMD64VFMADD213PSMasked256,
+ ssa.OpAMD64VFMADD213PSMasked256load,
+ ssa.OpAMD64VFMADD213PSMasked512,
+ ssa.OpAMD64VFMADD213PSMasked512load,
+ ssa.OpAMD64VFMADD213PDMasked128,
+ ssa.OpAMD64VFMADD213PDMasked128load,
+ ssa.OpAMD64VFMADD213PDMasked256,
+ ssa.OpAMD64VFMADD213PDMasked256load,
+ ssa.OpAMD64VFMADD213PDMasked512,
+ ssa.OpAMD64VFMADD213PDMasked512load,
+ ssa.OpAMD64VFMSUBADD213PSMasked128,
+ ssa.OpAMD64VFMSUBADD213PSMasked128load,
+ ssa.OpAMD64VFMSUBADD213PSMasked256,
+ ssa.OpAMD64VFMSUBADD213PSMasked256load,
+ ssa.OpAMD64VFMSUBADD213PSMasked512,
+ ssa.OpAMD64VFMSUBADD213PSMasked512load,
+ ssa.OpAMD64VFMSUBADD213PDMasked128,
+ ssa.OpAMD64VFMSUBADD213PDMasked128load,
+ ssa.OpAMD64VFMSUBADD213PDMasked256,
+ ssa.OpAMD64VFMSUBADD213PDMasked256load,
+ ssa.OpAMD64VFMSUBADD213PDMasked512,
+ ssa.OpAMD64VFMSUBADD213PDMasked512load,
+ ssa.OpAMD64VPMULHWMasked128,
+ ssa.OpAMD64VPMULHWMasked128load,
+ ssa.OpAMD64VPMULHWMasked256,
+ ssa.OpAMD64VPMULHWMasked256load,
+ ssa.OpAMD64VPMULHWMasked512,
+ ssa.OpAMD64VPMULHUWMasked128,
+ ssa.OpAMD64VPMULHUWMasked128load,
+ ssa.OpAMD64VPMULHUWMasked256,
+ ssa.OpAMD64VPMULHUWMasked256load,
+ ssa.OpAMD64VPMULHUWMasked512,
+ ssa.OpAMD64VMULPSMasked128,
+ ssa.OpAMD64VMULPSMasked128load,
+ ssa.OpAMD64VMULPSMasked256,
+ ssa.OpAMD64VMULPSMasked256load,
+ ssa.OpAMD64VMULPSMasked512,
+ ssa.OpAMD64VMULPSMasked512load,
+ ssa.OpAMD64VMULPDMasked128,
+ ssa.OpAMD64VMULPDMasked128load,
+ ssa.OpAMD64VMULPDMasked256,
+ ssa.OpAMD64VMULPDMasked256load,
+ ssa.OpAMD64VMULPDMasked512,
+ ssa.OpAMD64VMULPDMasked512load,
+ ssa.OpAMD64VPMULLWMasked128,
+ ssa.OpAMD64VPMULLWMasked128load,
+ ssa.OpAMD64VPMULLWMasked256,
+ ssa.OpAMD64VPMULLWMasked256load,
+ ssa.OpAMD64VPMULLWMasked512,
+ ssa.OpAMD64VPMULLDMasked128,
+ ssa.OpAMD64VPMULLDMasked128load,
+ ssa.OpAMD64VPMULLDMasked256,
+ ssa.OpAMD64VPMULLDMasked256load,
+ ssa.OpAMD64VPMULLDMasked512,
+ ssa.OpAMD64VPMULLDMasked512load,
+ ssa.OpAMD64VPMULLQMasked128,
+ ssa.OpAMD64VPMULLQMasked128load,
+ ssa.OpAMD64VPMULLQMasked256,
+ ssa.OpAMD64VPMULLQMasked256load,
+ ssa.OpAMD64VPMULLQMasked512,
+ ssa.OpAMD64VPMULLQMasked512load,
+ ssa.OpAMD64VPOPCNTBMasked128,
+ ssa.OpAMD64VPOPCNTBMasked128load,
+ ssa.OpAMD64VPOPCNTBMasked256,
+ ssa.OpAMD64VPOPCNTBMasked256load,
+ ssa.OpAMD64VPOPCNTBMasked512,
+ ssa.OpAMD64VPOPCNTWMasked128,
+ ssa.OpAMD64VPOPCNTWMasked128load,
+ ssa.OpAMD64VPOPCNTWMasked256,
+ ssa.OpAMD64VPOPCNTWMasked256load,
+ ssa.OpAMD64VPOPCNTWMasked512,
+ ssa.OpAMD64VPOPCNTDMasked128,
+ ssa.OpAMD64VPOPCNTDMasked128load,
+ ssa.OpAMD64VPOPCNTDMasked256,
+ ssa.OpAMD64VPOPCNTDMasked256load,
+ ssa.OpAMD64VPOPCNTDMasked512,
+ ssa.OpAMD64VPOPCNTDMasked512load,
+ ssa.OpAMD64VPOPCNTQMasked128,
+ ssa.OpAMD64VPOPCNTQMasked128load,
+ ssa.OpAMD64VPOPCNTQMasked256,
+ ssa.OpAMD64VPOPCNTQMasked256load,
+ ssa.OpAMD64VPOPCNTQMasked512,
+ ssa.OpAMD64VPOPCNTQMasked512load,
+ ssa.OpAMD64VPORDMasked128,
+ ssa.OpAMD64VPORDMasked128load,
+ ssa.OpAMD64VPORDMasked256,
+ ssa.OpAMD64VPORDMasked256load,
+ ssa.OpAMD64VPORDMasked512,
+ ssa.OpAMD64VPORDMasked512load,
+ ssa.OpAMD64VPORQMasked128,
+ ssa.OpAMD64VPORQMasked128load,
+ ssa.OpAMD64VPORQMasked256,
+ ssa.OpAMD64VPORQMasked256load,
+ ssa.OpAMD64VPORQMasked512,
+ ssa.OpAMD64VPORQMasked512load,
+ ssa.OpAMD64VPERMBMasked128,
+ ssa.OpAMD64VPERMBMasked128load,
+ ssa.OpAMD64VPERMBMasked256,
+ ssa.OpAMD64VPERMBMasked256load,
+ ssa.OpAMD64VPERMBMasked512,
+ ssa.OpAMD64VPERMWMasked128,
+ ssa.OpAMD64VPERMWMasked128load,
+ ssa.OpAMD64VPERMWMasked256,
+ ssa.OpAMD64VPERMWMasked256load,
+ ssa.OpAMD64VPERMWMasked512,
+ ssa.OpAMD64VPERMPSMasked256,
+ ssa.OpAMD64VPERMPSMasked256load,
+ ssa.OpAMD64VPERMDMasked256,
+ ssa.OpAMD64VPERMDMasked256load,
+ ssa.OpAMD64VPERMPSMasked512,
+ ssa.OpAMD64VPERMPSMasked512load,
+ ssa.OpAMD64VPERMDMasked512,
+ ssa.OpAMD64VPERMDMasked512load,
+ ssa.OpAMD64VPERMPDMasked256,
+ ssa.OpAMD64VPERMPDMasked256load,
+ ssa.OpAMD64VPERMQMasked256,
+ ssa.OpAMD64VPERMQMasked256load,
+ ssa.OpAMD64VPERMPDMasked512,
+ ssa.OpAMD64VPERMPDMasked512load,
+ ssa.OpAMD64VPERMQMasked512,
+ ssa.OpAMD64VPERMQMasked512load,
+ ssa.OpAMD64VPSHUFBMasked256,
+ ssa.OpAMD64VPSHUFBMasked256load,
+ ssa.OpAMD64VPSHUFBMasked512,
+ ssa.OpAMD64VPSHUFBMasked128,
+ ssa.OpAMD64VPSHUFBMasked128load,
+ ssa.OpAMD64VRCP14PSMasked128,
+ ssa.OpAMD64VRCP14PSMasked128load,
+ ssa.OpAMD64VRCP14PSMasked256,
+ ssa.OpAMD64VRCP14PSMasked256load,
+ ssa.OpAMD64VRCP14PSMasked512,
+ ssa.OpAMD64VRCP14PSMasked512load,
+ ssa.OpAMD64VRCP14PDMasked128,
+ ssa.OpAMD64VRCP14PDMasked128load,
+ ssa.OpAMD64VRCP14PDMasked256,
+ ssa.OpAMD64VRCP14PDMasked256load,
+ ssa.OpAMD64VRCP14PDMasked512,
+ ssa.OpAMD64VRCP14PDMasked512load,
+ ssa.OpAMD64VRSQRT14PSMasked128,
+ ssa.OpAMD64VRSQRT14PSMasked128load,
+ ssa.OpAMD64VRSQRT14PSMasked256,
+ ssa.OpAMD64VRSQRT14PSMasked256load,
+ ssa.OpAMD64VRSQRT14PSMasked512,
+ ssa.OpAMD64VRSQRT14PSMasked512load,
+ ssa.OpAMD64VRSQRT14PDMasked128,
+ ssa.OpAMD64VRSQRT14PDMasked128load,
+ ssa.OpAMD64VRSQRT14PDMasked256,
+ ssa.OpAMD64VRSQRT14PDMasked256load,
+ ssa.OpAMD64VRSQRT14PDMasked512,
+ ssa.OpAMD64VRSQRT14PDMasked512load,
+ ssa.OpAMD64VPROLVDMasked128,
+ ssa.OpAMD64VPROLVDMasked128load,
+ ssa.OpAMD64VPROLVDMasked256,
+ ssa.OpAMD64VPROLVDMasked256load,
+ ssa.OpAMD64VPROLVDMasked512,
+ ssa.OpAMD64VPROLVDMasked512load,
+ ssa.OpAMD64VPROLVQMasked128,
+ ssa.OpAMD64VPROLVQMasked128load,
+ ssa.OpAMD64VPROLVQMasked256,
+ ssa.OpAMD64VPROLVQMasked256load,
+ ssa.OpAMD64VPROLVQMasked512,
+ ssa.OpAMD64VPROLVQMasked512load,
+ ssa.OpAMD64VPRORVDMasked128,
+ ssa.OpAMD64VPRORVDMasked128load,
+ ssa.OpAMD64VPRORVDMasked256,
+ ssa.OpAMD64VPRORVDMasked256load,
+ ssa.OpAMD64VPRORVDMasked512,
+ ssa.OpAMD64VPRORVDMasked512load,
+ ssa.OpAMD64VPRORVQMasked128,
+ ssa.OpAMD64VPRORVQMasked128load,
+ ssa.OpAMD64VPRORVQMasked256,
+ ssa.OpAMD64VPRORVQMasked256load,
+ ssa.OpAMD64VPRORVQMasked512,
+ ssa.OpAMD64VPRORVQMasked512load,
+ ssa.OpAMD64VPMOVSWBMasked128_128,
+ ssa.OpAMD64VPMOVSWBMasked128_256,
+ ssa.OpAMD64VPMOVSWBMasked256,
+ ssa.OpAMD64VPMOVSDBMasked128_128,
+ ssa.OpAMD64VPMOVSDBMasked128_256,
+ ssa.OpAMD64VPMOVSDBMasked128_512,
+ ssa.OpAMD64VPMOVSQBMasked128_128,
+ ssa.OpAMD64VPMOVSQBMasked128_256,
+ ssa.OpAMD64VPMOVSQBMasked128_512,
+ ssa.OpAMD64VPACKSSDWMasked256,
+ ssa.OpAMD64VPACKSSDWMasked256load,
+ ssa.OpAMD64VPACKSSDWMasked512,
+ ssa.OpAMD64VPACKSSDWMasked512load,
+ ssa.OpAMD64VPACKSSDWMasked128,
+ ssa.OpAMD64VPACKSSDWMasked128load,
+ ssa.OpAMD64VPMOVSDWMasked128_128,
+ ssa.OpAMD64VPMOVSDWMasked128_256,
+ ssa.OpAMD64VPMOVSDWMasked256,
+ ssa.OpAMD64VPMOVSQWMasked128_128,
+ ssa.OpAMD64VPMOVSQWMasked128_256,
+ ssa.OpAMD64VPMOVSQWMasked128_512,
+ ssa.OpAMD64VPMOVSQDMasked128_128,
+ ssa.OpAMD64VPMOVSQDMasked128_256,
+ ssa.OpAMD64VPMOVSQDMasked256,
+ ssa.OpAMD64VPMOVUSWBMasked128_128,
+ ssa.OpAMD64VPMOVUSWBMasked128_256,
+ ssa.OpAMD64VPMOVUSWBMasked256,
+ ssa.OpAMD64VPMOVUSDBMasked128_128,
+ ssa.OpAMD64VPMOVUSDBMasked128_256,
+ ssa.OpAMD64VPMOVUSDBMasked128_512,
+ ssa.OpAMD64VPMOVUSQBMasked128_128,
+ ssa.OpAMD64VPMOVUSQBMasked128_256,
+ ssa.OpAMD64VPMOVUSQBMasked128_512,
+ ssa.OpAMD64VPACKUSDWMasked256,
+ ssa.OpAMD64VPACKUSDWMasked256load,
+ ssa.OpAMD64VPACKUSDWMasked512,
+ ssa.OpAMD64VPACKUSDWMasked512load,
+ ssa.OpAMD64VPACKUSDWMasked128,
+ ssa.OpAMD64VPACKUSDWMasked128load,
+ ssa.OpAMD64VPMOVUSDWMasked128_128,
+ ssa.OpAMD64VPMOVUSDWMasked128_256,
+ ssa.OpAMD64VPMOVUSDWMasked256,
+ ssa.OpAMD64VPMOVUSQWMasked128_128,
+ ssa.OpAMD64VPMOVUSQWMasked128_256,
+ ssa.OpAMD64VPMOVUSQWMasked128_512,
+ ssa.OpAMD64VPMOVUSQDMasked128_128,
+ ssa.OpAMD64VPMOVUSQDMasked128_256,
+ ssa.OpAMD64VPMOVUSQDMasked256,
+ ssa.OpAMD64VSCALEFPSMasked128,
+ ssa.OpAMD64VSCALEFPSMasked128load,
+ ssa.OpAMD64VSCALEFPSMasked256,
+ ssa.OpAMD64VSCALEFPSMasked256load,
+ ssa.OpAMD64VSCALEFPSMasked512,
+ ssa.OpAMD64VSCALEFPSMasked512load,
+ ssa.OpAMD64VSCALEFPDMasked128,
+ ssa.OpAMD64VSCALEFPDMasked128load,
+ ssa.OpAMD64VSCALEFPDMasked256,
+ ssa.OpAMD64VSCALEFPDMasked256load,
+ ssa.OpAMD64VSCALEFPDMasked512,
+ ssa.OpAMD64VSCALEFPDMasked512load,
+ ssa.OpAMD64VPSHLDWMasked128,
+ ssa.OpAMD64VPSHLDWMasked128load,
+ ssa.OpAMD64VPSHLDWMasked256,
+ ssa.OpAMD64VPSHLDWMasked256load,
+ ssa.OpAMD64VPSHLDWMasked512,
+ ssa.OpAMD64VPSHLDDMasked128,
+ ssa.OpAMD64VPSHLDDMasked128load,
+ ssa.OpAMD64VPSHLDDMasked256,
+ ssa.OpAMD64VPSHLDDMasked256load,
+ ssa.OpAMD64VPSHLDDMasked512,
+ ssa.OpAMD64VPSHLDDMasked512load,
+ ssa.OpAMD64VPSHLDQMasked128,
+ ssa.OpAMD64VPSHLDQMasked128load,
+ ssa.OpAMD64VPSHLDQMasked256,
+ ssa.OpAMD64VPSHLDQMasked256load,
+ ssa.OpAMD64VPSHLDQMasked512,
+ ssa.OpAMD64VPSHLDQMasked512load,
+ ssa.OpAMD64VPSLLWMasked128,
+ ssa.OpAMD64VPSLLWMasked256,
+ ssa.OpAMD64VPSLLWMasked512,
+ ssa.OpAMD64VPSLLDMasked128,
+ ssa.OpAMD64VPSLLDMasked256,
+ ssa.OpAMD64VPSLLDMasked512,
+ ssa.OpAMD64VPSLLQMasked128,
+ ssa.OpAMD64VPSLLQMasked256,
+ ssa.OpAMD64VPSLLQMasked512,
+ ssa.OpAMD64VPSHRDWMasked128,
+ ssa.OpAMD64VPSHRDWMasked128load,
+ ssa.OpAMD64VPSHRDWMasked256,
+ ssa.OpAMD64VPSHRDWMasked256load,
+ ssa.OpAMD64VPSHRDWMasked512,
+ ssa.OpAMD64VPSHRDDMasked128,
+ ssa.OpAMD64VPSHRDDMasked128load,
+ ssa.OpAMD64VPSHRDDMasked256,
+ ssa.OpAMD64VPSHRDDMasked256load,
+ ssa.OpAMD64VPSHRDDMasked512,
+ ssa.OpAMD64VPSHRDDMasked512load,
+ ssa.OpAMD64VPSHRDQMasked128,
+ ssa.OpAMD64VPSHRDQMasked128load,
+ ssa.OpAMD64VPSHRDQMasked256,
+ ssa.OpAMD64VPSHRDQMasked256load,
+ ssa.OpAMD64VPSHRDQMasked512,
+ ssa.OpAMD64VPSHRDQMasked512load,
+ ssa.OpAMD64VPSRAWMasked128,
+ ssa.OpAMD64VPSRAWMasked256,
+ ssa.OpAMD64VPSRAWMasked512,
+ ssa.OpAMD64VPSRADMasked128,
+ ssa.OpAMD64VPSRADMasked256,
+ ssa.OpAMD64VPSRADMasked512,
+ ssa.OpAMD64VPSRAQMasked128,
+ ssa.OpAMD64VPSRAQMasked256,
+ ssa.OpAMD64VPSRAQMasked512,
+ ssa.OpAMD64VPSRLWMasked128,
+ ssa.OpAMD64VPSRLWMasked256,
+ ssa.OpAMD64VPSRLWMasked512,
+ ssa.OpAMD64VPSRLDMasked128,
+ ssa.OpAMD64VPSRLDMasked256,
+ ssa.OpAMD64VPSRLDMasked512,
+ ssa.OpAMD64VPSRLQMasked128,
+ ssa.OpAMD64VPSRLQMasked256,
+ ssa.OpAMD64VPSRLQMasked512,
+ ssa.OpAMD64VPSHLDVWMasked128,
+ ssa.OpAMD64VPSHLDVWMasked128load,
+ ssa.OpAMD64VPSHLDVWMasked256,
+ ssa.OpAMD64VPSHLDVWMasked256load,
+ ssa.OpAMD64VPSHLDVWMasked512,
+ ssa.OpAMD64VPSHLDVDMasked128,
+ ssa.OpAMD64VPSHLDVDMasked128load,
+ ssa.OpAMD64VPSHLDVDMasked256,
+ ssa.OpAMD64VPSHLDVDMasked256load,
+ ssa.OpAMD64VPSHLDVDMasked512,
+ ssa.OpAMD64VPSHLDVDMasked512load,
+ ssa.OpAMD64VPSHLDVQMasked128,
+ ssa.OpAMD64VPSHLDVQMasked128load,
+ ssa.OpAMD64VPSHLDVQMasked256,
+ ssa.OpAMD64VPSHLDVQMasked256load,
+ ssa.OpAMD64VPSHLDVQMasked512,
+ ssa.OpAMD64VPSHLDVQMasked512load,
+ ssa.OpAMD64VPSLLVWMasked128,
+ ssa.OpAMD64VPSLLVWMasked128load,
+ ssa.OpAMD64VPSLLVWMasked256,
+ ssa.OpAMD64VPSLLVWMasked256load,
+ ssa.OpAMD64VPSLLVWMasked512,
+ ssa.OpAMD64VPSLLVDMasked128,
+ ssa.OpAMD64VPSLLVDMasked128load,
+ ssa.OpAMD64VPSLLVDMasked256,
+ ssa.OpAMD64VPSLLVDMasked256load,
+ ssa.OpAMD64VPSLLVDMasked512,
+ ssa.OpAMD64VPSLLVDMasked512load,
+ ssa.OpAMD64VPSLLVQMasked128,
+ ssa.OpAMD64VPSLLVQMasked128load,
+ ssa.OpAMD64VPSLLVQMasked256,
+ ssa.OpAMD64VPSLLVQMasked256load,
+ ssa.OpAMD64VPSLLVQMasked512,
+ ssa.OpAMD64VPSLLVQMasked512load,
+ ssa.OpAMD64VPSHRDVWMasked128,
+ ssa.OpAMD64VPSHRDVWMasked128load,
+ ssa.OpAMD64VPSHRDVWMasked256,
+ ssa.OpAMD64VPSHRDVWMasked256load,
+ ssa.OpAMD64VPSHRDVWMasked512,
+ ssa.OpAMD64VPSHRDVDMasked128,
+ ssa.OpAMD64VPSHRDVDMasked128load,
+ ssa.OpAMD64VPSHRDVDMasked256,
+ ssa.OpAMD64VPSHRDVDMasked256load,
+ ssa.OpAMD64VPSHRDVDMasked512,
+ ssa.OpAMD64VPSHRDVDMasked512load,
+ ssa.OpAMD64VPSHRDVQMasked128,
+ ssa.OpAMD64VPSHRDVQMasked128load,
+ ssa.OpAMD64VPSHRDVQMasked256,
+ ssa.OpAMD64VPSHRDVQMasked256load,
+ ssa.OpAMD64VPSHRDVQMasked512,
+ ssa.OpAMD64VPSHRDVQMasked512load,
+ ssa.OpAMD64VPSRAVWMasked128,
+ ssa.OpAMD64VPSRAVWMasked128load,
+ ssa.OpAMD64VPSRAVWMasked256,
+ ssa.OpAMD64VPSRAVWMasked256load,
+ ssa.OpAMD64VPSRAVWMasked512,
+ ssa.OpAMD64VPSRAVDMasked128,
+ ssa.OpAMD64VPSRAVDMasked128load,
+ ssa.OpAMD64VPSRAVDMasked256,
+ ssa.OpAMD64VPSRAVDMasked256load,
+ ssa.OpAMD64VPSRAVDMasked512,
+ ssa.OpAMD64VPSRAVDMasked512load,
+ ssa.OpAMD64VPSRAVQMasked128,
+ ssa.OpAMD64VPSRAVQMasked128load,
+ ssa.OpAMD64VPSRAVQMasked256,
+ ssa.OpAMD64VPSRAVQMasked256load,
+ ssa.OpAMD64VPSRAVQMasked512,
+ ssa.OpAMD64VPSRAVQMasked512load,
+ ssa.OpAMD64VPSRLVWMasked128,
+ ssa.OpAMD64VPSRLVWMasked128load,
+ ssa.OpAMD64VPSRLVWMasked256,
+ ssa.OpAMD64VPSRLVWMasked256load,
+ ssa.OpAMD64VPSRLVWMasked512,
+ ssa.OpAMD64VPSRLVDMasked128,
+ ssa.OpAMD64VPSRLVDMasked128load,
+ ssa.OpAMD64VPSRLVDMasked256,
+ ssa.OpAMD64VPSRLVDMasked256load,
+ ssa.OpAMD64VPSRLVDMasked512,
+ ssa.OpAMD64VPSRLVDMasked512load,
+ ssa.OpAMD64VPSRLVQMasked128,
+ ssa.OpAMD64VPSRLVQMasked128load,
+ ssa.OpAMD64VPSRLVQMasked256,
+ ssa.OpAMD64VPSRLVQMasked256load,
+ ssa.OpAMD64VPSRLVQMasked512,
+ ssa.OpAMD64VPSRLVQMasked512load,
+ ssa.OpAMD64VSQRTPSMasked128,
+ ssa.OpAMD64VSQRTPSMasked128load,
+ ssa.OpAMD64VSQRTPSMasked256,
+ ssa.OpAMD64VSQRTPSMasked256load,
+ ssa.OpAMD64VSQRTPSMasked512,
+ ssa.OpAMD64VSQRTPSMasked512load,
+ ssa.OpAMD64VSQRTPDMasked128,
+ ssa.OpAMD64VSQRTPDMasked128load,
+ ssa.OpAMD64VSQRTPDMasked256,
+ ssa.OpAMD64VSQRTPDMasked256load,
+ ssa.OpAMD64VSQRTPDMasked512,
+ ssa.OpAMD64VSQRTPDMasked512load,
+ ssa.OpAMD64VSUBPSMasked128,
+ ssa.OpAMD64VSUBPSMasked128load,
+ ssa.OpAMD64VSUBPSMasked256,
+ ssa.OpAMD64VSUBPSMasked256load,
+ ssa.OpAMD64VSUBPSMasked512,
+ ssa.OpAMD64VSUBPSMasked512load,
+ ssa.OpAMD64VSUBPDMasked128,
+ ssa.OpAMD64VSUBPDMasked128load,
+ ssa.OpAMD64VSUBPDMasked256,
+ ssa.OpAMD64VSUBPDMasked256load,
+ ssa.OpAMD64VSUBPDMasked512,
+ ssa.OpAMD64VSUBPDMasked512load,
+ ssa.OpAMD64VPSUBBMasked128,
+ ssa.OpAMD64VPSUBBMasked128load,
+ ssa.OpAMD64VPSUBBMasked256,
+ ssa.OpAMD64VPSUBBMasked256load,
+ ssa.OpAMD64VPSUBBMasked512,
+ ssa.OpAMD64VPSUBWMasked128,
+ ssa.OpAMD64VPSUBWMasked128load,
+ ssa.OpAMD64VPSUBWMasked256,
+ ssa.OpAMD64VPSUBWMasked256load,
+ ssa.OpAMD64VPSUBWMasked512,
+ ssa.OpAMD64VPSUBDMasked128,
+ ssa.OpAMD64VPSUBDMasked128load,
+ ssa.OpAMD64VPSUBDMasked256,
+ ssa.OpAMD64VPSUBDMasked256load,
+ ssa.OpAMD64VPSUBDMasked512,
+ ssa.OpAMD64VPSUBDMasked512load,
+ ssa.OpAMD64VPSUBQMasked128,
+ ssa.OpAMD64VPSUBQMasked128load,
+ ssa.OpAMD64VPSUBQMasked256,
+ ssa.OpAMD64VPSUBQMasked256load,
+ ssa.OpAMD64VPSUBQMasked512,
+ ssa.OpAMD64VPSUBQMasked512load,
+ ssa.OpAMD64VPSUBSBMasked128,
+ ssa.OpAMD64VPSUBSBMasked128load,
+ ssa.OpAMD64VPSUBSBMasked256,
+ ssa.OpAMD64VPSUBSBMasked256load,
+ ssa.OpAMD64VPSUBSBMasked512,
+ ssa.OpAMD64VPSUBSWMasked128,
+ ssa.OpAMD64VPSUBSWMasked128load,
+ ssa.OpAMD64VPSUBSWMasked256,
+ ssa.OpAMD64VPSUBSWMasked256load,
+ ssa.OpAMD64VPSUBSWMasked512,
+ ssa.OpAMD64VPSUBUSBMasked128,
+ ssa.OpAMD64VPSUBUSBMasked128load,
+ ssa.OpAMD64VPSUBUSBMasked256,
+ ssa.OpAMD64VPSUBUSBMasked256load,
+ ssa.OpAMD64VPSUBUSBMasked512,
+ ssa.OpAMD64VPSUBUSWMasked128,
+ ssa.OpAMD64VPSUBUSWMasked128load,
+ ssa.OpAMD64VPSUBUSWMasked256,
+ ssa.OpAMD64VPSUBUSWMasked256load,
+ ssa.OpAMD64VPSUBUSWMasked512,
+ ssa.OpAMD64VPMOVWBMasked128_128,
+ ssa.OpAMD64VPMOVWBMasked128_256,
+ ssa.OpAMD64VPMOVWBMasked256,
+ ssa.OpAMD64VPMOVDBMasked128_128,
+ ssa.OpAMD64VPMOVDBMasked128_256,
+ ssa.OpAMD64VPMOVDBMasked128_512,
+ ssa.OpAMD64VPMOVQBMasked128_128,
+ ssa.OpAMD64VPMOVQBMasked128_256,
+ ssa.OpAMD64VPMOVQBMasked128_512,
+ ssa.OpAMD64VPMOVDWMasked128_128,
+ ssa.OpAMD64VPMOVDWMasked128_256,
+ ssa.OpAMD64VPMOVDWMasked256,
+ ssa.OpAMD64VPMOVQWMasked128_128,
+ ssa.OpAMD64VPMOVQWMasked128_256,
+ ssa.OpAMD64VPMOVQWMasked128_512,
+ ssa.OpAMD64VPMOVQDMasked128_128,
+ ssa.OpAMD64VPMOVQDMasked128_256,
+ ssa.OpAMD64VPMOVQDMasked256,
+ ssa.OpAMD64VPXORDMasked128,
+ ssa.OpAMD64VPXORDMasked128load,
+ ssa.OpAMD64VPXORDMasked256,
+ ssa.OpAMD64VPXORDMasked256load,
+ ssa.OpAMD64VPXORDMasked512,
+ ssa.OpAMD64VPXORDMasked512load,
+ ssa.OpAMD64VPXORQMasked128,
+ ssa.OpAMD64VPXORQMasked128load,
+ ssa.OpAMD64VPXORQMasked256,
+ ssa.OpAMD64VPXORQMasked256load,
+ ssa.OpAMD64VPXORQMasked512,
+ ssa.OpAMD64VPXORQMasked512load,
+ ssa.OpAMD64VPBROADCASTQMasked128,
+ ssa.OpAMD64VPBROADCASTQMasked128load,
+ ssa.OpAMD64VBROADCASTSSMasked128,
+ ssa.OpAMD64VBROADCASTSSMasked128load,
+ ssa.OpAMD64VBROADCASTSDMasked256,
+ ssa.OpAMD64VBROADCASTSDMasked256load,
+ ssa.OpAMD64VPBROADCASTDMasked128,
+ ssa.OpAMD64VPBROADCASTDMasked128load,
+ ssa.OpAMD64VPBROADCASTQMasked256,
+ ssa.OpAMD64VPBROADCASTQMasked256load,
+ ssa.OpAMD64VBROADCASTSSMasked256,
+ ssa.OpAMD64VBROADCASTSSMasked256load,
+ ssa.OpAMD64VBROADCASTSDMasked512,
+ ssa.OpAMD64VBROADCASTSDMasked512load,
+ ssa.OpAMD64VPBROADCASTWMasked128,
+ ssa.OpAMD64VPBROADCASTDMasked256,
+ ssa.OpAMD64VPBROADCASTDMasked256load,
+ ssa.OpAMD64VPBROADCASTQMasked512,
+ ssa.OpAMD64VPBROADCASTQMasked512load,
+ ssa.OpAMD64VBROADCASTSSMasked512,
+ ssa.OpAMD64VBROADCASTSSMasked512load,
+ ssa.OpAMD64VPBROADCASTBMasked128,
+ ssa.OpAMD64VPBROADCASTBMasked128load,
+ ssa.OpAMD64VPBROADCASTWMasked256,
+ ssa.OpAMD64VPBROADCASTDMasked512,
+ ssa.OpAMD64VPBROADCASTDMasked512load,
+ ssa.OpAMD64VPBROADCASTBMasked256,
+ ssa.OpAMD64VPBROADCASTBMasked256load,
+ ssa.OpAMD64VPBROADCASTWMasked512,
+ ssa.OpAMD64VPBROADCASTBMasked512,
+ ssa.OpAMD64VPBROADCASTBMasked512load,
+ ssa.OpAMD64VMOVDQU8Masked128,
+ ssa.OpAMD64VMOVDQU8Masked256,
+ ssa.OpAMD64VMOVDQU8Masked512,
+ ssa.OpAMD64VMOVDQU16Masked128,
+ ssa.OpAMD64VMOVDQU16Masked256,
+ ssa.OpAMD64VMOVDQU16Masked512,
+ ssa.OpAMD64VMOVDQU32Masked128,
+ ssa.OpAMD64VMOVDQU32Masked256,
+ ssa.OpAMD64VMOVDQU32Masked512,
+ ssa.OpAMD64VMOVDQU64Masked128,
+ ssa.OpAMD64VMOVDQU64Masked256,
+ ssa.OpAMD64VMOVDQU64Masked512,
+ ssa.OpAMD64VPSHUFDMasked256,
+ ssa.OpAMD64VPSHUFDMasked256load,
+ ssa.OpAMD64VPSHUFDMasked512,
+ ssa.OpAMD64VPSHUFDMasked512load,
+ ssa.OpAMD64VPSHUFHWMasked256,
+ ssa.OpAMD64VPSHUFHWMasked256load,
+ ssa.OpAMD64VPSHUFHWMasked512,
+ ssa.OpAMD64VPSHUFHWMasked128,
+ ssa.OpAMD64VPSHUFHWMasked128load,
+ ssa.OpAMD64VPSHUFLWMasked256,
+ ssa.OpAMD64VPSHUFLWMasked256load,
+ ssa.OpAMD64VPSHUFLWMasked512,
+ ssa.OpAMD64VPSHUFLWMasked128,
+ ssa.OpAMD64VPSHUFLWMasked128load,
+ ssa.OpAMD64VPSHUFDMasked128,
+ ssa.OpAMD64VPSHUFDMasked128load,
+ ssa.OpAMD64VPROLDMasked128,
+ ssa.OpAMD64VPROLDMasked128load,
+ ssa.OpAMD64VPROLDMasked256,
+ ssa.OpAMD64VPROLDMasked256load,
+ ssa.OpAMD64VPROLDMasked512,
+ ssa.OpAMD64VPROLDMasked512load,
+ ssa.OpAMD64VPROLQMasked128,
+ ssa.OpAMD64VPROLQMasked128load,
+ ssa.OpAMD64VPROLQMasked256,
+ ssa.OpAMD64VPROLQMasked256load,
+ ssa.OpAMD64VPROLQMasked512,
+ ssa.OpAMD64VPROLQMasked512load,
+ ssa.OpAMD64VPRORDMasked128,
+ ssa.OpAMD64VPRORDMasked128load,
+ ssa.OpAMD64VPRORDMasked256,
+ ssa.OpAMD64VPRORDMasked256load,
+ ssa.OpAMD64VPRORDMasked512,
+ ssa.OpAMD64VPRORDMasked512load,
+ ssa.OpAMD64VPRORQMasked128,
+ ssa.OpAMD64VPRORQMasked128load,
+ ssa.OpAMD64VPRORQMasked256,
+ ssa.OpAMD64VPRORQMasked256load,
+ ssa.OpAMD64VPRORQMasked512,
+ ssa.OpAMD64VPRORQMasked512load,
+ ssa.OpAMD64VPSLLWMasked128const,
+ ssa.OpAMD64VPSLLWMasked128constload,
+ ssa.OpAMD64VPSLLWMasked256const,
+ ssa.OpAMD64VPSLLWMasked256constload,
+ ssa.OpAMD64VPSLLWMasked512const,
+ ssa.OpAMD64VPSLLDMasked128const,
+ ssa.OpAMD64VPSLLDMasked128constload,
+ ssa.OpAMD64VPSLLDMasked256const,
+ ssa.OpAMD64VPSLLDMasked256constload,
+ ssa.OpAMD64VPSLLDMasked512const,
+ ssa.OpAMD64VPSLLDMasked512constload,
+ ssa.OpAMD64VPSLLQMasked128const,
+ ssa.OpAMD64VPSLLQMasked128constload,
+ ssa.OpAMD64VPSLLQMasked256const,
+ ssa.OpAMD64VPSLLQMasked256constload,
+ ssa.OpAMD64VPSLLQMasked512const,
+ ssa.OpAMD64VPSLLQMasked512constload,
+ ssa.OpAMD64VPSRAWMasked128const,
+ ssa.OpAMD64VPSRAWMasked128constload,
+ ssa.OpAMD64VPSRAWMasked256const,
+ ssa.OpAMD64VPSRAWMasked256constload,
+ ssa.OpAMD64VPSRAWMasked512const,
+ ssa.OpAMD64VPSRADMasked128const,
+ ssa.OpAMD64VPSRADMasked128constload,
+ ssa.OpAMD64VPSRADMasked256const,
+ ssa.OpAMD64VPSRADMasked256constload,
+ ssa.OpAMD64VPSRADMasked512const,
+ ssa.OpAMD64VPSRADMasked512constload,
+ ssa.OpAMD64VPSRAQMasked128const,
+ ssa.OpAMD64VPSRAQMasked128constload,
+ ssa.OpAMD64VPSRAQMasked256const,
+ ssa.OpAMD64VPSRAQMasked256constload,
+ ssa.OpAMD64VPSRAQMasked512const,
+ ssa.OpAMD64VPSRAQMasked512constload,
+ ssa.OpAMD64VPSRLWMasked128const,
+ ssa.OpAMD64VPSRLWMasked128constload,
+ ssa.OpAMD64VPSRLWMasked256const,
+ ssa.OpAMD64VPSRLWMasked256constload,
+ ssa.OpAMD64VPSRLWMasked512const,
+ ssa.OpAMD64VPSRLDMasked128const,
+ ssa.OpAMD64VPSRLDMasked128constload,
+ ssa.OpAMD64VPSRLDMasked256const,
+ ssa.OpAMD64VPSRLDMasked256constload,
+ ssa.OpAMD64VPSRLDMasked512const,
+ ssa.OpAMD64VPSRLDMasked512constload,
+ ssa.OpAMD64VPSRLQMasked128const,
+ ssa.OpAMD64VPSRLQMasked128constload,
+ ssa.OpAMD64VPSRLQMasked256const,
+ ssa.OpAMD64VPSRLQMasked256constload,
+ ssa.OpAMD64VPSRLQMasked512const,
+ ssa.OpAMD64VPSRLQMasked512constload:
+ x86.ParseSuffix(p, "Z")
+ }
+
+ // Ensure p is marked as used (may not be used in all generated code paths)
+ _ = p
+ return true
+}
diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go
index 6331501dc92fdb..cd5a2be7057c59 100644
--- a/src/cmd/compile/internal/amd64/ssa.go
+++ b/src/cmd/compile/internal/amd64/ssa.go
@@ -43,11 +43,31 @@ func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
}
}
-// loadByType returns the load instruction of the given type.
-func loadByType(t *types.Type) obj.As {
- // Avoid partial register write
- if !t.IsFloat() {
- switch t.Size() {
+func isGPReg(r int16) bool {
+ return x86.REG_AL <= r && r <= x86.REG_R15
+}
+
+func isFPReg(r int16) bool {
+ return x86.REG_X0 <= r && r <= x86.REG_Z31
+}
+
+func isKReg(r int16) bool {
+ return x86.REG_K0 <= r && r <= x86.REG_K7
+}
+
+func isLowFPReg(r int16) bool {
+ return x86.REG_X0 <= r && r <= x86.REG_X15
+}
+
+func isHighFPReg(r int16) bool {
+ return x86.REG_X16 <= r && r <= x86.REG_X31 || x86.REG_Y16 <= r && r <= x86.REG_Y31 || x86.REG_Z16 <= r && r <= x86.REG_Z31
+}
+
+// loadByRegWidth returns the load instruction of the given register of a given width.
+func loadByRegWidth(r int16, width int64) obj.As {
+ // Avoid partial register write for GPR
+ if !isFPReg(r) && !isKReg(r) {
+ switch width {
case 1:
return x86.AMOVBLZX
case 2:
@@ -55,61 +75,95 @@ func loadByType(t *types.Type) obj.As {
}
}
// Otherwise, there's no difference between load and store opcodes.
- return storeByType(t)
+ return storeByRegWidth(r, width)
}
-// storeByType returns the store instruction of the given type.
-func storeByType(t *types.Type) obj.As {
- width := t.Size()
- if t.IsFloat() {
+// storeByRegWidth returns the store instruction of the given register of a given width.
+// It's also used for loading const to a reg.
+func storeByRegWidth(r int16, width int64) obj.As {
+ if isHighFPReg(r) {
+ // High registers require AVX512 instruction
+ return x86.AVMOVDQU64
+ }
+ if isFPReg(r) {
switch width {
case 4:
return x86.AMOVSS
case 8:
return x86.AMOVSD
- }
- } else {
- switch width {
- case 1:
- return x86.AMOVB
- case 2:
- return x86.AMOVW
- case 4:
- return x86.AMOVL
- case 8:
- return x86.AMOVQ
case 16:
+ // int128s are in SSE registers
return x86.AMOVUPS
+ case 32:
+ return x86.AVMOVDQU
+ case 64:
+ return x86.AVMOVDQU64
}
}
- panic(fmt.Sprintf("bad store type %v", t))
+ if isKReg(r) {
+ return x86.AKMOVQ
+ }
+ // gp
+ switch width {
+ case 1:
+ return x86.AMOVB
+ case 2:
+ return x86.AMOVW
+ case 4:
+ return x86.AMOVL
+ case 8:
+ return x86.AMOVQ
+ }
+ panic(fmt.Sprintf("bad store reg=%v, width=%d", r, width))
}
-// moveByType returns the reg->reg move instruction of the given type.
-func moveByType(t *types.Type) obj.As {
- if t.IsFloat() {
+// moveByRegsWidth returns the reg->reg move instruction of the given dest/src registers of a given width.
+func moveByRegsWidth(dest, src int16, width int64) obj.As {
+ // fp -> fp
+ if isFPReg(dest) && isFPReg(src) {
+ if isHighFPReg(src) || isHighFPReg(dest) {
+ // High registers require AVX512 instruction
+ return x86.AVMOVDQU64
+ }
// Moving the whole sse2 register is faster
// than moving just the correct low portion of it.
// There is no xmm->xmm move with 1 byte opcode,
// so use movups, which has 2 byte opcode.
- return x86.AMOVUPS
- } else {
- switch t.Size() {
- case 1:
- // Avoids partial register write
- return x86.AMOVL
- case 2:
- return x86.AMOVL
- case 4:
- return x86.AMOVL
- case 8:
- return x86.AMOVQ
- case 16:
- return x86.AMOVUPS // int128s are in SSE registers
- default:
- panic(fmt.Sprintf("bad int register width %d:%v", t.Size(), t))
+ if width <= 16 {
+ return x86.AMOVUPS
+ }
+ if width <= 32 {
+ return x86.AVMOVDQU
+ }
+ return x86.AVMOVDQU64
+ }
+ // k -> gp, gp -> k, k -> k
+ if isKReg(dest) || isKReg(src) {
+ if isFPReg(dest) || isFPReg(src) {
+ panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
}
+ return x86.AKMOVQ
}
+ // gp -> fp, fp -> gp, gp -> gp
+ switch width {
+ case 1:
+ // Avoids partial register write
+ return x86.AMOVL
+ case 2:
+ return x86.AMOVL
+ case 4:
+ return x86.AMOVL
+ case 8:
+ return x86.AMOVQ
+ case 16:
+ // int128s are in SSE registers
+ return x86.AMOVUPS
+ case 32:
+ return x86.AVMOVDQU
+ case 64:
+ return x86.AVMOVDQU64
+ }
+ panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
}
// opregreg emits instructions for
@@ -259,15 +313,6 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg()
p.From.Reg = x86.REG_X15
- case ssa.OpAMD64SHRDQ, ssa.OpAMD64SHLDQ:
- p := s.Prog(v.Op.Asm())
- lo, hi, bits := v.Args[0].Reg(), v.Args[1].Reg(), v.Args[2].Reg()
- p.From.Type = obj.TYPE_REG
- p.From.Reg = bits
- p.To.Type = obj.TYPE_REG
- p.To.Reg = lo
- p.AddRestSourceReg(hi)
-
case ssa.OpAMD64BLSIQ, ssa.OpAMD64BLSIL,
ssa.OpAMD64BLSMSKQ, ssa.OpAMD64BLSMSKL,
ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL:
@@ -605,7 +650,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
// But this requires a way for regalloc to know that SRC might be
// clobbered by this instruction.
t := v.RegTmp()
- opregreg(s, moveByType(v.Type), t, v.Args[1].Reg())
+ opregreg(s, moveByRegsWidth(t, v.Args[1].Reg(), v.Type.Size()), t, v.Args[1].Reg())
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
@@ -650,11 +695,33 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpAMD64SUBQconst, ssa.OpAMD64SUBLconst,
ssa.OpAMD64ANDLconst,
ssa.OpAMD64ORQconst, ssa.OpAMD64ORLconst,
- ssa.OpAMD64XORQconst, ssa.OpAMD64XORLconst,
- ssa.OpAMD64SHLQconst, ssa.OpAMD64SHLLconst,
+ ssa.OpAMD64XORQconst, ssa.OpAMD64XORLconst:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = v.AuxInt
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
+
+ case ssa.OpAMD64SHLQconst, ssa.OpAMD64SHLLconst,
ssa.OpAMD64SHRQconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SHRWconst, ssa.OpAMD64SHRBconst,
ssa.OpAMD64SARQconst, ssa.OpAMD64SARLconst, ssa.OpAMD64SARWconst, ssa.OpAMD64SARBconst,
ssa.OpAMD64ROLQconst, ssa.OpAMD64ROLLconst, ssa.OpAMD64ROLWconst, ssa.OpAMD64ROLBconst:
+ var maxShift int64
+ switch v.Op {
+ case ssa.OpAMD64SHLQconst, ssa.OpAMD64SHRQconst, ssa.OpAMD64SARQconst, ssa.OpAMD64ROLQconst:
+ maxShift = 63
+ case ssa.OpAMD64SHLLconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SARLconst, ssa.OpAMD64ROLLconst:
+ maxShift = 31
+ case ssa.OpAMD64SHRWconst, ssa.OpAMD64SARWconst, ssa.OpAMD64ROLWconst:
+ maxShift = 15
+ case ssa.OpAMD64SHRBconst, ssa.OpAMD64SARBconst, ssa.OpAMD64ROLBconst:
+ maxShift = 7
+ default:
+ panic("unreachable")
+ }
+ if v.AuxInt < 0 || v.AuxInt > maxShift {
+ v.Fatalf("shift amount out of range [0,%d]: %d", maxShift, v.AuxInt)
+ }
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_CONST
p.From.Offset = v.AuxInt
@@ -777,9 +844,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.From.Offset = v.AuxInt
p.To.Type = obj.TYPE_REG
p.To.Reg = x
+
case ssa.OpAMD64MOVSSconst, ssa.OpAMD64MOVSDconst:
x := v.Reg()
- p := s.Prog(v.Op.Asm())
+ if !isFPReg(x) && v.AuxInt == 0 && v.Aux == nil {
+ opregreg(s, x86.AXORL, x, x)
+ break
+ }
+ p := s.Prog(storeByRegWidth(x, v.Type.Size()))
p.From.Type = obj.TYPE_FCONST
p.From.Val = math.Float64frombits(uint64(v.AuxInt))
p.To.Type = obj.TYPE_REG
@@ -1174,29 +1246,50 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
if v.Type.IsMemory() {
return
}
- x := v.Args[0].Reg()
+ arg := v.Args[0]
+ x := arg.Reg()
y := v.Reg()
+ if v.Type.IsSIMD() {
+ x = simdOrMaskReg(arg)
+ y = simdOrMaskReg(v)
+ }
if x != y {
- opregreg(s, moveByType(v.Type), y, x)
+ width := v.Type.Size()
+ if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg, 3) {
+ // The source was naturally zext-ed from 32 to 64 bits,
+ // but we are asked to do a full 64-bit copy.
+ // Save the REX prefix byte in I-CACHE by using a 32-bit move,
+ // since it zeroes the upper 32 bits anyway.
+ width = 4
+ }
+ opregreg(s, moveByRegsWidth(y, x, width), y, x)
}
case ssa.OpLoadReg:
if v.Type.IsFlags() {
v.Fatalf("load flags not implemented: %v", v.LongString())
return
}
- p := s.Prog(loadByType(v.Type))
+ r := v.Reg()
+ p := s.Prog(loadByRegWidth(r, v.Type.Size()))
ssagen.AddrAuto(&p.From, v.Args[0])
p.To.Type = obj.TYPE_REG
- p.To.Reg = v.Reg()
+ if v.Type.IsSIMD() {
+ r = simdOrMaskReg(v)
+ }
+ p.To.Reg = r
case ssa.OpStoreReg:
if v.Type.IsFlags() {
v.Fatalf("store flags not implemented: %v", v.LongString())
return
}
- p := s.Prog(storeByType(v.Type))
+ r := v.Args[0].Reg()
+ if v.Type.IsSIMD() {
+ r = simdOrMaskReg(v.Args[0])
+ }
+ p := s.Prog(storeByRegWidth(r, v.Type.Size()))
p.From.Type = obj.TYPE_REG
- p.From.Reg = v.Args[0].Reg()
+ p.From.Reg = r
ssagen.AddrAuto(&p.To, v)
case ssa.OpAMD64LoweredHasCPUFeature:
p := s.Prog(x86.AMOVBLZX)
@@ -1210,8 +1303,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
for _, ap := range v.Block.Func.RegArgs {
// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.
addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)
+ reg := ap.Reg
+ t := ap.Type
+ sz := t.Size()
+ if t.IsSIMD() {
+ reg = simdRegBySize(reg, sz)
+ }
s.FuncInfo().AddSpill(
- obj.RegSpill{Reg: ap.Reg, Addr: addr, Unspill: loadByType(ap.Type), Spill: storeByType(ap.Type)})
+ obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByRegWidth(reg, sz), Spill: storeByRegWidth(reg, sz)})
}
v.Block.Func.RegArgs = nil
ssagen.CheckArgReg(v)
@@ -1224,21 +1323,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
}
r := v.Reg()
getgFromTLS(s, r)
- case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail:
+ case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail, ssa.OpAMD64CALLtailinter:
if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
// zeroing X15 when entering ABIInternal from ABI0
- opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
+ zeroX15(s)
// set G register from TLS
getgFromTLS(s, x86.REG_R14)
}
- if v.Op == ssa.OpAMD64CALLtail {
+ if v.Op == ssa.OpAMD64CALLtail || v.Op == ssa.OpAMD64CALLtailinter {
s.TailCall(v)
break
}
s.Call(v)
if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
// zeroing X15 when entering ABIInternal from ABI0
- opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
+ zeroX15(s)
// set G register from TLS
getgFromTLS(s, x86.REG_R14)
}
@@ -1409,7 +1508,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
}
case ssa.OpAMD64LoweredRound32F, ssa.OpAMD64LoweredRound64F:
// input is already rounded
- case ssa.OpAMD64ROUNDSD:
+ case ssa.OpAMD64ROUNDSD, ssa.OpAMD64ROUNDSS:
p := s.Prog(v.Op.Asm())
val := v.AuxInt
// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc
@@ -1643,11 +1742,703 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.From.Offset = int64(x)
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
+
+ // SIMD ops
+ case ssa.OpAMD64VZEROUPPER, ssa.OpAMD64VZEROALL:
+ s.Prog(v.Op.Asm())
+
+ case ssa.OpAMD64Zero128: // no code emitted
+
+ case ssa.OpAMD64Zero256, ssa.OpAMD64Zero512:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v)
+ p.AddRestSourceReg(simdReg(v))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+
+ case ssa.OpAMD64VMOVSSf2v, ssa.OpAMD64VMOVSDf2v:
+ // These are for initializing the least 32/64 bits of a SIMD register from a "float".
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[0].Reg()
+ p.AddRestSourceReg(x86.REG_X15)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+
+ case ssa.OpAMD64VMOVQload, ssa.OpAMD64VMOVDload,
+ ssa.OpAMD64VMOVSSload, ssa.OpAMD64VMOVSDload:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+
+ case ssa.OpAMD64VMOVSSconst, ssa.OpAMD64VMOVSDconst:
+ // for loading constants directly into SIMD registers
+ x := simdReg(v)
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_FCONST
+ p.From.Val = math.Float64frombits(uint64(v.AuxInt))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = x
+
+ case ssa.OpAMD64VMOVD, ssa.OpAMD64VMOVQ:
+ // These are for initializing the least 32/64 bits of a SIMD register from an "int".
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[0].Reg()
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+
+ case ssa.OpAMD64VMOVDQUload128, ssa.OpAMD64VMOVDQUload256, ssa.OpAMD64VMOVDQUload512,
+ ssa.OpAMD64KMOVBload, ssa.OpAMD64KMOVWload, ssa.OpAMD64KMOVDload, ssa.OpAMD64KMOVQload:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdOrMaskReg(v)
+ case ssa.OpAMD64VMOVDQUstore128, ssa.OpAMD64VMOVDQUstore256, ssa.OpAMD64VMOVDQUstore512,
+ ssa.OpAMD64KMOVBstore, ssa.OpAMD64KMOVWstore, ssa.OpAMD64KMOVDstore, ssa.OpAMD64KMOVQstore:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdOrMaskReg(v.Args[1])
+ p.To.Type = obj.TYPE_MEM
+ p.To.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.To, v)
+
+ case ssa.OpAMD64VPMASK32load128, ssa.OpAMD64VPMASK64load128, ssa.OpAMD64VPMASK32load256, ssa.OpAMD64VPMASK64load256:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
+
+ case ssa.OpAMD64VPMASK32store128, ssa.OpAMD64VPMASK64store128, ssa.OpAMD64VPMASK32store256, ssa.OpAMD64VPMASK64store256:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.To.Type = obj.TYPE_MEM
+ p.To.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.To, v)
+ p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
+
+ case ssa.OpAMD64VPMASK64load512, ssa.OpAMD64VPMASK32load512, ssa.OpAMD64VPMASK16load512, ssa.OpAMD64VPMASK8load512:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
+ x86.ParseSuffix(p, "Z") // must be zero if not in mask
+
+ case ssa.OpAMD64KANDB, ssa.OpAMD64KANDW, ssa.OpAMD64KANDD, ssa.OpAMD64KANDQ,
+ ssa.OpAMD64KORB, ssa.OpAMD64KORW, ssa.OpAMD64KORD, ssa.OpAMD64KORQ,
+ ssa.OpAMD64KXORB, ssa.OpAMD64KXORW, ssa.OpAMD64KXORD, ssa.OpAMD64KXORQ,
+ ssa.OpAMD64KXNORB, ssa.OpAMD64KXNORW, ssa.OpAMD64KXNORD, ssa.OpAMD64KXNORQ: // XNOR == EQ
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[0].Reg()
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
+ p.AddRestSourceReg(v.Args[1].Reg()) // masking simd reg
+
+ case ssa.OpAMD64VPMASK64store512, ssa.OpAMD64VPMASK32store512, ssa.OpAMD64VPMASK16store512, ssa.OpAMD64VPMASK8store512:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.To.Type = obj.TYPE_MEM
+ p.To.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.To, v)
+ p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
+
+ case ssa.OpAMD64VPMOVMToVec8x16,
+ ssa.OpAMD64VPMOVMToVec8x32,
+ ssa.OpAMD64VPMOVMToVec8x64,
+ ssa.OpAMD64VPMOVMToVec16x8,
+ ssa.OpAMD64VPMOVMToVec16x16,
+ ssa.OpAMD64VPMOVMToVec16x32,
+ ssa.OpAMD64VPMOVMToVec32x4,
+ ssa.OpAMD64VPMOVMToVec32x8,
+ ssa.OpAMD64VPMOVMToVec32x16,
+ ssa.OpAMD64VPMOVMToVec64x2,
+ ssa.OpAMD64VPMOVMToVec64x4,
+ ssa.OpAMD64VPMOVMToVec64x8:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[0].Reg()
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+
+ case ssa.OpAMD64VPMOVVec8x16ToM,
+ ssa.OpAMD64VPMOVVec8x32ToM,
+ ssa.OpAMD64VPMOVVec8x64ToM,
+ ssa.OpAMD64VPMOVVec16x8ToM,
+ ssa.OpAMD64VPMOVVec16x16ToM,
+ ssa.OpAMD64VPMOVVec16x32ToM,
+ ssa.OpAMD64VPMOVVec32x4ToM,
+ ssa.OpAMD64VPMOVVec32x8ToM,
+ ssa.OpAMD64VPMOVVec32x16ToM,
+ ssa.OpAMD64VPMOVVec64x2ToM,
+ ssa.OpAMD64VPMOVVec64x4ToM,
+ ssa.OpAMD64VPMOVVec64x8ToM,
+ ssa.OpAMD64VPMOVMSKB128,
+ ssa.OpAMD64VPMOVMSKB256,
+ ssa.OpAMD64VMOVMSKPS128,
+ ssa.OpAMD64VMOVMSKPS256,
+ ssa.OpAMD64VMOVMSKPD128,
+ ssa.OpAMD64VMOVMSKPD256:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[0])
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
+
+ case ssa.OpAMD64KMOVQk, ssa.OpAMD64KMOVDk, ssa.OpAMD64KMOVWk, ssa.OpAMD64KMOVBk,
+ ssa.OpAMD64KMOVQi, ssa.OpAMD64KMOVDi, ssa.OpAMD64KMOVWi, ssa.OpAMD64KMOVBi:
+ // See also ssa.OpAMD64KMOVQload
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[0].Reg()
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
+ case ssa.OpAMD64VPTEST:
+ // Some instructions setting flags put their second operand into the destination reg.
+ // See also CMP[BWDQ].
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[0])
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v.Args[1])
+
default:
- v.Fatalf("genValue not implemented: %s", v.LongString())
+ if !ssaGenSIMDValue(s, v) {
+ v.Fatalf("genValue not implemented: %s", v.LongString())
+ }
}
}
+// zeroX15 zeroes the X15 register.
+func zeroX15(s *ssagen.State) {
+ opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
+}
+
+// Example instruction: VRSQRTPS X1, X1
+func simdV11(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[0])
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPSUBD X1, X2, X3
+func simdV21(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ // Vector registers operands follows a right-to-left order.
+ // e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
+ p.From.Reg = simdReg(v.Args[1])
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// This function is to accustomize the shifts.
+// The 2nd arg is an XMM, and this function merely checks that.
+// Example instruction: VPSLLQ Z1, X1, Z2
+func simdVfpv(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ // Vector registers operands follows a right-to-left order.
+ // e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
+ p.From.Reg = v.Args[1].Reg()
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPCMPEQW Z26, Z30, K4
+func simdV2k(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[1])
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VPMINUQ X21, X3, K3, X31
+func simdV2kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[1])
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ // These "simd*" series of functions assumes:
+ // Any "K" register that serves as the write-mask
+ // or "predicate" for "predicated AVX512 instructions"
+ // sits right at the end of the operand list.
+ // TODO: verify this assumption.
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPABSB X1, X2, K3 (masking merging)
+func simdV2kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[1])
+ // These "simd*" series of functions assumes:
+ // Any "K" register that serves as the write-mask
+ // or "predicate" for "predicated AVX512 instructions"
+ // sits right at the end of the operand list.
+ // TODO: verify this assumption.
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// This function is to accustomize the shifts.
+// The 2nd arg is an XMM, and this function merely checks that.
+// Example instruction: VPSLLQ Z1, X1, K1, Z2
+func simdVfpkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = v.Args[1].Reg()
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPCMPEQW Z26, Z30, K1, K4
+func simdV2kk(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[1])
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VPOPCNTB X14, K4, X16
+func simdVkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[0])
+ p.AddRestSourceReg(maskReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VROUNDPD $7, X2, X2
+func simdV11Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VREDUCEPD $126, X1, K3, X31
+func simdVkvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VCMPPS $7, X2, X9, X2
+func simdV21Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPINSRB $3, DX, X0, X0
+func simdVgpvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(v.Args[1].Reg())
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+func simdVgpvImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ // within simdgen, the choice of intrinsic shape and the output
+ // intruction format are linked. In the case of VgpImm, there is
+ // a difference in the intrinsic, but no difference in the
+ // instruction, it is just like VgpvImm8.
+ //
+ // See also, simdVgpImm and simdVgpImm8
+ return simdVgpvImm8(s, v)
+}
+
+// Example instruction: VPCMPD $1, Z1, Z2, K1
+func simdV2kImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VPCMPD $1, Z1, Z2, K2, K1
+func simdV2kkImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+func simdV2kvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VFMADD213PD Z2, Z1, Z0
+func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+func simdV31ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+
+ p.AddRestSourceReg(simdReg(v.Args[2]))
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ // p.AddRestSourceReg(x86.REG_K0)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// v31loadResultInArg0Imm8
+// Example instruction:
+// for (VPTERNLOGD128load {sym} [makeValAndOff(int32(int8(c)),off)] x y ptr mem)
+func simdV31loadResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[2].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ return p
+}
+
+// Example instruction: VFMADD213PD Z2, Z1, K1, Z0
+func simdV3kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(maskReg(v.Args[3]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+func simdVgpImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
+ return p
+}
+
+func simdVgpImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ // within simdgen, the choice of intrinsic shape and the output
+ // intruction format are linked. In the case of VgpImm, there is
+ // a difference in the intrinsic, but no difference in the
+ // instruction, it is just like VgpImm8.
+ return simdVgpImm8(s, v)
+}
+
+// Currently unused
+func simdV31(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Currently unused
+func simdV3kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[2])
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[3]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VRCP14PS (DI), K6, X22
+func simdVkvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(maskReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPSLLVD (DX), X7, X18
+func simdV21load(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[1].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPDPWSSD (SI), X24, X18
+func simdV31loadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[2].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPDPWSSD (SI), X24, K1, X18
+func simdV3kvloadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[2].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.AddRestSourceReg(maskReg(v.Args[3]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPSLLVD (SI), X1, K1, X2
+func simdV2kvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[1].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPCMPEQD (SI), X1, K1
+func simdV2kload(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[1].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VCVTTPS2DQ (BX), X2
+func simdV11load(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_MEM
+ p.From.Reg = v.Args[0].Reg()
+ ssagen.AddAux(&p.From, v)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPSHUFD $7, (BX), X11
+func simdV11loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPRORD $81, -15(R14), K7, Y1
+func simdVkvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.AddRestSourceReg(maskReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VPSHLDD $82, 7(SI), Y21, Y3
+func simdV21loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: VCMPPS $81, -7(DI), Y16, K3
+func simdV2kloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VCMPPS $81, -7(DI), Y16, K1, K3
+func simdV2kkloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = maskReg(v)
+ return p
+}
+
+// Example instruction: VGF2P8AFFINEINVQB $64, -17(BP), X31, K3, X26
+func simdV2kvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ sc := v.AuxValAndOff()
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = sc.Val64()
+ m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
+ ssagen.AddAux2(&m, v, sc.Off64())
+ p.AddRestSource(m)
+ p.AddRestSourceReg(simdReg(v.Args[0]))
+ p.AddRestSourceReg(maskReg(v.Args[2]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: SHA1NEXTE X2, X2
+func simdV21ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdReg(v.Args[1])
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: SHA1RNDS4 $1, X2, X2
+func simdV21ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Offset = int64(v.AuxUInt8())
+ p.From.Type = obj.TYPE_CONST
+ p.AddRestSourceReg(simdReg(v.Args[1]))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdReg(v)
+ return p
+}
+
+// Example instruction: SHA256RNDS2 X0, X11, X2
+func simdV31x0AtIn2ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
+ return simdV31ResultInArg0(s, v)
+}
+
var blockJump = [...]struct {
asm, invasm obj.As
}{
@@ -1732,7 +2523,7 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
}
func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
- p := s.Prog(loadByType(t))
+ p := s.Prog(loadByRegWidth(reg, t.Size()))
p.From.Type = obj.TYPE_MEM
p.From.Name = obj.NAME_AUTO
p.From.Sym = n.Linksym()
@@ -1743,7 +2534,7 @@ func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir
}
func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
- p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
+ p = pp.Append(p, storeByRegWidth(reg, t.Size()), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
p.To.Name = obj.NAME_PARAM
p.To.Sym = n.Linksym()
p.Pos = p.Pos.WithNotStmt()
@@ -1778,3 +2569,58 @@ func move16(s *ssagen.State, src, dst, tmp int16, off int64) {
p.To.Reg = dst
p.To.Offset = off
}
+
+// XXX maybe make this part of v.Reg?
+// On the other hand, it is architecture-specific.
+func simdReg(v *ssa.Value) int16 {
+ t := v.Type
+ if !t.IsSIMD() {
+ base.Fatalf("simdReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
+ }
+ return simdRegBySize(v.Reg(), t.Size())
+}
+
+func simdRegBySize(reg int16, size int64) int16 {
+ switch size {
+ case 16:
+ return reg
+ case 32:
+ return reg + (x86.REG_Y0 - x86.REG_X0)
+ case 64:
+ return reg + (x86.REG_Z0 - x86.REG_X0)
+ }
+ panic("simdRegBySize: bad size")
+}
+
+// XXX k mask
+func maskReg(v *ssa.Value) int16 {
+ t := v.Type
+ if !t.IsSIMD() {
+ base.Fatalf("maskReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
+ }
+ switch t.Size() {
+ case 8:
+ return v.Reg()
+ }
+ panic("unreachable")
+}
+
+// XXX k mask + vec
+func simdOrMaskReg(v *ssa.Value) int16 {
+ t := v.Type
+ if t.Size() <= 8 {
+ return maskReg(v)
+ }
+ return simdReg(v)
+}
+
+// XXX this is used for shift operations only.
+// regalloc will issue OpCopy with incorrect type, but the assigned
+// register should be correct, and this function is merely checking
+// the sanity of this part.
+func simdCheckRegOnly(v *ssa.Value, regStart, regEnd int16) int16 {
+ if v.Reg() > regEnd || v.Reg() < regStart {
+ panic("simdCheckRegOnly: not the desired register")
+ }
+ return v.Reg()
+}
diff --git a/src/cmd/compile/internal/amd64/versions_nosimd_test.go b/src/cmd/compile/internal/amd64/versions_nosimd_test.go
new file mode 100644
index 00000000000000..bbfe1c1aeaf85f
--- /dev/null
+++ b/src/cmd/compile/internal/amd64/versions_nosimd_test.go
@@ -0,0 +1,14 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !(amd64 && goexperiment.simd && !boringcrypto)
+
+package amd64_test
+
+func testLoop2(n int, x float64) float64 {
+ return 0
+}
+func testLoop3(n int, x float64) float64 {
+ return 0
+}
diff --git a/src/cmd/compile/internal/amd64/versions_simd_test.go b/src/cmd/compile/internal/amd64/versions_simd_test.go
new file mode 100644
index 00000000000000..e05311ce664dc9
--- /dev/null
+++ b/src/cmd/compile/internal/amd64/versions_simd_test.go
@@ -0,0 +1,36 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build amd64 && goexperiment.simd && !boringcrypto
+
+package amd64_test
+
+import "simd/archsimd"
+
+//go:noinline
+func testLoop2(n int, x float64) float64 {
+ var r float64
+ for range n {
+ if archsimd.X86.AVX2() {
+ v := archsimd.BroadcastFloat64x2(x)
+ v = v.Mul(v)
+ r += v.GetElem(0)
+ }
+ }
+ return r
+}
+
+//go:noinline
+func testLoop3(n int, x float64) float64 {
+ var r float64
+ if !archsimd.X86.AVX2() {
+ n = 0
+ }
+ for range n {
+ v := archsimd.BroadcastFloat64x2(x)
+ v = v.Mul(v)
+ r += v.GetElem(0)
+ }
+ return r
+}
diff --git a/src/cmd/compile/internal/amd64/versions_test.go b/src/cmd/compile/internal/amd64/versions_test.go
index 15395fc5e5283f..2563d1b3118f73 100644
--- a/src/cmd/compile/internal/amd64/versions_test.go
+++ b/src/cmd/compile/internal/amd64/versions_test.go
@@ -247,6 +247,7 @@ func setOf(keys ...string) map[string]bool {
var runtimeFeatures = setOf(
"adx", "aes", "avx", "avx2", "bmi1", "bmi2", "erms", "fma",
"pclmulqdq", "popcnt", "rdtscp", "sse3", "sse41", "sse42", "ssse3",
+ "avx512f", "avx512dq",
)
var featureToOpcodes = map[string][]string{
@@ -279,6 +280,7 @@ var featureToOpcodes = map[string][]string{
"fma": {"vfmadd231sd"},
"movbe": {"movbeqq", "movbeq", "movbell", "movbel", "movbe"},
"lzcnt": {"lzcntq", "lzcntl", "lzcnt"},
+ "avx2": {"vmovsd", "vpbroadcastq", "vmulpd", "vpextrq"}, // not a complete list
}
// Test to use POPCNT instruction, if available
@@ -432,3 +434,21 @@ func TestFMA(t *testing.T) {
}
}
}
+
+func TestLoop(t *testing.T) {
+ // Make sure non-v1 ops are not lifted out of loops (and
+ // hence, out of the cpu feature test that guards them).
+ // See issue 78997.
+ testLoop1(6, 7)
+ testLoop2(6, 7) // in versions_(no)simd_test.go
+ testLoop3(6, 7) // in versions_(no)simd_test.go
+}
+
+//go:noinline
+func testLoop1(n int, x uint64) int {
+ var r int
+ for range n {
+ r += bits.OnesCount64(x)
+ }
+ return r
+}
diff --git a/src/cmd/compile/internal/arm/ssa.go b/src/cmd/compile/internal/arm/ssa.go
index a3bfb491b8b25f..9dca52bf11a6d2 100644
--- a/src/cmd/compile/internal/arm/ssa.go
+++ b/src/cmd/compile/internal/arm/ssa.go
@@ -245,6 +245,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Type = obj.TYPE_REG
p.To.Reg = r
case ssa.OpARMADDS,
+ ssa.OpARMADCS,
ssa.OpARMSUBS:
r := v.Reg0()
r1 := v.Args[0].Reg()
@@ -700,7 +701,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg()
case ssa.OpARMCALLstatic, ssa.OpARMCALLclosure, ssa.OpARMCALLinter:
s.Call(v)
- case ssa.OpARMCALLtail:
+ case ssa.OpARMCALLtail, ssa.OpARMCALLtailinter:
s.TailCall(v)
case ssa.OpARMCALLudiv:
p := s.Prog(obj.ACALL)
diff --git a/src/cmd/compile/internal/arm64/simdssa.go b/src/cmd/compile/internal/arm64/simdssa.go
new file mode 100644
index 00000000000000..41ae2925425494
--- /dev/null
+++ b/src/cmd/compile/internal/arm64/simdssa.go
@@ -0,0 +1,469 @@
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -arch arm64 -arm64Path $ARM64_ISA_PATH go_arm64.yaml types.yaml categories.yaml'; DO NOT EDIT.
+
+package arm64
+
+import (
+ "cmd/compile/internal/ssa"
+ "cmd/compile/internal/ssagen"
+ "cmd/internal/obj"
+ "cmd/internal/obj/arm64"
+)
+
+func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
+ var p *obj.Prog
+ switch v.Op {
+ case ssa.OpARM64VABS16B,
+ ssa.OpARM64VCLS16B,
+ ssa.OpARM64VCLZ16B,
+ ssa.OpARM64VNEG16B,
+ ssa.OpARM64VNOT16B,
+ ssa.OpARM64VCNT16B:
+ p = simdV11(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFABS2D,
+ ssa.OpARM64VABS2D,
+ ssa.OpARM64VFRINTP2D,
+ ssa.OpARM64VSCVTF2D,
+ ssa.OpARM64VUCVTF2D,
+ ssa.OpARM64VFCVTZS2D,
+ ssa.OpARM64VFCVTZU2D,
+ ssa.OpARM64VFRINTM2D,
+ ssa.OpARM64VFNEG2D,
+ ssa.OpARM64VNEG2D,
+ ssa.OpARM64VFRINTN2D,
+ ssa.OpARM64VFSQRT2D,
+ ssa.OpARM64VFRINTZ2D:
+ p = simdV11(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VFABS4S,
+ ssa.OpARM64VABS4S,
+ ssa.OpARM64VFRINTP4S,
+ ssa.OpARM64VSCVTF4S,
+ ssa.OpARM64VUCVTF4S,
+ ssa.OpARM64VFCVTZS4S,
+ ssa.OpARM64VFCVTZU4S,
+ ssa.OpARM64VFRINTM4S,
+ ssa.OpARM64VCLS4S,
+ ssa.OpARM64VCLZ4S,
+ ssa.OpARM64VFNEG4S,
+ ssa.OpARM64VNEG4S,
+ ssa.OpARM64VFRINTN4S,
+ ssa.OpARM64VFSQRT4S,
+ ssa.OpARM64VFRINTZ4S:
+ p = simdV11(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VABS8H,
+ ssa.OpARM64VCLS8H,
+ ssa.OpARM64VCLZ8H,
+ ssa.OpARM64VNEG8H:
+ p = simdV11(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSHL16B,
+ ssa.OpARM64VSSHR16B,
+ ssa.OpARM64VUSHR16B,
+ ssa.OpARM64VSQSHL16Bconst,
+ ssa.OpARM64VUQSHL16Bconst:
+ p = simdV11Imm(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VSHL2D,
+ ssa.OpARM64VSSHR2D,
+ ssa.OpARM64VUSHR2D,
+ ssa.OpARM64VSQSHL2Dconst,
+ ssa.OpARM64VUQSHL2Dconst:
+ p = simdV11Imm(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSHL4S,
+ ssa.OpARM64VSSHR4S,
+ ssa.OpARM64VUSHR4S,
+ ssa.OpARM64VSQSHL4Sconst,
+ ssa.OpARM64VUQSHL4Sconst:
+ p = simdV11Imm(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSHL8H,
+ ssa.OpARM64VSSHR8H,
+ ssa.OpARM64VUSHR8H,
+ ssa.OpARM64VSQSHL8Hconst,
+ ssa.OpARM64VUQSHL8Hconst:
+ p = simdV11Imm(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VDUPBbcast:
+ p = simdV11ImmIn1(s, v, arm64.ARNG_B)
+
+ case ssa.OpARM64VDUPDbcast:
+ p = simdV11ImmIn1(s, v, arm64.ARNG_D)
+
+ case ssa.OpARM64VDUPHbcast:
+ p = simdV11ImmIn1(s, v, arm64.ARNG_H)
+
+ case ssa.OpARM64VDUPSbcast:
+ p = simdV11ImmIn1(s, v, arm64.ARNG_S)
+
+ case ssa.OpARM64VSMAXV16B,
+ ssa.OpARM64VUMAXV16B,
+ ssa.OpARM64VSMINV16B,
+ ssa.OpARM64VUMINV16B,
+ ssa.OpARM64VADDV16B:
+ p = simdV11Scalar(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFMAXV4S,
+ ssa.OpARM64VSMAXV4S,
+ ssa.OpARM64VUMAXV4S,
+ ssa.OpARM64VFMINV4S,
+ ssa.OpARM64VSMINV4S,
+ ssa.OpARM64VUMINV4S,
+ ssa.OpARM64VADDV4S:
+ p = simdV11Scalar(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSMAXV8H,
+ ssa.OpARM64VUMAXV8H,
+ ssa.OpARM64VSMINV8H,
+ ssa.OpARM64VUMINV8H,
+ ssa.OpARM64VADDV8H:
+ p = simdV11Scalar(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VDUPDextr:
+ p = simdV11ScalarImmIn1(s, v, arm64.ARNG_D)
+
+ case ssa.OpARM64VDUPSextr:
+ p = simdV11ScalarImmIn1(s, v, arm64.ARNG_S)
+
+ case ssa.OpARM64VADD16B,
+ ssa.OpARM64VSQADD16B,
+ ssa.OpARM64VUQADD16B,
+ ssa.OpARM64VAND16B,
+ ssa.OpARM64VBIC16B,
+ ssa.OpARM64VSRHADD16B,
+ ssa.OpARM64VURHADD16B,
+ ssa.OpARM64VUZP116B,
+ ssa.OpARM64VUZP216B,
+ ssa.OpARM64VCMEQ16B,
+ ssa.OpARM64VCMGT16B,
+ ssa.OpARM64VCMHI16B,
+ ssa.OpARM64VCMGE16B,
+ ssa.OpARM64VCMHS16B,
+ ssa.OpARM64VTRN116B,
+ ssa.OpARM64VZIP216B,
+ ssa.OpARM64VZIP116B,
+ ssa.OpARM64VTRN216B,
+ ssa.OpARM64VSMAX16B,
+ ssa.OpARM64VUMAX16B,
+ ssa.OpARM64VSMIN16B,
+ ssa.OpARM64VUMIN16B,
+ ssa.OpARM64VMUL16B,
+ ssa.OpARM64VORR16B,
+ ssa.OpARM64VORN16B,
+ ssa.OpARM64VSSHL16B,
+ ssa.OpARM64VUSHL16B,
+ ssa.OpARM64VSQSHL16B,
+ ssa.OpARM64VUQSHL16B,
+ ssa.OpARM64VSUB16B,
+ ssa.OpARM64VSQSUB16B,
+ ssa.OpARM64VUQSUB16B,
+ ssa.OpARM64VEOR16B:
+ p = simdV21(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFADD2D,
+ ssa.OpARM64VADD2D,
+ ssa.OpARM64VSQADD2D,
+ ssa.OpARM64VUQADD2D,
+ ssa.OpARM64VFADDP2D,
+ ssa.OpARM64VADDP2D,
+ ssa.OpARM64VUZP12D,
+ ssa.OpARM64VUZP22D,
+ ssa.OpARM64VFDIV2D,
+ ssa.OpARM64VFCMEQ2D,
+ ssa.OpARM64VCMEQ2D,
+ ssa.OpARM64VFCMGT2D,
+ ssa.OpARM64VCMGT2D,
+ ssa.OpARM64VCMHI2D,
+ ssa.OpARM64VFCMGE2D,
+ ssa.OpARM64VCMGE2D,
+ ssa.OpARM64VCMHS2D,
+ ssa.OpARM64VTRN12D,
+ ssa.OpARM64VZIP22D,
+ ssa.OpARM64VZIP12D,
+ ssa.OpARM64VTRN22D,
+ ssa.OpARM64VFMAX2D,
+ ssa.OpARM64VFMIN2D,
+ ssa.OpARM64VFMUL2D,
+ ssa.OpARM64VSSHL2D,
+ ssa.OpARM64VUSHL2D,
+ ssa.OpARM64VSQSHL2D,
+ ssa.OpARM64VUQSHL2D,
+ ssa.OpARM64VFSUB2D,
+ ssa.OpARM64VSUB2D,
+ ssa.OpARM64VSQSUB2D,
+ ssa.OpARM64VUQSUB2D:
+ p = simdV21(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VFADD4S,
+ ssa.OpARM64VADD4S,
+ ssa.OpARM64VSQADD4S,
+ ssa.OpARM64VUQADD4S,
+ ssa.OpARM64VSRHADD4S,
+ ssa.OpARM64VURHADD4S,
+ ssa.OpARM64VFADDP4S,
+ ssa.OpARM64VADDP4S,
+ ssa.OpARM64VUZP14S,
+ ssa.OpARM64VUZP24S,
+ ssa.OpARM64VFDIV4S,
+ ssa.OpARM64VFCMEQ4S,
+ ssa.OpARM64VCMEQ4S,
+ ssa.OpARM64VFCMGT4S,
+ ssa.OpARM64VCMGT4S,
+ ssa.OpARM64VCMHI4S,
+ ssa.OpARM64VFCMGE4S,
+ ssa.OpARM64VCMGE4S,
+ ssa.OpARM64VCMHS4S,
+ ssa.OpARM64VTRN14S,
+ ssa.OpARM64VZIP24S,
+ ssa.OpARM64VZIP14S,
+ ssa.OpARM64VTRN24S,
+ ssa.OpARM64VFMAX4S,
+ ssa.OpARM64VSMAX4S,
+ ssa.OpARM64VUMAX4S,
+ ssa.OpARM64VFMIN4S,
+ ssa.OpARM64VSMIN4S,
+ ssa.OpARM64VUMIN4S,
+ ssa.OpARM64VFMUL4S,
+ ssa.OpARM64VMUL4S,
+ ssa.OpARM64VSSHL4S,
+ ssa.OpARM64VUSHL4S,
+ ssa.OpARM64VSQSHL4S,
+ ssa.OpARM64VUQSHL4S,
+ ssa.OpARM64VFSUB4S,
+ ssa.OpARM64VSUB4S,
+ ssa.OpARM64VSQSUB4S,
+ ssa.OpARM64VUQSUB4S:
+ p = simdV21(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VADD8H,
+ ssa.OpARM64VSQADD8H,
+ ssa.OpARM64VUQADD8H,
+ ssa.OpARM64VSRHADD8H,
+ ssa.OpARM64VURHADD8H,
+ ssa.OpARM64VADDP8H,
+ ssa.OpARM64VUZP18H,
+ ssa.OpARM64VUZP28H,
+ ssa.OpARM64VCMEQ8H,
+ ssa.OpARM64VCMGT8H,
+ ssa.OpARM64VCMHI8H,
+ ssa.OpARM64VCMGE8H,
+ ssa.OpARM64VCMHS8H,
+ ssa.OpARM64VTRN18H,
+ ssa.OpARM64VZIP28H,
+ ssa.OpARM64VZIP18H,
+ ssa.OpARM64VTRN28H,
+ ssa.OpARM64VSMAX8H,
+ ssa.OpARM64VUMAX8H,
+ ssa.OpARM64VSMIN8H,
+ ssa.OpARM64VUMIN8H,
+ ssa.OpARM64VMUL8H,
+ ssa.OpARM64VSSHL8H,
+ ssa.OpARM64VUSHL8H,
+ ssa.OpARM64VSQSHL8H,
+ ssa.OpARM64VUQSHL8H,
+ ssa.OpARM64VSUB8H,
+ ssa.OpARM64VSQSUB8H,
+ ssa.OpARM64VUQSUB8H:
+ p = simdV21(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VEXT16B:
+ p = simdV21Imm(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VMLA16B,
+ ssa.OpARM64VBIF16B,
+ ssa.OpARM64VBIT16B:
+ p = simdV31ResultInArg0(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFMLA2D:
+ p = simdV31ResultInArg0(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VFMLA4S,
+ ssa.OpARM64VMLA4S:
+ p = simdV31ResultInArg0(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VMLA8H:
+ p = simdV31ResultInArg0(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VMOVBextr:
+ p = simdVgpImmIn1(s, v, arm64.ARNG_B)
+
+ case ssa.OpARM64VMOVDextr:
+ p = simdVgpImmIn1(s, v, arm64.ARNG_D)
+
+ case ssa.OpARM64VMOVHextr:
+ p = simdVgpImmIn1(s, v, arm64.ARNG_H)
+
+ case ssa.OpARM64VMOVSextr:
+ p = simdVgpImmIn1(s, v, arm64.ARNG_S)
+
+ case ssa.OpARM64VMOVBins:
+ p = simdVgpvResultInArg0ImmOutIn0(s, v, arm64.ARNG_B)
+
+ case ssa.OpARM64VMOVDins:
+ p = simdVgpvResultInArg0ImmOutIn0(s, v, arm64.ARNG_D)
+
+ case ssa.OpARM64VMOVHins:
+ p = simdVgpvResultInArg0ImmOutIn0(s, v, arm64.ARNG_H)
+
+ case ssa.OpARM64VMOVSins:
+ p = simdVgpvResultInArg0ImmOutIn0(s, v, arm64.ARNG_S)
+
+ case ssa.OpARM64VMOVDins0:
+ p = simdVfpvResultInArg0ImmOutIn1(s, v, arm64.ARNG_D)
+
+ case ssa.OpARM64VMOVSins0:
+ p = simdVfpvResultInArg0ImmOutIn1(s, v, arm64.ARNG_S)
+
+ case ssa.OpARM64VSXTL16B,
+ ssa.OpARM64VUXTL16B:
+ p = simdV11Long(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFCVTL4S,
+ ssa.OpARM64VSXTL4S,
+ ssa.OpARM64VUXTL4S:
+ p = simdV11Long(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSXTL8H,
+ ssa.OpARM64VUXTL8H:
+ p = simdV11Long(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VFCVTN2D,
+ ssa.OpARM64VSQXTN2D,
+ ssa.OpARM64VSQXTUN2D,
+ ssa.OpARM64VUQXTN2D,
+ ssa.OpARM64VXTN2D:
+ p = simdV11Narrow(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSQXTN4S,
+ ssa.OpARM64VSQXTUN4S,
+ ssa.OpARM64VUQXTN4S,
+ ssa.OpARM64VXTN4S:
+ p = simdV11Narrow(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSQXTN8H,
+ ssa.OpARM64VSQXTUN8H,
+ ssa.OpARM64VUQXTN8H,
+ ssa.OpARM64VXTN8H:
+ p = simdV11Narrow(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSHRN2D:
+ p = simdV11ImmNarrow(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSHRN4S:
+ p = simdV11ImmNarrow(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSHRN8H:
+ p = simdV11ImmNarrow(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSSHLL16B,
+ ssa.OpARM64VUSHLL16B:
+ p = simdV11ImmLong(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VSSHLL4S,
+ ssa.OpARM64VUSHLL4S:
+ p = simdV11ImmLong(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSSHLL8H,
+ ssa.OpARM64VUSHLL8H:
+ p = simdV11ImmLong(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSMULL16B,
+ ssa.OpARM64VUMULL16B:
+ p = simdV21Long(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VPMULL2D:
+ p = simdV21Long(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSMULL4S,
+ ssa.OpARM64VUMULL4S:
+ p = simdV21Long(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSMULL8H,
+ ssa.OpARM64VUMULL8H:
+ p = simdV21Long(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSXTL2_16B,
+ ssa.OpARM64VUXTL2_16B:
+ p = simdV11Long2(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VFCVTL2_4S,
+ ssa.OpARM64VSXTL2_4S,
+ ssa.OpARM64VUXTL2_4S:
+ p = simdV11Long2(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSXTL2_8H,
+ ssa.OpARM64VUXTL2_8H:
+ p = simdV11Long2(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VFCVTN2_2D,
+ ssa.OpARM64VSQXTN2_2D,
+ ssa.OpARM64VSQXTUN2_2D,
+ ssa.OpARM64VUQXTN2_2D,
+ ssa.OpARM64VXTN2_2D:
+ p = simdV21Narrow2(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSQXTN2_4S,
+ ssa.OpARM64VSQXTUN2_4S,
+ ssa.OpARM64VUQXTN2_4S,
+ ssa.OpARM64VXTN2_4S:
+ p = simdV21Narrow2(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSQXTN2_8H,
+ ssa.OpARM64VSQXTUN2_8H,
+ ssa.OpARM64VUQXTN2_8H,
+ ssa.OpARM64VXTN2_8H:
+ p = simdV21Narrow2(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSHRN2_2D:
+ p = simdV21ImmNarrow2(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSHRN2_4S:
+ p = simdV21ImmNarrow2(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSHRN2_8H:
+ p = simdV21ImmNarrow2(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSSHLL2_16B,
+ ssa.OpARM64VUSHLL2_16B:
+ p = simdV11ImmLong2(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VSSHLL2_4S,
+ ssa.OpARM64VUSHLL2_4S:
+ p = simdV11ImmLong2(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSSHLL2_8H,
+ ssa.OpARM64VUSHLL2_8H:
+ p = simdV11ImmLong2(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VSMULL2_16B,
+ ssa.OpARM64VUMULL2_16B:
+ p = simdV21Long2(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VPMULL2_2D:
+ p = simdV21Long2(s, v, arm64.ARNG_2D)
+
+ case ssa.OpARM64VSMULL2_4S,
+ ssa.OpARM64VUMULL2_4S:
+ p = simdV21Long2(s, v, arm64.ARNG_4S)
+
+ case ssa.OpARM64VSMULL2_8H,
+ ssa.OpARM64VUMULL2_8H:
+ p = simdV21Long2(s, v, arm64.ARNG_8H)
+
+ case ssa.OpARM64VTBL16B:
+ p = simdV21List(s, v, arm64.ARNG_16B)
+
+ case ssa.OpARM64VTBX16B:
+ p = simdV31ResultInArg0List(s, v, arm64.ARNG_16B)
+
+ default:
+ // Unknown reg shape
+ return false
+ }
+
+ // Ensure p is marked as used (may not be used in all generated code paths)
+ _ = p
+ return true
+}
diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go
index 43ecb6b4b715b4..44bf01dfa47db2 100644
--- a/src/cmd/compile/internal/arm64/ssa.go
+++ b/src/cmd/compile/internal/arm64/ssa.go
@@ -21,7 +21,11 @@ import (
// loadByType returns the load instruction of the given type.
func loadByType(t *types.Type) obj.As {
- if t.IsFloat() {
+ if t.IsSIMD() {
+ if t.Size() == 16 {
+ return arm64.AFMOVQ // Use FMOVQ (LDR Q) for 128-bit SIMD loads
+ }
+ } else if t.IsFloat() {
switch t.Size() {
case 4:
return arm64.AFMOVS
@@ -57,7 +61,11 @@ func loadByType(t *types.Type) obj.As {
// storeByType returns the store instruction of the given type.
func storeByType(t *types.Type) obj.As {
- if t.IsFloat() {
+ if t.IsSIMD() {
+ if t.Size() == 16 {
+ return arm64.AFMOVQ // Use FMOVQ (STR Q) for 128-bit SIMD stores
+ }
+ } else if t.IsFloat() {
switch t.Size() {
case 4:
return arm64.AFMOVS
@@ -162,6 +170,410 @@ func genIndexedOperand(op ssa.Op, base, idx int16) obj.Addr {
return mop
}
+// simdRegArng encodes ssa value's register with specified simd arrangement
+func simdRegArng(reg int16, arng int16) int16 {
+ if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
+ base.Fatalf("expected fp register: r%d", reg)
+ }
+ var err error
+ if reg, err = arm64.RegisterArrangement(reg, arng, false); err != nil {
+ base.Fatalf("bad simd register arrangement: %v", err)
+ }
+ return reg
+}
+
+// simdRegElem encodes ssa value's reference to a vector register element
+func simdRegElem(reg int16, arng int16, idx int16) (res obj.Addr) {
+ if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
+ base.Fatalf("expected fp register: r%d", reg)
+ }
+ elem, err := arm64.RegisterArrangement(reg, arng, true /*indexing*/)
+ if err != nil {
+ base.Fatalf("bad simd register indexing arrangement: %v", err)
+ }
+ res.Type = obj.TYPE_REG
+ res.Class = arm64.C_ELEM
+ res.Index = idx
+ res.Reg = elem
+ return
+}
+
+// allLanes converts an element arrangement to its 128-bit vector arrangement.
+// e.g., ARNG_B -> ARNG_16B, ARNG_S -> ARNG_4S
+func allLanes(arng int16) int16 {
+ switch arng {
+ case arm64.ARNG_B:
+ return arm64.ARNG_16B
+ case arm64.ARNG_H:
+ return arm64.ARNG_8H
+ case arm64.ARNG_S:
+ return arm64.ARNG_4S
+ case arm64.ARNG_D:
+ return arm64.ARNG_2D
+ default:
+ base.Fatalf("unsupported element arrangement: %d", arng)
+ return 0
+ }
+}
+
+// arngNarrow converts arng to its narrow (halved element width and vector width) arrangement.
+func arngNarrow(arng int16) int16 {
+ switch arng {
+ case arm64.ARNG_8H:
+ return arm64.ARNG_8B
+ case arm64.ARNG_4S:
+ return arm64.ARNG_4H
+ case arm64.ARNG_2D:
+ return arm64.ARNG_2S
+ default:
+ base.Fatalf("unsupported narrow input arrangement: %d", arng)
+ return 0
+ }
+}
+
+// arngLong converts a half-lane arrangement to its long (doubled element width and vector width) arrangement.
+func arngLong(arng int16) int16 {
+ switch arng {
+ case arm64.ARNG_8B:
+ return arm64.ARNG_8H
+ case arm64.ARNG_4H:
+ return arm64.ARNG_4S
+ case arm64.ARNG_2S:
+ return arm64.ARNG_2D
+ case arm64.ARNG_1D:
+ return arm64.ARNG_1Q
+ default:
+ base.Fatalf("unsupported long input arrangement: %d", arng)
+ return 0
+ }
+}
+
+// arngHalfLanes converts a full-width arrangement to its half-lane (64-bit) arrangement.
+// Same element width, half the lanes. Used for long base variant sources.
+func arngHalfLanes(arng int16) int16 {
+ switch arng {
+ case arm64.ARNG_16B:
+ return arm64.ARNG_8B
+ case arm64.ARNG_8H:
+ return arm64.ARNG_4H
+ case arm64.ARNG_4S:
+ return arm64.ARNG_2S
+ case arm64.ARNG_2D:
+ return arm64.ARNG_1D
+ default:
+ base.Fatalf("unsupported halfLanes input arrangement: %d", arng)
+ return 0
+ }
+}
+
+// arngTwiceLanes converts a half-lane (64-bit) arrangement to its full-width arrangement.
+// Same element width, double the lanes. Inverse of arngHalfLanes.
+func arngTwiceLanes(arng int16) int16 {
+ switch arng {
+ case arm64.ARNG_8B:
+ return arm64.ARNG_16B
+ case arm64.ARNG_4H:
+ return arm64.ARNG_8H
+ case arm64.ARNG_2S:
+ return arm64.ARNG_4S
+ default:
+ base.Fatalf("unsupported twiceLanes input arrangement: %d", arng)
+ return 0
+ }
+}
+
+// simdV01Imm generates a VMOVI-like instruction, e.g. VMOVI $0, V0.B16
+func simdV01Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdV11Asm generates element-wise unary vector operations with explicit asm, e.g. VMOV V1.B16, V0.B16
+func simdV11Asm(s *ssagen.State, asm obj.As, src, dst int16, arrangement int16) *obj.Prog {
+ p := s.Prog(asm)
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(src, arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(dst, arrangement)
+ return p
+}
+
+// simdV11 generates element-wise unary vector operations, e.g. VCNT V1.B8, V0.B8
+func simdV11(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ return simdV11Asm(s, v.Op.Asm(), v.Args[0].Reg(), v.Reg(), arrangement)
+}
+
+// simdV11Imm generates a unary vector operation with immediate constant,
+// e.g. VUSHR $3, V1.B16, V0.B16
+func simdV11Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdV11ImmIn1 generates a broadcast1ToN instruction,
+// e.g. VDUP V1.S[0], V0.S4 (duplicate element 0 to all lanes)
+// The arrangement parameter specifies the element arrangement (e.g., ARNG_S, ARNG_D)
+func simdV11ImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), allLanes(arrangement))
+ return p
+}
+
+// simdV11Scalar generates vector-to-scalar reduction operations, e.g. VUADDLV V1.B8, V0
+func simdV11Scalar(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
+ return p
+}
+
+// simdV11ScalarImmIn1 generates a SIMD instruction with indexed input and
+// scalar-in-vector-register output, e.g. VDUP V1.S[1], V0
+// The arrangement parameter specifies the source arrangement (e.g., S, D)
+func simdV11ScalarImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
+ p.To.Class = arm64.C_VREG
+ return p
+}
+
+// simdV21 generates element-wise binary vector operations, e.g. VFADD V1.S4, V2.S4, V0.S4
+func simdV21(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdV21Imm generates a binary instruction with immediate, e.g. EXT $imm, Vm.16B, Vn.16B, Vd.16B
+func simdV21Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: simdRegArng(v.Args[1].Reg(), arrangement)})
+ return p
+}
+
+// simdV31ResultInArg0 generates a destructive 3-register instruction,
+// e.g. VBIT Vm.16B, Vn.16B, Vd.16B.
+func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
+ p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdV21List generates a binary instruction with register list, e.g. TBL Vm.Ta, {Vn.B16}, Vd.Ta.
+func simdV21List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ if v.Op.Asm() != arm64.AVTBL { // TODO: support other instructions as needed.
+ panic("simdV21List: expected VTBL")
+ }
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ // TBL requires B16 table arrangement.
+ // Also, multi-element register lists are not supported by regalloc.
+ const listB16 = int64(1 << 30)
+ regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, listB16, 0)
+ p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdV31ResultInArg0List generates a destructive 3-register instruction
+// with register list, e.g. TBX Vm.Ta, {Vn.B16}, Vd.Ta.
+func simdV31ResultInArg0List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ if v.Op.Asm() != arm64.AVTBX { // TODO: support other instructions as needed.
+ panic("simdV31ResultInArg0List: expected VTBX")
+ }
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
+ // TBX requires B16 table arrangement.
+ // Also, multi-element register lists are not supported by regalloc.
+ const listB16 = int64(1 << 30)
+ regList, _ := arm64.RegisterListOffset(int(v.Args[1].Reg()&31), 1, listB16, 0)
+ p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arrangement)
+ return p
+}
+
+// simdVfpvResultInArg0ImmOutIn1 generates vector floating-point SetElem,
+// e.g. VMOV V2.S[0], V1.S[3] (INS element instruction)
+// The arrangement parameter specifies the vector element arrangement (e.g., S, D)
+func simdVfpvResultInArg0ImmOutIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
+ p.From = simdRegElem(v.Args[1].Reg(), arrangement, 0)
+ return p
+}
+
+// simdVgpImmIn1 generates vector GetElem instruction VMOV V1.S[2], R0
+// The arrangement parameter specifies the vector element arrangement (e.g., S, D)
+func simdVgpImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
+ p.To.Reg = v.Reg()
+ p.To.Type = obj.TYPE_REG
+ return p
+}
+
+// simdVgpvResultInArg0ImmOutIn0 generates vector SetElem, e.g. VMOV R0, V1.S[2] (INS general instruction)
+// The arrangement parameter specifies the vector element arrangement (e.g., S, D)
+func simdVgpvResultInArg0ImmOutIn0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
+ p.From.Reg = v.Args[1].Reg()
+ p.From.Type = obj.TYPE_REG
+ return p
+}
+
+// Narrow and long lowering helpers
+
+// simdV11Narrow generates a pure narrowing instruction, e.g. XTN Vn.8H, Vd.8B
+func simdV11Narrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
+ return p
+}
+
+// simdV21Narrow2 generates a a destructive (updating upper half only) narrow "2" instruction,
+// e.g. XTN2 V1.4S, V0.8H. The arrangement parameter specifies the source arrangement.
+func simdV21Narrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
+ return p
+}
+
+// simdV11ImmNarrow generates a pure narrowing instruction with immediate, e.g. SHRN $imm, V1.4S, V0.8B
+// The arrangement parameter specifies the source arrangement.
+func simdV11ImmNarrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
+ return p
+}
+
+// simdV21ImmNarrow2 generates a destructive (updating upper half only) narrow "2" instruction
+// with immediate, e.g. SHRN2 $imm, V1.4S, V0.16B. The arrangement parameter specifies the source arrangement.
+func simdV21ImmNarrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
+ return p
+}
+
+// simdV11Long generates a unary long instruction, e.g. SXTL V1.4H, V0.8H
+// The instruction reads the lower half of the source, the destination has 2x element size.
+func simdV11Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ src := arngHalfLanes(arrangement)
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[0].Reg(), src)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
+ return p
+}
+
+// simdV11Long2 generates a unary long "2" instruction, e.g. SXTL2 V1.4S, V0.2D
+// The instruction reads the upper half of the source, the destination has 2x element size.
+func simdV11Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
+ return p
+}
+
+// simdV11ImmLong generates a long instruction with immediate, e.g. USHLL $imm, V1.4H, V0.8H
+// The instruction reads the lower half of the source, the destination has 2x element size.
+func simdV11ImmLong(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ src := arngHalfLanes(arrangement)
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[0].Reg(), src)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
+ return p
+}
+
+// simdV11ImmLong2 generates a long "2" instruction with immediate, e.g. USHLL2 $imm, V1.4S, V0.2D
+// The instruction reads the upper half of the source, the destination has 2x element size.
+func simdV11ImmLong2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = int64(v.AuxUInt8())
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
+ return p
+}
+
+// simdV21Long generates a binary long instruction, e.g. UMULL V1.4H, V2.4H, V0.8H
+// The instruction reads lower halves of its sources, the destination has 2x element size.
+func simdV21Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ src := arngHalfLanes(arrangement)
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[1].Reg(), src)
+ p.Reg = simdRegArng(v.Args[0].Reg(), src)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
+ return p
+}
+
+// simdV21Long2 generates a binary long "2" instruction, e.g. UMULL2 V1.4S, V2.4S, V0.2D
+// The instruction reads upper halves of its sources, the destination has 2x element size.
+func simdV21Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
+ p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
+ return p
+}
+
func ssaGenValue(s *ssagen.State, v *ssa.Value) {
switch v.Op {
case ssa.OpCopy, ssa.OpARM64MOVDreg:
@@ -183,6 +595,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
default:
panic("bad float size")
}
+ } else if v.Type.IsSIMD() {
+ if v.Type.Size() == 16 {
+ simdV11Asm(s, arm64.AVMOV, x, y, arm64.ARNG_16B)
+ return
+ } else {
+ panic("bad simd size")
+ }
}
p := s.Prog(as)
p.From.Type = obj.TYPE_REG
@@ -191,6 +610,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = y
case ssa.OpARM64MOVDnop, ssa.OpARM64ZERO:
// nothing to do
+ case ssa.OpARM64VMOVI16B:
+ simdV01Imm(s, v, arm64.ARNG_16B)
case ssa.OpLoadReg:
if v.Type.IsFlags() {
v.Fatalf("load flags not implemented: %v", v.LongString())
@@ -510,14 +931,15 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpARM64MOVWUload,
ssa.OpARM64MOVDload,
ssa.OpARM64FMOVSload,
- ssa.OpARM64FMOVDload:
+ ssa.OpARM64FMOVDload,
+ ssa.OpARM64FMOVQload:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_MEM
p.From.Reg = v.Args[0].Reg()
ssagen.AddAux(&p.From, v)
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
- case ssa.OpARM64LDP, ssa.OpARM64LDPW, ssa.OpARM64LDPSW, ssa.OpARM64FLDPD, ssa.OpARM64FLDPS:
+ case ssa.OpARM64LDP, ssa.OpARM64LDPW, ssa.OpARM64LDPSW, ssa.OpARM64FLDPD, ssa.OpARM64FLDPS, ssa.OpARM64FLDPQ:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_MEM
p.From.Reg = v.Args[0].Reg()
@@ -560,6 +982,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpARM64MOVDstore,
ssa.OpARM64FMOVSstore,
ssa.OpARM64FMOVDstore,
+ ssa.OpARM64FMOVQstore,
ssa.OpARM64STLRB,
ssa.OpARM64STLR,
ssa.OpARM64STLRW:
@@ -584,7 +1007,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[2].Reg()
- case ssa.OpARM64STP, ssa.OpARM64STPW, ssa.OpARM64FSTPD, ssa.OpARM64FSTPS:
+ case ssa.OpARM64STP, ssa.OpARM64STPW, ssa.OpARM64FSTPD, ssa.OpARM64FSTPS, ssa.OpARM64FSTPQ:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REGREG
p.From.Reg = v.Args[1].Reg()
@@ -963,6 +1386,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpARM64MVN,
ssa.OpARM64NEG,
ssa.OpARM64FABSD,
+ ssa.OpARM64FABSS,
ssa.OpARM64FMOVDfpgp,
ssa.OpARM64FMOVDgpfp,
ssa.OpARM64FMOVSfpgp,
@@ -1001,7 +1425,12 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpARM64FRINTMD,
ssa.OpARM64FRINTND,
ssa.OpARM64FRINTPD,
- ssa.OpARM64FRINTZD:
+ ssa.OpARM64FRINTZD,
+ ssa.OpARM64FRINTAS,
+ ssa.OpARM64FRINTMS,
+ ssa.OpARM64FRINTNS,
+ ssa.OpARM64FRINTPS,
+ ssa.OpARM64FRINTZS:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[0].Reg()
@@ -1010,17 +1439,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F:
// input is already rounded
case ssa.OpARM64VCNT:
- p := s.Prog(v.Op.Asm())
- p.From.Type = obj.TYPE_REG
- p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
- p.To.Type = obj.TYPE_REG
- p.To.Reg = (v.Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
+ simdV11(s, v, arm64.ARNG_8B)
case ssa.OpARM64VUADDLV:
- p := s.Prog(v.Op.Asm())
- p.From.Type = obj.TYPE_REG
- p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
- p.To.Type = obj.TYPE_REG
- p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
+ simdV11Scalar(s, v, arm64.ARNG_8B)
case ssa.OpARM64CSEL, ssa.OpARM64CSEL0:
r1 := int16(arm64.REGZERO)
if v.Op != ssa.OpARM64CSEL0 {
@@ -1314,7 +1735,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter:
s.Call(v)
- case ssa.OpARM64CALLtail:
+ case ssa.OpARM64CALLtail, ssa.OpARM64CALLtailinter:
s.TailCall(v)
case ssa.OpARM64LoweredWB:
p := s.Prog(obj.ACALL)
@@ -1322,6 +1743,11 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Name = obj.NAME_EXTERN
// AuxInt encodes how many buffer entries we need.
p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
+ case ssa.OpARM64LoweredMemEq:
+ p := s.Prog(obj.ACALL)
+ p.To.Type = obj.TYPE_MEM
+ p.To.Name = obj.NAME_EXTERN
+ p.To.Sym = ir.Syms.Memequal
case ssa.OpARM64LoweredPanicBoundsRR, ssa.OpARM64LoweredPanicBoundsRC, ssa.OpARM64LoweredPanicBoundsCR, ssa.OpARM64LoweredPanicBoundsCC:
// Compute the constant we put in the PCData entry for this call.
@@ -1507,7 +1933,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
default:
- v.Fatalf("genValue not implemented: %s", v.LongString())
+ if !ssaGenSIMDValue(s, v) {
+ v.Fatalf("genValue not implemented: %s", v.LongString())
+ }
}
}
diff --git a/src/cmd/compile/internal/base/base.go b/src/cmd/compile/internal/base/base.go
index ee3772c5ca2bfa..405c7938a51aad 100644
--- a/src/cmd/compile/internal/base/base.go
+++ b/src/cmd/compile/internal/base/base.go
@@ -5,11 +5,7 @@
package base
import (
- "fmt"
"os"
- "runtime"
- "runtime/debug"
- "runtime/metrics"
)
var atExitFuncs []func()
@@ -29,193 +25,3 @@ func Exit(code int) {
// To enable tracing support (-t flag), set EnableTrace to true.
const EnableTrace = false
-
-// forEachGC calls fn each GC cycle until it returns false.
-func forEachGC(fn func() bool) {
- type T [32]byte // large enough to avoid runtime's tiny object allocator
-
- var finalizer func(*T)
- finalizer = func(p *T) {
- if fn() {
- runtime.SetFinalizer(p, finalizer)
- }
- }
-
- finalizer(new(T))
-}
-
-// AdjustStartingHeap modifies GOGC so that GC should not occur until the heap
-// grows to the requested size. This is intended but not promised, though it
-// is true-mostly, depending on when the adjustment occurs and on the
-// compiler's input and behavior. Once this size is approximately reached
-// GOGC is reset to 100; subsequent GCs may reduce the heap below the requested
-// size, but this function does not affect that.
-//
-// -d=gcadjust=1 enables logging of GOGC adjustment events.
-//
-// NOTE: If you think this code would help startup time in your own
-// application and you decide to use it, please benchmark first to see if it
-// actually works for you (it may not: the Go compiler is not typical), and
-// whatever the outcome, please leave a comment on bug #56546. This code
-// uses supported interfaces, but depends more than we like on
-// current+observed behavior of the garbage collector, so if many people need
-// this feature, we should consider/propose a better way to accomplish it.
-func AdjustStartingHeap(requestedHeapGoal uint64) {
- logHeapTweaks := Debug.GCAdjust == 1
- mp := runtime.GOMAXPROCS(0)
- gcConcurrency := Flag.LowerC
-
- const (
- goal = "/gc/heap/goal:bytes"
- count = "/gc/cycles/total:gc-cycles"
- allocs = "/gc/heap/allocs:bytes"
- frees = "/gc/heap/frees:bytes"
- )
-
- sample := []metrics.Sample{{Name: goal}, {Name: count}, {Name: allocs}, {Name: frees}}
- const (
- GOAL = 0
- COUNT = 1
- ALLOCS = 2
- FREES = 3
- )
-
- // Assumptions and observations of Go's garbage collector, as of Go 1.17-1.20:
-
- // - the initial heap goal is 4M, by fiat. It is possible for Go to start
- // with a heap as small as 512k, so this may change in the future.
-
- // - except for the first heap goal, heap goal is a function of
- // observed-live at the previous GC and current GOGC. After the first
- // GC, adjusting GOGC immediately updates GOGC; before the first GC,
- // adjusting GOGC does not modify goal (but the change takes effect after
- // the first GC).
-
- // - the before/after first GC behavior is not guaranteed anywhere, it's
- // just behavior, and it's a bad idea to rely on it.
-
- // - we don't know exactly when GC will run, even after we adjust GOGC; the
- // first GC may not have happened yet, may have already happened, or may
- // be currently in progress, and GCs can start for several reasons.
-
- // - forEachGC above will run the provided function at some delay after each
- // GC's mark phase terminates; finalizers are run after marking as the
- // spans containing finalizable objects are swept, driven by GC
- // background activity and allocation demand.
-
- // - "live at last GC" is not available through the current metrics
- // interface. Instead, live is estimated by knowing the adjusted value of
- // GOGC and the new heap goal following a GC (this requires knowing that
- // at least one GC has occurred):
- // estLive = 100 * newGoal / (100 + currentGogc)
- // this new value of GOGC
- // newGogc = 100*requestedHeapGoal/estLive - 100
- // will result in the desired goal. The logging code checks that the
- // resulting goal is correct.
-
- // There's a small risk that the finalizer will be slow to run after a GC
- // that expands the goal to a huge value, and that this will lead to
- // out-of-memory. This doesn't seem to happen; in experiments on a variety
- // of machines with a variety of extra loads to disrupt scheduling, the
- // worst overshoot observed was 50% past requestedHeapGoal.
-
- metrics.Read(sample)
- for _, s := range sample {
- if s.Value.Kind() == metrics.KindBad {
- // Just return, a slightly slower compilation is a tolerable outcome.
- if logHeapTweaks {
- fmt.Fprintf(os.Stderr, "GCAdjust: Regret unexpected KindBad for metric %s\n", s.Name)
- }
- return
- }
- }
-
- // Tinker with GOGC to make the heap grow rapidly at first.
- currentGoal := sample[GOAL].Value.Uint64() // Believe this will be 4MByte or less, perhaps 512k
- myGogc := 100 * requestedHeapGoal / currentGoal
- if myGogc <= 150 {
- return
- }
-
- if logHeapTweaks {
- sample := append([]metrics.Sample(nil), sample...) // avoid races with GC callback
- AtExit(func() {
- metrics.Read(sample)
- goal := sample[GOAL].Value.Uint64()
- count := sample[COUNT].Value.Uint64()
- oldGogc := debug.SetGCPercent(100)
- if oldGogc == 100 {
- fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d\n",
- goal, oldGogc, count, mp, gcConcurrency)
- } else {
- inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
- overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
- fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %d gogc %d count %d maxprocs %d gcConcurrency %d overPct %d\n",
- goal, oldGogc, count, mp, gcConcurrency, overPct)
-
- }
- })
- }
-
- debug.SetGCPercent(int(myGogc))
-
- adjustFunc := func() bool {
-
- metrics.Read(sample)
- goal := sample[GOAL].Value.Uint64()
- count := sample[COUNT].Value.Uint64()
-
- if goal <= requestedHeapGoal { // Stay the course
- if logHeapTweaks {
- fmt.Fprintf(os.Stderr, "GCAdjust: Reuse GOGC adjust, current goal %d, count is %d, current gogc %d\n",
- goal, count, myGogc)
- }
- return true
- }
-
- // Believe goal has been adjusted upwards, else it would be less-than-or-equal than requestedHeapGoal
- calcLive := 100 * goal / (100 + myGogc)
-
- if 2*calcLive < requestedHeapGoal { // calcLive can exceed requestedHeapGoal!
- myGogc = 100*requestedHeapGoal/calcLive - 100
-
- if myGogc > 125 {
- // Not done growing the heap.
- oldGogc := debug.SetGCPercent(int(myGogc))
-
- if logHeapTweaks {
- // Check that the new goal looks right
- inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
- metrics.Read(sample)
- newGoal := sample[GOAL].Value.Uint64()
- pctOff := 100 * (int64(newGoal) - int64(requestedHeapGoal)) / int64(requestedHeapGoal)
- // Check that the new goal is close to requested. 3% of make.bash fails this test. Why, TBD.
- if pctOff < 2 {
- fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d\n",
- goal, count, oldGogc, myGogc, calcLive, pctOff)
- } else {
- // The GC is being annoying and not giving us the goal that we requested, say more to help understand when/why.
- fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %d, count is %d, gogc was %d, is now %d, calcLive %d pctOff %d inUse %d\n",
- goal, count, oldGogc, myGogc, calcLive, pctOff, inUse)
- }
- }
- return true
- }
- }
-
- // In this case we're done boosting GOGC, set it to 100 and don't set a new finalizer.
- oldGogc := debug.SetGCPercent(100)
- // inUse helps estimate how late the finalizer ran; at the instant the previous GC ended,
- // it was (in theory) equal to the previous GC's heap goal. In a growing heap it is
- // expected to grow to the new heap goal.
- inUse := sample[ALLOCS].Value.Uint64() - sample[FREES].Value.Uint64()
- overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
- if logHeapTweaks {
- fmt.Fprintf(os.Stderr, "GCAdjust: Reset GOGC adjust, old goal %d, count is %d, gogc was %d, calcLive %d inUse %d overPct %d\n",
- goal, count, oldGogc, calcLive, inUse, overPct)
- }
- return false
- }
-
- forEachGC(adjustFunc)
-}
diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go
index 9e8ab2f488bb4d..a4cbe724e88c2f 100644
--- a/src/cmd/compile/internal/base/debug.go
+++ b/src/cmd/compile/internal/base/debug.go
@@ -12,38 +12,47 @@ var Debug DebugFlags
// DebugFlags defines the debugging configuration values (see var Debug).
// Each struct field is a different value, named for the lower-case of the field name.
// Each field must be an int or string and must have a `help` struct tag.
+// Don't forget to note that concurrency is "ok" when it is,
+// else the compiler will serialize for that flag.
//
// The -d option takes a comma-separated list of settings.
// Each setting is name=value; for ints, name is short for name=1.
type DebugFlags struct {
AlignHot int `help:"enable hot block alignment (currently requires -pgo)" concurrent:"ok"`
Append int `help:"print information about append compilation"`
+ AstDump string `help:"for specified function/method, dump AST/IR at interesting points in compilation, to file pkg.func.ast. Use leading ~ for regular expression match."`
Checkptr int `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation" concurrent:"ok"`
Closure int `help:"print information about closure compilation"`
+ CompressInstructions int `help:"use compressed instructions when possible (if supported by architecture)" concurrent:"ok"`
Converthash string `help:"hash value for use in debugging changes to platform-dependent float-to-[u]int conversion" concurrent:"ok"`
Defer int `help:"print information about defer compilation"`
DisableNil int `help:"disable nil checks" concurrent:"ok"`
DumpInlFuncProps string `help:"dump function properties from inl heuristics to specified file"`
DumpInlCallSiteScores int `help:"dump scored callsites during inlining"`
- InlScoreAdj string `help:"set inliner score adjustments (ex: -d=inlscoreadj=panicPathAdj:10/passConstToNestedIfAdj:-90)"`
+ InlScoreAdj string `help:"set inliner score adjustments (ex: -d=inlscoreadj=panicPathAdj:10/passConstToNestedIfAdj:-90)" concurrent:"ok"`
InlBudgetSlack int `help:"amount to expand the initial inline budget when new inliner enabled. Defaults to 80 if option not set." concurrent:"ok"`
DumpPtrs int `help:"show Node pointers values in dump output"`
DwarfInl int `help:"print information about DWARF inlined function creation"`
EscapeMutationsCalls int `help:"print extra escape analysis diagnostics about mutations and calls" concurrent:"ok"`
EscapeDebug int `help:"print information about escape analysis and resulting optimizations" concurrent:"ok"`
+ EscapeAlias int `help:"print information about alias analysis" concurrent:"ok"`
+ EscapeAliasCheck int `help:"enable additional validation for alias analysis" concurrent:"ok"`
Export int `help:"print export data"`
FIPSHash string `help:"hash value for FIPS debugging" concurrent:"ok"`
Fmahash string `help:"hash value for use in debugging platform-dependent multiply-add use" concurrent:"ok"`
+ FreeAppend int `help:"insert frees of append results when proven safe (0 disabled, 1 enabled, 2 enabled + log)" concurrent:"ok"`
GCAdjust int `help:"log adjustments to GOGC" concurrent:"ok"`
GCCheck int `help:"check heap/gc use by compiler" concurrent:"ok"`
GCProg int `help:"print dump of GC programs"`
+ GCStart int `help:"specify \"starting\" compiler's heap size in MiB" concurrent:"ok"`
Gossahash string `help:"hash value for use in debugging the compiler"`
InlFuncsWithClosures int `help:"allow functions with closures to be inlined" concurrent:"ok"`
InlStaticInit int `help:"allow static initialization of inlined calls" concurrent:"ok"`
+ InterfaceCycles int `help:"allow anonymous interface cycles" concurrent:"ok"`
Libfuzzer int `help:"enable coverage instrumentation for libfuzzer"`
LiteralAllocHash string `help:"hash value for use in debugging literal allocation optimizations" concurrent:"ok"`
- LoopVar int `help:"shared (0, default), 1 (private loop variables), 2, private + log"`
- LoopVarHash string `help:"for debugging changes in loop behavior. Overrides experiment and loopvar flag."`
+ LoopVar int `help:"shared (0), 1 (private loop variables, default), 2, private + log" concurrent:"ok"`
+ LoopVarHash string `help:"for debugging changes in loop behavior. Overrides experiment and loopvar flag." concurrent:"ok"`
LocationLists int `help:"print information about DWARF location list creation"`
MaxShapeLen int `help:"hash shape names longer than this threshold (default 500)" concurrent:"ok"`
MergeLocals int `help:"merge together non-interfering local stack slots" concurrent:"ok"`
@@ -59,10 +68,11 @@ type DebugFlags struct {
Panic int `help:"show all compiler panics"`
Reshape int `help:"print information about expression reshaping"`
Shapify int `help:"print information about shaping recursive types"`
+ Simd int `help:"print information about simd analysis and code transformation" concurrent:"ok"`
Slice int `help:"print information about slice compilation"`
SoftFloat int `help:"force compiler to emit soft-float code" concurrent:"ok"`
StaticCopy int `help:"print information about missed static copies" concurrent:"ok"`
- SyncFrames int `help:"how many writer stack frames to include at sync points in unified export data"`
+ SyncFrames int `help:"how many writer stack frames to include at sync points in unified export data" concurrent:"ok"`
TailCall int `help:"print information about tail calls"`
TypeAssert int `help:"print information about type assertion inlining"`
WB int `help:"print information about write barriers"`
@@ -78,7 +88,7 @@ type DebugFlags struct {
VariableMakeHash string `help:"hash value for debugging stack allocation of variable-sized make results" concurrent:"ok"`
VariableMakeThreshold int `help:"threshold in bytes for possible stack allocation of variable-sized make results" concurrent:"ok"`
WrapGlobalMapDbg int `help:"debug trace output for global map init wrapping"`
- WrapGlobalMapCtl int `help:"global map init wrap control (0 => default, 1 => off, 2 => stress mode, no size cutoff)"`
+ WrapGlobalMapCtl int `help:"global map init wrap control (0 => default, 1 => off, 2 => stress mode, no size cutoff)" concurrent:"ok"`
ZeroCopy int `help:"enable zero-copy string->[]byte conversions" concurrent:"ok"`
ConcurrentOk bool // true if only concurrentOk flags seen
diff --git a/src/cmd/compile/internal/base/flag.go b/src/cmd/compile/internal/base/flag.go
index 1d211e0a2dd9f4..c8f1b2db30db6d 100644
--- a/src/cmd/compile/internal/base/flag.go
+++ b/src/cmd/compile/internal/base/flag.go
@@ -78,6 +78,7 @@ type CmdFlags struct {
LowerR CountFlag "help:\"debug generated wrappers\""
LowerT bool "help:\"enable tracing for debugging the compiler\""
LowerW CountFlag "help:\"debug type checking\""
+ LowerU CountFlag "help:\"emit unsorted warnings/errors\""
LowerV *bool "help:\"increase debug verbosity\""
// Special characters
@@ -177,10 +178,12 @@ func ParseFlags() {
Flag.WB = true
Debug.ConcurrentOk = true
+ Debug.CompressInstructions = 1
Debug.MaxShapeLen = 500
Debug.AlignHot = 1
Debug.InlFuncsWithClosures = 1
Debug.InlStaticInit = 1
+ Debug.FreeAppend = 1
Debug.PGOInline = 1
Debug.PGODevirtualize = 2
Debug.SyncFrames = -1 // disable sync markers by default
@@ -299,6 +302,7 @@ func ParseFlags() {
}
parseSpectre(Flag.Spectre) // left as string for RecordFlags
+ Ctxt.CompressInstructions = Debug.CompressInstructions != 0
Ctxt.Flag_shared = Ctxt.Flag_dynlink || Ctxt.Flag_shared
Ctxt.Flag_optimize = Flag.N == 0
Ctxt.Debugasm = int(Flag.S)
@@ -471,7 +475,6 @@ func concurrentFlagOk() bool {
Flag.E == 0 &&
Flag.K == 0 &&
Flag.L == 0 &&
- Flag.LowerH == 0 &&
Flag.LowerJ == 0 &&
Flag.LowerM == 0 &&
Flag.LowerR == 0
diff --git a/src/cmd/compile/internal/base/print.go b/src/cmd/compile/internal/base/print.go
index 9e3348c1ecca89..10c47b43c40080 100644
--- a/src/cmd/compile/internal/base/print.go
+++ b/src/cmd/compile/internal/base/print.go
@@ -45,7 +45,7 @@ func SyntaxErrors() int {
}
// addErrorMsg adds a new errorMsg (which may be a warning) to errorMsgs.
-func addErrorMsg(pos src.XPos, code errors.Code, format string, args ...interface{}) {
+func addErrorMsg(pos src.XPos, code errors.Code, format string, args ...any) {
msg := fmt.Sprintf(format, args...)
// Only add the position if know the position.
// See issue golang.org/issue/11361.
@@ -83,7 +83,9 @@ func FlushErrors() {
if len(errorMsgs) == 0 {
return
}
- sort.Stable(byPos(errorMsgs))
+ if Flag.LowerU == 0 {
+ sort.Stable(byPos(errorMsgs))
+ }
for i, err := range errorMsgs {
if i == 0 || err.msg != errorMsgs[i-1].msg {
fmt.Print(err.msg)
@@ -108,12 +110,12 @@ func sameline(a, b src.XPos) bool {
}
// Errorf reports a formatted error at the current line.
-func Errorf(format string, args ...interface{}) {
+func Errorf(format string, args ...any) {
ErrorfAt(Pos, 0, format, args...)
}
// ErrorfAt reports a formatted error message at pos.
-func ErrorfAt(pos src.XPos, code errors.Code, format string, args ...interface{}) {
+func ErrorfAt(pos src.XPos, code errors.Code, format string, args ...any) {
msg := fmt.Sprintf(format, args...)
if strings.HasPrefix(msg, "syntax error") {
@@ -164,7 +166,7 @@ func UpdateErrorDot(line string, name, expr string) {
// In general the Go compiler does NOT generate warnings,
// so this should be used only when the user has opted in
// to additional output by setting a particular flag.
-func Warn(format string, args ...interface{}) {
+func Warn(format string, args ...any) {
WarnfAt(Pos, format, args...)
}
@@ -172,7 +174,7 @@ func Warn(format string, args ...interface{}) {
// In general the Go compiler does NOT generate warnings,
// so this should be used only when the user has opted in
// to additional output by setting a particular flag.
-func WarnfAt(pos src.XPos, format string, args ...interface{}) {
+func WarnfAt(pos src.XPos, format string, args ...any) {
addErrorMsg(pos, 0, format, args...)
if Flag.LowerM != 0 {
FlushErrors()
@@ -191,7 +193,7 @@ func WarnfAt(pos src.XPos, format string, args ...interface{}) {
// prints a stack trace.
//
// If -h has been specified, Fatalf panics to force the usual runtime info dump.
-func Fatalf(format string, args ...interface{}) {
+func Fatalf(format string, args ...any) {
FatalfAt(Pos, format, args...)
}
@@ -209,7 +211,7 @@ var bugStack = counter.NewStack("compile/bug", 16) // 16 is arbitrary; used by g
// prints a stack trace.
//
// If -h has been specified, FatalfAt panics to force the usual runtime info dump.
-func FatalfAt(pos src.XPos, format string, args ...interface{}) {
+func FatalfAt(pos src.XPos, format string, args ...any) {
FlushErrors()
bugStack.Inc()
@@ -244,14 +246,14 @@ func Assert(b bool) {
}
// Assertf reports a fatal error with Fatalf, unless b is true.
-func Assertf(b bool, format string, args ...interface{}) {
+func Assertf(b bool, format string, args ...any) {
if !b {
Fatalf(format, args...)
}
}
// AssertfAt reports a fatal error with FatalfAt, unless b is true.
-func AssertfAt(b bool, pos src.XPos, format string, args ...interface{}) {
+func AssertfAt(b bool, pos src.XPos, format string, args ...any) {
if !b {
FatalfAt(pos, format, args...)
}
diff --git a/src/cmd/compile/internal/base/startheap.go b/src/cmd/compile/internal/base/startheap.go
new file mode 100644
index 00000000000000..1d2713efdbc3f9
--- /dev/null
+++ b/src/cmd/compile/internal/base/startheap.go
@@ -0,0 +1,272 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+ "fmt"
+ "os"
+ "runtime"
+ "runtime/debug"
+ "runtime/metrics"
+ "sync"
+)
+
+// forEachGC calls fn each GC cycle until it returns false.
+func forEachGC(fn func() bool) {
+ type T [32]byte // large enough to avoid runtime's tiny object allocator
+ var finalizer func(*T)
+ finalizer = func(p *T) {
+
+ if fn() {
+ runtime.SetFinalizer(p, finalizer)
+ }
+ }
+
+ finalizer(new(T))
+}
+
+// AdjustStartingHeap modifies GOGC so that GC should not occur until the heap
+// grows to the requested size. This is intended but not promised, though it
+// is true-mostly, depending on when the adjustment occurs and on the
+// compiler's input and behavior. Once the live heap is approximately half
+// this size, GOGC is reset to its value when AdjustStartingHeap was called;
+// subsequent GCs may reduce the heap below the requested size, but this
+// function does not affect that.
+//
+// logHeapTweaks (-d=gcadjust=1) enables logging of GOGC adjustment events.
+//
+// The temporarily requested GOGC is derated from what would be the "obvious"
+// value necessary to hit the starting heap goal because the obvious
+// (goal/live-1)*100 value seems to grow RSS a little more than it "should"
+// (compared to GOMEMLIMIT, e.g.) and the assumption is that the GC's control
+// algorithms are tuned for GOGC near 100, and not tuned for huge values of
+// GOGC. Different derating factors apply for "lo" and "hi" values of GOGC;
+// lo is below derateBreak, hi is above derateBreak. The derating factors,
+// expressed as integer percentages, are derateLoPct and derateHiPct.
+// 60-75 is an okay value for derateLoPct, 30-65 seems like a good value for
+// derateHiPct, and 600 seems like a good value for derateBreak. If these
+// are zero, defaults are used instead.
+//
+// NOTE: If you think this code would help startup time in your own
+// application and you decide to use it, please benchmark first to see if it
+// actually works for you (it may not: the Go compiler is not typical), and
+// whatever the outcome, please leave a comment on bug #56546. This code
+// uses supported interfaces, but depends more than we like on
+// current+observed behavior of the garbage collector, so if many people need
+// this feature, we should consider/propose a better way to accomplish it.
+func AdjustStartingHeap(requestedHeapGoal, derateBreak, derateLoPct, derateHiPct uint64, logHeapTweaks bool) {
+ mp := runtime.GOMAXPROCS(0)
+
+ const (
+ SHgoal = "/gc/heap/goal:bytes"
+ SHcount = "/gc/cycles/total:gc-cycles"
+ SHallocs = "/gc/heap/allocs:bytes"
+ SHfrees = "/gc/heap/frees:bytes"
+ )
+
+ var sample = []metrics.Sample{{Name: SHgoal}, {Name: SHcount}, {Name: SHallocs}, {Name: SHfrees}}
+
+ const (
+ SH_GOAL = 0
+ SH_COUNT = 1
+ SH_ALLOCS = 2
+ SH_FREES = 3
+
+ MB = 1_000_000
+ )
+
+ // These particular magic numbers are designed to make the RSS footprint of -d=-gcstart=2000
+ // resemble that of GOMEMLIMIT=2000MiB GOGC=10000 when building large projects
+ // (e.g. the Go compiler itself, and the microsoft's typescript AST package),
+ // with the further restriction that these magic numbers did a good job of reducing user-cpu
+ // for builds at either gcstart=2000 or gcstart=128.
+ //
+ // The benchmarking to obtain this was (a version of):
+ //
+ // for i in {1..50} ; do
+ // for what in std cmd/compile cmd/fix cmd/go github.com/microsoft/typescript-go/internal/ast ; do
+ // whatbase=`basename ${what}`
+ // for sh in 128 2000 ; do
+ // for br in 500 600 ; do
+ // for shlo in 65 70; do
+ // for shhi in 55 60 ; do
+ // benchcmd -n=2 ${whatbase} go build -a \
+ // -gcflags=all=-d=gcstart=${sh},gcstartloderate=${shlo},gcstarthiderate=${shhi},gcstartbreak=${br} \
+ // ${what} | tee -a startheap${sh}_${br}_${shhi}_${shlo}.bench
+ // done
+ // done
+ // done
+ // done
+ // done
+ // done
+ //
+ // benchcmd is "go install github.com/aclements/go-misc/benchcmd@latest"
+
+ if derateBreak == 0 {
+ derateBreak = 600
+ }
+ if derateLoPct == 0 {
+ derateLoPct = 70
+ }
+ if derateHiPct == 0 {
+ derateHiPct = 55
+ }
+
+ gogcDerate := func(myGogc uint64) uint64 {
+ if myGogc < derateBreak {
+ return (myGogc * derateLoPct) / 100
+ }
+ return (myGogc * derateHiPct) / 100
+ }
+
+ // Assumptions and observations of Go's garbage collector, as of Go 1.17-1.20:
+
+ // - the initial heap goal is 4MiB, by fiat. It is possible for Go to start
+ // with a heap as small as 512k, so this may change in the future.
+
+ // - except for the first heap goal, heap goal is a function of
+ // observed-live at the previous GC and current GOGC. After the first
+ // GC, adjusting GOGC immediately updates GOGC; before the first GC,
+ // adjusting GOGC does not modify goal (but the change takes effect after
+ // the first GC).
+
+ // - the before/after first GC behavior is not guaranteed anywhere, it's
+ // just behavior, and it's a bad idea to rely on it.
+
+ // - we don't know exactly when GC will run, even after we adjust GOGC; the
+ // first GC may not have happened yet, may have already happened, or may
+ // be currently in progress, and GCs can start for several reasons.
+
+ // - forEachGC above will run the provided function at some delay after each
+ // GC's mark phase terminates; finalizers are run after marking as the
+ // spans containing finalizable objects are swept, driven by GC
+ // background activity and allocation demand.
+
+ // - "live at last GC" is not available through the current metrics
+ // interface. Instead, live is estimated by knowing the adjusted value of
+ // GOGC and the new heap goal following a GC (this requires knowing that
+ // at least one GC has occurred):
+ // estLive = 100 * newGoal / (100 + currentGogc)
+ // this new value of GOGC
+ // newGogc = 100*requestedHeapGoal/estLive - 100
+ // will result in the desired goal. The logging code checks that the
+ // resulting goal is correct.
+
+ // There's a small risk that the finalizer will be slow to run after a GC
+ // that expands the goal to a huge value, and that this will lead to
+ // out-of-memory. This doesn't seem to happen; in experiments on a variety
+ // of machines with a variety of extra loads to disrupt scheduling, the
+ // worst overshoot observed was 50% past requestedHeapGoal.
+
+ metrics.Read(sample)
+ for _, s := range sample {
+ if s.Value.Kind() == metrics.KindBad {
+ // Just return, a slightly slower compilation is a tolerable outcome.
+ if logHeapTweaks {
+ fmt.Fprintf(os.Stderr, "GCAdjust: Regret unexpected KindBad for metric %s\n", s.Name)
+ }
+ return
+ }
+ }
+
+ // Tinker with GOGC to make the heap grow rapidly at first.
+ currentGoal := sample[SH_GOAL].Value.Uint64() // Believe this will be 4MByte or less, perhaps 512k
+ myGogc := 100 * requestedHeapGoal / currentGoal
+ myGogc = gogcDerate(myGogc)
+ if myGogc <= 125 {
+ return
+ }
+
+ if logHeapTweaks {
+ sample := append([]metrics.Sample(nil), sample...) // avoid races with GC callback
+ AtExit(func() {
+ metrics.Read(sample)
+ goal := sample[SH_GOAL].Value.Uint64()
+ count := sample[SH_COUNT].Value.Uint64()
+ oldGogc := debug.SetGCPercent(100)
+ if oldGogc == 100 {
+ fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %dMB gogc %d count %d maxprocs %d\n",
+ goal/MB, oldGogc, count, mp)
+ } else {
+ inUse := sample[SH_ALLOCS].Value.Uint64() - sample[SH_FREES].Value.Uint64()
+ overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
+ fmt.Fprintf(os.Stderr, "GCAdjust: AtExit goal %dMB gogc %d count %d maxprocs %d overPct %d\n",
+ goal/MB, oldGogc, count, mp, overPct)
+
+ }
+ })
+ }
+
+ originalGOGC := debug.SetGCPercent(int(myGogc))
+
+ // forEachGC finalizers ought not overlap, but they could run in separate threads.
+ // This ought not matter, but just in case it bothers the/a race detector,
+ // use this mutex.
+ var forEachGCLock sync.Mutex
+
+ adjustFunc := func() bool {
+
+ forEachGCLock.Lock()
+ defer forEachGCLock.Unlock()
+
+ metrics.Read(sample)
+ goal := sample[SH_GOAL].Value.Uint64()
+ count := sample[SH_COUNT].Value.Uint64()
+
+ if goal <= requestedHeapGoal { // Stay the course
+ if logHeapTweaks {
+ fmt.Fprintf(os.Stderr, "GCAdjust: Reuse GOGC adjust, current goal %dMB, count is %d, current gogc %d\n",
+ goal/MB, count, myGogc)
+ }
+ return true
+ }
+
+ // Believe goal has been adjusted upwards, else it would be less-than-or-equal to requestedHeapGoal
+ calcLive := 100 * goal / (100 + myGogc)
+
+ if 2*calcLive < requestedHeapGoal { // calcLive can exceed requestedHeapGoal!
+ myGogc = 100*requestedHeapGoal/calcLive - 100
+ myGogc = gogcDerate(myGogc)
+
+ if myGogc > 125 {
+ // Not done growing the heap.
+ oldGogc := debug.SetGCPercent(int(myGogc))
+
+ if logHeapTweaks {
+ // Check that the new goal looks right
+ inUse := sample[SH_ALLOCS].Value.Uint64() - sample[SH_FREES].Value.Uint64()
+ metrics.Read(sample)
+ newGoal := sample[SH_GOAL].Value.Uint64()
+ pctOff := 100 * (int64(newGoal) - int64(requestedHeapGoal)) / int64(requestedHeapGoal)
+ // Check that the new goal is close to requested. 3% of make.bash fails this test. Why, TBD.
+ if pctOff < 2 {
+ fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %dMB, count is %d, gogc was %d, is now %d, calcLive %dMB pctOff %d\n",
+ goal/MB, count, oldGogc, myGogc, calcLive/MB, pctOff)
+ } else {
+ // The GC is being annoying and not giving us the goal that we requested, say more to help understand when/why.
+ fmt.Fprintf(os.Stderr, "GCAdjust: Retry GOGC adjust, current goal %dMB, count is %d, gogc was %d, is now %d, calcLive %dMB pctOff %d inUse %dMB\n",
+ goal/MB, count, oldGogc, myGogc, calcLive/MB, pctOff, inUse/MB)
+ }
+ }
+ return true
+ }
+ }
+
+ // In this case we're done boosting GOGC, set it to its original value and don't set a new finalizer.
+ oldGogc := debug.SetGCPercent(originalGOGC)
+ // inUse helps estimate how late the finalizer ran; at the instant the previous GC ended,
+ // it was (in theory) equal to the previous GC's heap goal. In a growing heap it is
+ // expected to grow to the new heap goal.
+ if logHeapTweaks {
+ inUse := sample[SH_ALLOCS].Value.Uint64() - sample[SH_FREES].Value.Uint64()
+ overPct := 100 * (int(inUse) - int(requestedHeapGoal)) / int(requestedHeapGoal)
+ fmt.Fprintf(os.Stderr, "GCAdjust: Reset GOGC adjust, old goal %dMB, count is %d, gogc was %d, gogc is now %d, calcLive %dMB inUse %dMB overPct %d\n",
+ goal/MB, count, oldGogc, originalGOGC, calcLive/MB, inUse/MB, overPct)
+ }
+ return false
+ }
+
+ forEachGC(adjustFunc)
+}
diff --git a/src/cmd/compile/internal/base/timings.go b/src/cmd/compile/internal/base/timings.go
index f48ac93699b816..cbcd4dc6f55b55 100644
--- a/src/cmd/compile/internal/base/timings.go
+++ b/src/cmd/compile/internal/base/timings.go
@@ -168,7 +168,7 @@ type lines [][]string
func (lines *lines) add(label string, n int, dt, tot time.Duration, events []*event) {
var line []string
- add := func(format string, args ...interface{}) {
+ add := func(format string, args ...any) {
line = append(line, fmt.Sprintf(format, args...))
}
diff --git a/src/cmd/compile/internal/bitvec/bv.go b/src/cmd/compile/internal/bitvec/bv.go
index 9214aa6cd05560..57a8237ab08224 100644
--- a/src/cmd/compile/internal/bitvec/bv.go
+++ b/src/cmd/compile/internal/bitvec/bv.go
@@ -8,7 +8,6 @@ import (
"math/bits"
"cmd/compile/internal/base"
- "cmd/internal/src"
)
const (
@@ -34,12 +33,9 @@ type Bulk struct {
nword int32
}
-func NewBulk(nbit int32, count int32, pos src.XPos) Bulk {
+func NewBulk(nbit int32, count int32) Bulk {
nword := (nbit + wordBits - 1) / wordBits
size := int64(nword) * int64(count)
- if int64(int32(size*4)) != size*4 {
- base.FatalfAt(pos, "NewBulk too big: nbit=%d count=%d nword=%d size=%d", nbit, count, nword, size)
- }
return Bulk{
words: make([]uint32, size),
nbit: nbit,
diff --git a/src/cmd/compile/internal/bloop/bloop.go b/src/cmd/compile/internal/bloop/bloop.go
new file mode 100644
index 00000000000000..d5f001abf06ae1
--- /dev/null
+++ b/src/cmd/compile/internal/bloop/bloop.go
@@ -0,0 +1,366 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bloop
+
+// This file contains support routines for keeping
+// statements alive
+// in such loops (example):
+//
+// for b.Loop() {
+// var a, b int
+// a = 5
+// b = 6
+// f(a, b)
+// }
+//
+// The results of a, b and f(a, b) will be kept alive.
+//
+// Formally, the lhs (if they are [ir.Name]-s) of
+// [ir.AssignStmt], [ir.AssignListStmt],
+// [ir.AssignOpStmt], and the results of [ir.CallExpr]
+// or its args if it doesn't return a value will be kept
+// alive.
+//
+// The keep alive logic is implemented with as wrapping a
+// runtime.KeepAlive around the Name.
+//
+// TODO: currently this is implemented with KeepAlive
+// because it will prevent DSE and DCE which is probably
+// what we want right now. And KeepAlive takes an ssa
+// value instead of a symbol, which is easier to manage.
+// But since KeepAlive's context was mainly in the runtime
+// and GC, should we implement a new intrinsic that lowers
+// to OpVarLive? Peeling out the symbols is a bit tricky
+// and also VarLive seems to assume that there exists a
+// VarDef on the same symbol that dominates it.
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/ir"
+ "cmd/compile/internal/typecheck"
+ "cmd/compile/internal/types"
+ "cmd/internal/src"
+)
+
+// getNameFromNode tries to iteratively peel down the node to
+// get the name.
+func getNameFromNode(n ir.Node) *ir.Name {
+ // Tries to iteratively peel down the node to get the names.
+ for n != nil {
+ switch n.Op() {
+ case ir.ONAME:
+ // Found the name, stop the loop.
+ return n.(*ir.Name)
+ case ir.OSLICE, ir.OSLICE3:
+ n = n.(*ir.SliceExpr).X
+ case ir.ODOT:
+ n = n.(*ir.SelectorExpr).X
+ case ir.OCONV, ir.OCONVIFACE, ir.OCONVNOP:
+ n = n.(*ir.ConvExpr).X
+ case ir.OADDR:
+ n = n.(*ir.AddrExpr).X
+ case ir.ODOTPTR:
+ n = n.(*ir.SelectorExpr).X
+ case ir.OINDEX, ir.OINDEXMAP:
+ n = n.(*ir.IndexExpr).X
+ default:
+ n = nil
+ }
+ }
+ return nil
+}
+
+// getAddressableNameFromNode is like getNameFromNode but returns nil if the node is not addressable.
+func getAddressableNameFromNode(n ir.Node) *ir.Name {
+ if name := getNameFromNode(n); name != nil && ir.IsAddressable(name) {
+ return name
+ }
+ return nil
+}
+
+// getKeepAliveNodes analyzes an IR node and returns a list of nodes that must be kept alive.
+func getKeepAliveNodes(pos src.XPos, n ir.Node) ir.Nodes {
+ name := getAddressableNameFromNode(n)
+ if name != nil {
+ debugName(name, pos)
+ return ir.Nodes{name}
+ } else if deref, ok := n.(*ir.StarExpr); ok && deref != nil {
+ if base.Flag.LowerM > 1 {
+ base.WarnfAt(pos, "dereference will be kept alive")
+ }
+ return ir.Nodes{deref}
+ } else if base.Flag.LowerM > 1 {
+ base.WarnfAt(pos, "expr is unknown to bloop pass")
+ }
+ return nil
+}
+
+// keepAliveAt returns a statement that is either curNode, or a
+// block containing curNode followed by a call to runtime.KeepAlive for each
+// node in ns. These calls ensure that nodes in ns will be live until
+// after curNode's execution.
+func keepAliveAt(ns ir.Nodes, curNode ir.Node) ir.Node {
+ if len(ns) == 0 {
+ return curNode
+ }
+
+ pos := curNode.Pos()
+ calls := ir.Nodes{curNode}
+ for _, n := range ns {
+ if n == nil || n.Sym() == nil || n.Sym().IsBlank() {
+ continue
+ }
+ if !ir.IsAddressable(n) {
+ base.FatalfAt(n.Pos(), "keepAliveAt: node %v is not addressable", n)
+ }
+ arg := ir.NewConvExpr(pos, ir.OCONV, types.Types[types.TUNSAFEPTR], typecheck.NodAddr(n))
+ callExpr := typecheck.Call(pos, typecheck.LookupRuntime("KeepAlive"), ir.Nodes{arg}, false).(*ir.CallExpr)
+ callExpr.IsCompilerVarLive = true
+ callExpr.NoInline = true
+ calls = append(calls, callExpr)
+ }
+
+ return ir.NewBlockStmt(pos, calls)
+}
+
+func debugName(name *ir.Name, pos src.XPos) {
+ if base.Flag.LowerM > 1 {
+ if name.Linksym() != nil {
+ base.WarnfAt(pos, "%s will be kept alive", name.Linksym().Name)
+ } else {
+ base.WarnfAt(pos, "expr will be kept alive")
+ }
+ }
+}
+
+// preserveCallResults assigns the results of a call statement to temporary variables to ensure they remain alive.
+func preserveCallResults(curFn *ir.Func, call *ir.CallExpr) ir.Node {
+ var ns ir.Nodes
+ lhs := make(ir.Nodes, call.Fun.Type().NumResults())
+ for i, res := range call.Fun.Type().Results() {
+ tmp := typecheck.TempAt(call.Pos(), curFn, res.Type)
+ lhs[i] = tmp
+ ns = append(ns, tmp)
+ }
+
+ if base.Flag.LowerM > 1 {
+ plural := ""
+ if call.Fun.Type().NumResults() > 1 {
+ plural = "s"
+ }
+ base.WarnfAt(call.Pos(), "function result%s will be kept alive", plural)
+ }
+
+ assign := typecheck.AssignExpr(ir.NewAssignListStmt(call.Pos(), ir.OAS2, lhs, ir.Nodes{call})).(*ir.AssignListStmt)
+ assign.Def = true
+ for _, tmp := range lhs {
+ // Place temp declarations in the loop body to help escape analysis.
+ assign.PtrInit().Append(typecheck.Stmt(ir.NewDecl(assign.Pos(), ir.ODCL, tmp.(*ir.Name))))
+ }
+ return keepAliveAt(ns, assign)
+}
+
+// preserveCallArgs ensures the arguments of a call statement are kept alive by transforming them into temporaries if necessary.
+func preserveCallArgs(curFn *ir.Func, call *ir.CallExpr) ir.Node {
+ var argTmps ir.Nodes
+ var names ir.Nodes
+ preserveTmp := func(pos src.XPos, n ir.Node) ir.Node {
+ tmp := typecheck.TempAt(pos, curFn, n.Type())
+ assign := ir.NewAssignStmt(pos, tmp, n)
+ assign.Def = true
+ // Place temp declarations in the loop body to help escape analysis.
+ assign.PtrInit().Append(typecheck.Stmt(ir.NewDecl(assign.Pos(), ir.ODCL, tmp)))
+ argTmps = append(argTmps, typecheck.AssignExpr(assign))
+ names = append(names, tmp)
+ if base.Flag.LowerM > 1 {
+ base.WarnfAt(call.Pos(), "function arg will be kept alive")
+ }
+ return tmp
+ }
+ for i, a := range call.Args {
+ if name := getAddressableNameFromNode(a); name != nil {
+ // If they are name, keep them alive directly.
+ debugName(name, call.Pos())
+ names = append(names, name)
+ } else if a.Op() == ir.OSLICELIT {
+ // variadic args are encoded as slice literal.
+ s := a.(*ir.CompLitExpr)
+ var ns ir.Nodes
+ for i, elem := range s.List {
+ if name := getAddressableNameFromNode(elem); name != nil {
+ debugName(name, call.Pos())
+ ns = append(ns, name)
+ } else {
+ // We need a temporary to save this arg.
+ s.List[i] = preserveTmp(elem.Pos(), elem)
+ }
+ }
+ names = append(names, ns...)
+ } else {
+ // expressions, we need to assign them to temps and change the original arg to reference them.
+ call.Args[i] = preserveTmp(call.Pos(), a)
+ }
+ }
+ if len(argTmps) > 0 {
+ argTmps = append(argTmps, call)
+ return keepAliveAt(names, ir.NewBlockStmt(call.Pos(), argTmps))
+ }
+ return keepAliveAt(names, call)
+}
+
+// preserveStmt transforms stmt so that any names defined/assigned within it
+// are used after stmt's execution, preventing their dead code elimination
+// and dead store elimination. The return value is the transformed statement.
+func preserveStmt(curFn *ir.Func, stmt ir.Node) ir.Node {
+ switch n := stmt.(type) {
+ case *ir.AssignStmt:
+ // If the left hand side is blank, we need to assign it to a temp
+ // so that it can be kept alive.
+ if ir.IsBlank(n.X) {
+ tmp := typecheck.TempAt(n.Pos(), curFn, n.Y.Type())
+ n.X = tmp
+ n.Def = true
+ n.PtrInit().Append(typecheck.Stmt(ir.NewDecl(n.Pos(), ir.ODCL, tmp)))
+ stmt = typecheck.AssignExpr(n)
+ n = stmt.(*ir.AssignStmt)
+ }
+ return keepAliveAt(getKeepAliveNodes(n.Pos(), n.X), n)
+ case *ir.AssignListStmt:
+ var ns ir.Nodes
+ hasBlank := false
+ for i, lhs := range n.Lhs {
+ if ir.IsBlank(lhs) {
+ // If the left hand side has blanks, we need to assign them to temps
+ // so that they can be kept alive.
+ var typ *types.Type
+ // AssignListStmt can have tuple or a list of expressions on the right hand side.
+ if len(n.Rhs) == 1 && n.Rhs[0].Type() != nil &&
+ n.Rhs[0].Type().IsTuple() &&
+ len(n.Lhs) == n.Rhs[0].Type().NumFields() {
+ typ = n.Rhs[0].Type().Field(i).Type
+ } else if len(n.Rhs) == len(n.Lhs) {
+ typ = n.Rhs[i].Type()
+ } else {
+ // Unrecognized shapes, skip?
+ base.WarnfAt(n.Pos(), "unrecognized shape for assign list stmt for blank assignment")
+ continue
+ }
+ tmp := typecheck.TempAt(n.Pos(), curFn, typ)
+ n.Lhs[i] = tmp
+ n.PtrInit().Append(typecheck.Stmt(ir.NewDecl(n.Pos(), ir.ODCL, tmp)))
+ hasBlank = true
+ }
+ ns = append(ns, getKeepAliveNodes(n.Pos(), n.Lhs[i])...)
+ }
+ if hasBlank {
+ // blank nodes are rewritten to temps, we need to typecheck the node again.
+ n.Def = true
+ stmt = typecheck.AssignExpr(n)
+ n = stmt.(*ir.AssignListStmt)
+ }
+ return keepAliveAt(ns, n)
+ case *ir.AssignOpStmt:
+ return keepAliveAt(getKeepAliveNodes(n.Pos(), n.X), n)
+ case *ir.CallExpr:
+ // The function's results are not assigned, preserve them.
+ if n.Fun != nil && n.Fun.Type() != nil && n.Fun.Type().NumResults() != 0 {
+ return preserveCallResults(curFn, n)
+ }
+ // This function doesn't return anything, keep its args alive.
+ return preserveCallArgs(curFn, n)
+ }
+ return stmt
+}
+
+func preserveStmts(curFn *ir.Func, list ir.Nodes) {
+ for i := range list {
+ list[i] = preserveStmt(curFn, list[i])
+ }
+}
+
+// isTestingBLoop returns true if it matches the node as a
+// testing.(*B).Loop. See issue #61515.
+func isTestingBLoop(t ir.Node) bool {
+ if t.Op() != ir.OFOR {
+ return false
+ }
+ nFor, ok := t.(*ir.ForStmt)
+ if !ok || nFor.Cond == nil || nFor.Cond.Op() != ir.OCALLFUNC {
+ return false
+ }
+ n, ok := nFor.Cond.(*ir.CallExpr)
+ if !ok || n.Fun == nil || n.Fun.Op() != ir.OMETHEXPR {
+ return false
+ }
+ name := ir.MethodExprName(n.Fun)
+ if name == nil {
+ return false
+ }
+ if fSym := name.Sym(); fSym != nil && name.Class == ir.PFUNC && fSym.Pkg != nil &&
+ fSym.Name == "(*B).Loop" && fSym.Pkg.Path == "testing" {
+ // Attempting to match a function call to testing.(*B).Loop
+ return true
+ }
+ return false
+}
+
+type editor struct {
+ inBloop bool
+ curFn *ir.Func
+}
+
+func (e editor) edit(n ir.Node) ir.Node {
+ e.inBloop = isTestingBLoop(n) || e.inBloop
+ // It's in bloop, mark the stmts with bodies.
+ ir.EditChildren(n, e.edit)
+ if e.inBloop {
+ switch n := n.(type) {
+ case *ir.ForStmt:
+ preserveStmts(e.curFn, n.Body)
+ case *ir.IfStmt:
+ preserveStmts(e.curFn, n.Body)
+ preserveStmts(e.curFn, n.Else)
+ case *ir.BlockStmt:
+ preserveStmts(e.curFn, n.List)
+ case *ir.CaseClause:
+ preserveStmts(e.curFn, n.List)
+ preserveStmts(e.curFn, n.Body)
+ case *ir.CommClause:
+ preserveStmts(e.curFn, n.Body)
+ case *ir.RangeStmt:
+ preserveStmts(e.curFn, n.Body)
+ }
+ }
+ return n
+}
+
+// Walk performs a walk on all functions in the package
+// if it imports testing and wrap the results of all qualified
+// statements in a runtime.KeepAlive intrinsic call. See package
+// doc for more details.
+//
+// for b.Loop() {...}
+//
+// loop's body.
+func Walk(pkg *ir.Package) {
+ hasTesting := false
+ for _, i := range pkg.Imports {
+ if i.Path == "testing" {
+ hasTesting = true
+ break
+ }
+ }
+ if !hasTesting {
+ return
+ }
+ for _, fn := range pkg.Funcs {
+ e := editor{false, fn}
+ ir.EditChildren(fn, e.edit)
+ if ir.MatchAstDump(fn, "bloop") {
+ ir.AstDump(fn, "bloop, "+ir.FuncName(fn))
+ }
+ }
+
+}
diff --git a/src/cmd/compile/internal/compare/compare.go b/src/cmd/compile/internal/compare/compare.go
index 638eb37a622109..11793fc85ba102 100644
--- a/src/cmd/compile/internal/compare/compare.go
+++ b/src/cmd/compile/internal/compare/compare.go
@@ -257,8 +257,8 @@ func EqStruct(t *types.Type, np, nq ir.Node) ([]ir.Node, bool) {
func EqString(s, t ir.Node) (eqlen *ir.BinaryExpr, eqmem *ir.CallExpr) {
s = typecheck.Conv(s, types.Types[types.TSTRING])
t = typecheck.Conv(t, types.Types[types.TSTRING])
- sptr := ir.NewUnaryExpr(base.Pos, ir.OSPTR, s)
- tptr := ir.NewUnaryExpr(base.Pos, ir.OSPTR, t)
+ sptr := ir.NewConvExpr(base.Pos, ir.OCONVNOP, types.Types[types.TUNSAFEPTR], ir.NewUnaryExpr(base.Pos, ir.OSPTR, s))
+ tptr := ir.NewConvExpr(base.Pos, ir.OCONVNOP, types.Types[types.TUNSAFEPTR], ir.NewUnaryExpr(base.Pos, ir.OSPTR, t))
slen := typecheck.Conv(ir.NewUnaryExpr(base.Pos, ir.OLEN, s), types.Types[types.TUINTPTR])
tlen := typecheck.Conv(ir.NewUnaryExpr(base.Pos, ir.OLEN, t), types.Types[types.TUINTPTR])
@@ -293,7 +293,7 @@ func EqString(s, t ir.Node) (eqlen *ir.BinaryExpr, eqmem *ir.CallExpr) {
cmplen = tlen
}
- fn := typecheck.LookupRuntime("memequal", types.Types[types.TUINT8], types.Types[types.TUINT8])
+ fn := typecheck.LookupRuntime("memequal")
call := typecheck.Call(base.Pos, fn, []ir.Node{sptr, tptr, ir.Copy(cmplen)}, false).(*ir.CallExpr)
cmp := ir.NewBinaryExpr(base.Pos, ir.OEQ, slen, tlen)
diff --git a/src/cmd/compile/internal/deadlocals/deadlocals.go b/src/cmd/compile/internal/deadlocals/deadlocals.go
index 238450416a9463..9eeb1fb379faa3 100644
--- a/src/cmd/compile/internal/deadlocals/deadlocals.go
+++ b/src/cmd/compile/internal/deadlocals/deadlocals.go
@@ -23,7 +23,11 @@ func Funcs(fns []*ir.Func) {
zero := ir.NewBasicLit(base.AutogeneratedPos, types.Types[types.TINT], constant.MakeInt64(0))
for _, fn := range fns {
+
if fn.IsClosure() {
+ if ir.MatchAstDump(fn, "deadlocals closure") {
+ ir.AstDump(fn, "deadlocals closure skipped, "+ir.FuncName(fn))
+ }
continue
}
@@ -44,6 +48,14 @@ func Funcs(fns []*ir.Func) {
*as.lhs = ir.BlankNode
*as.rhs = zero
}
+ if len(assigns) > 0 {
+ // k.Defn might be pointing at one of the
+ // assignments we're overwriting.
+ k.Defn = nil
+ }
+ }
+ if ir.MatchAstDump(fn, "deadlocals") {
+ ir.AstDump(fn, "deadLocals, "+ir.FuncName(fn))
}
}
}
diff --git a/src/cmd/compile/internal/devirtualize/devirtualize.go b/src/cmd/compile/internal/devirtualize/devirtualize.go
index cb4608a0246574..08af63c39e8465 100644
--- a/src/cmd/compile/internal/devirtualize/devirtualize.go
+++ b/src/cmd/compile/internal/devirtualize/devirtualize.go
@@ -131,7 +131,7 @@ func StaticCall(s *State, call *ir.CallExpr) {
// type assertion that we make here would also have failed, but with a different
// panic "pkg.Iface is nil, not *pkg.Impl", where previously we would get a nil panic.
// We fix this, by introducing an additional nilcheck on the itab.
- // Calling a method on an nil interface (in most cases) is a bug in a program, so it is fine
+ // Calling a method on a nil interface (in most cases) is a bug in a program, so it is fine
// to devirtualize and further (possibly) inline them, even though we would never reach
// the called function.
dt.UseNilPanic = true
@@ -182,6 +182,17 @@ const concreteTypeDebug = false
// Returns nil when the concrete type could not be determined, or when there are multiple
// (different) types assigned to an interface.
func concreteType(s *State, n ir.Node) (typ *types.Type) {
+ if concreteTypeDebug {
+ base.Warn("concreteType(%v) - analyzing", n)
+ defer func() {
+ t := typ.String()
+ if typ == nil {
+ t = " (unknown static type)"
+ }
+ base.Warn("concreteType(%v) -> %v", n, t)
+ }()
+ }
+
typ = concreteType1(s, n, make(map[*ir.Name]struct{}))
if typ == &noType {
return nil
@@ -197,7 +208,7 @@ var noType types.Type
// concreteType1 analyzes the node n and returns its concrete type if it is statically known.
// Otherwise, it returns a nil Type, indicating that a concrete type was not determined.
-// When n is known to be statically nil or a self-assignment is detected, in returns a sentinel [noType] type instead.
+// When n is known to be statically nil or a self-assignment is detected, it returns a sentinel [noType] type instead.
func concreteType1(s *State, n ir.Node, seen map[*ir.Name]struct{}) (outT *types.Type) {
nn := n // for debug messages
@@ -207,6 +218,9 @@ func concreteType1(s *State, n ir.Node, seen map[*ir.Name]struct{}) (outT *types
if outT != &noType {
t = outT.String()
}
+ if outT == nil {
+ t = " (unknown static type)"
+ }
base.Warn("concreteType1(%v) -> %v", nn, t)
}()
}
@@ -293,9 +307,22 @@ func concreteType1(s *State, n ir.Node, seen map[*ir.Name]struct{}) (outT *types
continue
}
}
- if t == nil || (typ != nil && !types.Identical(typ, t)) {
- return nil
+ if t == nil {
+ return nil // unknown concrete type
}
+
+ // Methods are only declared on named types, and each named type
+ // is represented by a unique [*types.Type], thus pointer comparison
+ // is fine here.
+ //
+ // The only scenario where [types.IdenticalStrict] could help here is with
+ // unnamed struct types that embed another type (e.g. foo = struct { Impl }{}).
+ // However, such patterns are uncommon and not worth the additional complexity
+ // in the devirtualizer.
+ if typ != nil && typ != t {
+ return nil // assigned with a different type
+ }
+
typ = t
}
@@ -310,7 +337,7 @@ func concreteType1(s *State, n ir.Node, seen map[*ir.Name]struct{}) (outT *types
// assignment can be one of:
// - nil - assignment from an interface type.
// - *types.Type - assignment from a concrete type (non-interface).
-// - ir.Node - assignment from a ir.Node.
+// - ir.Node - assignment from an ir.Node.
//
// In most cases assignment should be an [ir.Node], but in cases where we
// do not follow the data-flow, we return either a concrete type (*types.Type) or a nil.
@@ -560,8 +587,8 @@ func (s *State) analyze(nodes ir.Nodes) {
assign(n.Key, nil)
assign(n.Value, nil)
} else {
- // We will not reach here in case of an range-over-func, as it is
- // rewrtten to function calls in the noder package.
+ // We will not reach here in case of a range-over-func, as it is
+ // rewritten to function calls in the noder package.
base.FatalfAt(n.Pos(), "range over unexpected type %v", n.X.Type())
}
case ir.OSWITCH:
diff --git a/src/cmd/compile/internal/devirtualize/pgo.go b/src/cmd/compile/internal/devirtualize/pgo.go
index a8980bb86b1547..5bb0454f8b5ee6 100644
--- a/src/cmd/compile/internal/devirtualize/pgo.go
+++ b/src/cmd/compile/internal/devirtualize/pgo.go
@@ -484,6 +484,7 @@ func condCall(curfn *ir.Func, pos src.XPos, cond ir.Node, thenCall, elseCall *ir
res := ir.NewInlinedCallExpr(pos, body, retvars)
res.SetType(thenCall.Type())
res.SetTypecheck(1)
+ res.Reshape = thenCall.Reshape
return res
}
diff --git a/src/cmd/compile/internal/dwarfgen/dwarf.go b/src/cmd/compile/internal/dwarfgen/dwarf.go
index 9d975e0bc1ac7d..3f654fabee7e97 100644
--- a/src/cmd/compile/internal/dwarfgen/dwarf.go
+++ b/src/cmd/compile/internal/dwarfgen/dwarf.go
@@ -93,7 +93,7 @@ func Info(ctxt *obj.Link, fnsym *obj.LSym, infosym *obj.LSym, curfn obj.Func) (s
default:
continue
}
- if !ssa.IsVarWantedForDebug(n) {
+ if !shouldEmitDwarfVar(n) {
continue
}
apdecls = append(apdecls, n)
@@ -178,18 +178,78 @@ func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir
// Collect a raw list of DWARF vars.
var vars []*dwarf.Var
var decls []*ir.Name
- var selected ir.NameSet
- if base.Ctxt.Flag_locationlists && base.Ctxt.Flag_optimize && fn.DebugInfo != nil && complexOK {
- decls, vars, selected = createComplexVars(fnsym, fn, closureVars)
- } else if fn.ABI == obj.ABIInternal && base.Flag.N != 0 && complexOK {
- decls, vars, selected = createABIVars(fnsym, fn, apDecls, closureVars)
- } else {
- decls, vars, selected = createSimpleVars(fnsym, apDecls, closureVars)
- }
+ // Build a VarID lookup map for SSA debug info if available.
+ var debug *ssa.FuncDebug
+ var varIDMap map[*ir.Name]ssa.VarID
if fn.DebugInfo != nil {
- // Recover zero sized variables eliminated by the stackframe pass
- for _, n := range fn.DebugInfo.(*ssa.FuncDebug).OptDcl {
+ debug = fn.DebugInfo.(*ssa.FuncDebug)
+ varIDMap = make(map[*ir.Name]ssa.VarID, len(debug.Vars))
+ for i, n := range debug.Vars {
+ varIDMap[n] = ssa.VarID(i)
+ }
+ }
+ canUseComplex := complexOK && debug != nil
+
+ // markVarSeen marks a variable and all its associated slot names as seen.
+ // This is needed because decomposed variables may have slots whose ir.Name
+ // differs from the variable itself (e.g., PAUTO vs PPARAMOUT for the same
+ // logical variable). Without this, the dcl loop could create duplicate
+ // conservative entries for names that are already covered by a complex var.
+ seen := make(map[*ir.Name]bool)
+ markVarSeen := func(n *ir.Name, varID ssa.VarID) {
+ seen[n] = true
+ if debug != nil && int(varID) < len(debug.VarSlots) {
+ for _, slot := range debug.VarSlots[varID] {
+ seen[debug.Slots[slot].N] = true
+ }
+ }
+ }
+
+ // Unified loop: for each variable in apDecls, try createComplexVar
+ // (SSA debug info) first, then fall back to createSimpleVar.
+ for _, n := range apDecls {
+ if !shouldEmitDwarfVar(n) {
+ continue
+ }
+ if canUseComplex {
+ if vid, ok := varIDMap[n]; ok {
+ if dvar := createComplexVar(fnsym, fn, vid, closureVars); dvar != nil {
+ decls = append(decls, n)
+ vars = append(vars, dvar)
+ markVarSeen(n, vid)
+ continue
+ }
+ }
+ }
+ seen[n] = true
+ decls = append(decls, n)
+ vars = append(vars, createSimpleVar(fnsym, n, closureVars))
+ }
+
+ // Add SSA-tracked vars not in apDecls.
+ if canUseComplex {
+ for i, n := range debug.Vars {
+ if seen[n] {
+ continue
+ }
+ if !shouldEmitDwarfVar(n) {
+ continue
+ }
+ if dvar := createComplexVar(fnsym, fn, ssa.VarID(i), closureVars); dvar != nil {
+ decls = append(decls, n)
+ vars = append(vars, dvar)
+ markVarSeen(n, ssa.VarID(i))
+ }
+ }
+ }
+
+ // Recover zero-sized variables eliminated by the stackframe pass.
+ if debug != nil {
+ for _, n := range debug.OptDcl {
+ if seen[n] {
+ continue
+ }
if n.Class != ir.PAUTO {
continue
}
@@ -199,21 +259,23 @@ func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir
vars = append(vars, createSimpleVar(fnsym, n, closureVars))
vars[len(vars)-1].StackOffset = 0
fnsym.Func().RecordAutoType(reflectdata.TypeLinksym(n.Type()))
+ seen[n] = true
}
}
}
+ // For inlined functions or functions with register output params,
+ // collect additional declarations that may not be in apDecls.
dcl := apDecls
if fnsym.WasInlined() {
dcl = preInliningDcls(fnsym)
- } else {
+ } else if debug != nil {
// The backend's stackframe pass prunes away entries from the
// fn's Dcl list, including PARAMOUT nodes that correspond to
// output params passed in registers. Add back in these
// entries here so that we can process them properly during
// DWARF-gen. See issue 48573 for more details.
- debugInfo := fn.DebugInfo.(*ssa.FuncDebug)
- for _, n := range debugInfo.RegOutputParams {
+ for _, n := range debug.RegOutputParams {
if !ssa.IsVarWantedForDebug(n) {
continue
}
@@ -224,83 +286,33 @@ func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir
}
}
- // If optimization is enabled, the list above will typically be
- // missing some of the original pre-optimization variables in the
- // function (they may have been promoted to registers, folded into
- // constants, dead-coded away, etc). Input arguments not eligible
- // for SSA optimization are also missing. Here we add back in entries
- // for selected missing vars. Note that the recipe below creates a
- // conservative location. The idea here is that we want to
- // communicate to the user that "yes, there is a variable named X
- // in this function, but no, I don't have enough information to
- // reliably report its contents."
- // For non-SSA-able arguments, however, the correct information
- // is known -- they have a single home on the stack.
+ // Process remaining variables not yet handled. For each variable,
+ // try createComplexVar first, then fall back to createSimpleVar
+ // for non-SSA-able params, or createConservativeVar for the rest.
for _, n := range dcl {
- if selected.Has(n) {
+ if seen[n] {
continue
}
- c := n.Sym().Name[0]
- if c == '.' || n.Type().IsUntyped() {
+ if !shouldEmitDwarfVar(n) {
continue
}
+ seen[n] = true
+ if canUseComplex {
+ if vid, ok := varIDMap[n]; ok {
+ if dvar := createComplexVar(fnsym, fn, vid, closureVars); dvar != nil {
+ decls = append(decls, n)
+ vars = append(vars, dvar)
+ continue
+ }
+ }
+ }
if n.Class == ir.PPARAM && !ssa.CanSSA(n.Type()) {
- // SSA-able args get location lists, and may move in and
- // out of registers, so those are handled elsewhere.
- // Autos and named output params seem to get handled
- // with VARDEF, which creates location lists.
- // Args not of SSA-able type are treated here; they
- // are homed on the stack in a single place for the
- // entire call.
- vars = append(vars, createSimpleVar(fnsym, n, closureVars))
decls = append(decls, n)
+ vars = append(vars, createSimpleVar(fnsym, n, closureVars))
continue
}
- typename := dwarf.InfoPrefix + types.TypeSymName(n.Type())
decls = append(decls, n)
- tag := dwarf.DW_TAG_variable
- isReturnValue := (n.Class == ir.PPARAMOUT)
- if n.Class == ir.PPARAM || n.Class == ir.PPARAMOUT {
- tag = dwarf.DW_TAG_formal_parameter
- }
- inlIndex := 0
- if base.Flag.GenDwarfInl > 1 {
- if n.InlFormal() || n.InlLocal() {
- inlIndex = posInlIndex(n.Pos()) + 1
- if n.InlFormal() {
- tag = dwarf.DW_TAG_formal_parameter
- }
- }
- }
- declpos := base.Ctxt.InnermostPos(n.Pos())
- dvar := &dwarf.Var{
- Name: n.Sym().Name,
- IsReturnValue: isReturnValue,
- Tag: tag,
- WithLoclist: true,
- StackOffset: int32(n.FrameOffset()),
- Type: base.Ctxt.Lookup(typename),
- DeclFile: declpos.RelFilename(),
- DeclLine: declpos.RelLine(),
- DeclCol: declpos.RelCol(),
- InlIndex: int32(inlIndex),
- ChildIndex: -1,
- DictIndex: n.DictIndex,
- ClosureOffset: closureOffset(n, closureVars),
- }
- if n.Esc() == ir.EscHeap {
- if n.Heapaddr == nil {
- base.Fatalf("invalid heap allocated var without Heapaddr")
- }
- debug := fn.DebugInfo.(*ssa.FuncDebug)
- list := createHeapDerefLocationList(n, debug.EntryID)
- dvar.PutLocationList = func(listSym, startPC dwarf.Sym) {
- debug.PutLocationList(list, base.Ctxt, listSym.(*obj.LSym), startPC.(*obj.LSym))
- }
- }
- vars = append(vars, dvar)
- // Record go type to ensure that it gets emitted by the linker.
- fnsym.Func().RecordAutoType(reflectdata.TypeLinksym(n.Type()))
+ vars = append(vars, createConservativeVar(fnsym, fn, n, closureVars))
}
// Sort decls and vars.
@@ -309,6 +321,64 @@ func createDwarfVars(fnsym *obj.LSym, complexOK bool, fn *ir.Func, apDecls []*ir
return decls, vars
}
+// createConservativeVar creates a DWARF variable with a conservative location
+// description. This is used for variables that were optimized away or otherwise
+// don't have precise location info. The intent is to communicate that "yes,
+// there is a variable named X in this function, but no, I don't have enough
+// information to reliably report its contents."
+// For heap-escaped variables, a location list is created that describes
+// dereferencing the pointer at the stack offset.
+func createConservativeVar(fnsym *obj.LSym, fn *ir.Func, n *ir.Name, closureVars map[*ir.Name]int64) *dwarf.Var {
+ typename := dwarf.InfoPrefix + types.TypeSymName(n.Type())
+ tag := dwarf.DW_TAG_variable
+ isReturnValue := (n.Class == ir.PPARAMOUT)
+ if n.Class == ir.PPARAM || n.Class == ir.PPARAMOUT {
+ tag = dwarf.DW_TAG_formal_parameter
+ }
+ inlIndex := 0
+ if base.Flag.GenDwarfInl > 1 {
+ if n.InlFormal() || n.InlLocal() {
+ inlIndex = posInlIndex(n.Pos()) + 1
+ if n.InlFormal() {
+ tag = dwarf.DW_TAG_formal_parameter
+ }
+ }
+ }
+ declpos := base.Ctxt.InnermostPos(n.Pos())
+ dvar := &dwarf.Var{
+ Name: n.Sym().Name,
+ IsReturnValue: isReturnValue,
+ Tag: tag,
+ WithLoclist: true,
+ StackOffset: int32(n.FrameOffset()),
+ Type: base.Ctxt.Lookup(typename),
+ DeclFile: declpos.RelFilename(),
+ DeclLine: declpos.RelLine(),
+ DeclCol: declpos.RelCol(),
+ InlIndex: int32(inlIndex),
+ ChildIndex: -1,
+ DictIndex: n.DictIndex,
+ ClosureOffset: closureOffset(n, closureVars),
+ }
+ if n.Esc() == ir.EscHeap && n.Heapaddr != nil {
+ // The variable was promoted to the heap and has a known heap
+ // address, so describe its location by dereferencing the pointer
+ // stored at its stack offset. A heap-escaped variable may have no
+ // Heapaddr if it was declared in unreachable code: escape analysis
+ // marks it as heap-allocated, but SSA generation skips the dead
+ // declaration and never allocates the address. In that case fall
+ // through and emit a conservative variable with no location list.
+ debug := fn.DebugInfo.(*ssa.FuncDebug)
+ list := createHeapDerefLocationList(n, debug.EntryID)
+ dvar.PutLocationList = func(listSym, startPC dwarf.Sym) {
+ debug.PutLocationList(list, base.Ctxt, listSym.(*obj.LSym), startPC.(*obj.LSym))
+ }
+ }
+ // Record go type to ensure that it gets emitted by the linker.
+ fnsym.Func().RecordAutoType(reflectdata.TypeLinksym(n.Type()))
+ return dvar
+}
+
// sortDeclsAndVars sorts the decl and dwarf var lists according to
// parameter declaration order, so as to insure that when a subprogram
// DIE is emitted, its parameter children appear in declaration order.
@@ -368,10 +438,7 @@ func preInliningDcls(fnsym *obj.LSym) []*ir.Name {
fn := base.Ctxt.DwFixups.GetPrecursorFunc(fnsym).(*ir.Func)
var rdcl []*ir.Name
for _, n := range fn.Inl.Dcl {
- c := n.Sym().Name[0]
- // Avoid reporting "_" parameters, since if there are more than
- // one, it can result in a collision later on, as in #23179.
- if n.Sym().Name == "_" || c == '.' || n.Type().IsUntyped() {
+ if n.Sym().Name[0] == '.' || !shouldEmitDwarfVarSafe(n) {
continue
}
rdcl = append(rdcl, n)
@@ -379,24 +446,6 @@ func preInliningDcls(fnsym *obj.LSym) []*ir.Name {
return rdcl
}
-// createSimpleVars creates a DWARF entry for every variable declared in the
-// function, claiming that they are permanently on the stack.
-func createSimpleVars(fnsym *obj.LSym, apDecls []*ir.Name, closureVars map[*ir.Name]int64) ([]*ir.Name, []*dwarf.Var, ir.NameSet) {
- var vars []*dwarf.Var
- var decls []*ir.Name
- var selected ir.NameSet
- for _, n := range apDecls {
- if ir.IsAutoTmp(n) {
- continue
- }
-
- decls = append(decls, n)
- vars = append(vars, createSimpleVar(fnsym, n, closureVars))
- selected.Add(n)
- }
- return decls, vars, selected
-}
-
func createSimpleVar(fnsym *obj.LSym, n *ir.Name, closureVars map[*ir.Name]int64) *dwarf.Var {
var tag int
var offs int64
@@ -457,63 +506,6 @@ func createSimpleVar(fnsym *obj.LSym, n *ir.Name, closureVars map[*ir.Name]int64
}
}
-// createABIVars creates DWARF variables for functions in which the
-// register ABI is enabled but optimization is turned off. It uses a
-// hybrid approach in which register-resident input params are
-// captured with location lists, and all other vars use the "simple"
-// strategy.
-func createABIVars(fnsym *obj.LSym, fn *ir.Func, apDecls []*ir.Name, closureVars map[*ir.Name]int64) ([]*ir.Name, []*dwarf.Var, ir.NameSet) {
-
- // Invoke createComplexVars to generate dwarf vars for input parameters
- // that are register-allocated according to the ABI rules.
- decls, vars, selected := createComplexVars(fnsym, fn, closureVars)
-
- // Now fill in the remainder of the variables: input parameters
- // that are not register-resident, output parameters, and local
- // variables.
- for _, n := range apDecls {
- if ir.IsAutoTmp(n) {
- continue
- }
- if _, ok := selected[n]; ok {
- // already handled
- continue
- }
-
- decls = append(decls, n)
- vars = append(vars, createSimpleVar(fnsym, n, closureVars))
- selected.Add(n)
- }
-
- return decls, vars, selected
-}
-
-// createComplexVars creates recomposed DWARF vars with location lists,
-// suitable for describing optimized code.
-func createComplexVars(fnsym *obj.LSym, fn *ir.Func, closureVars map[*ir.Name]int64) ([]*ir.Name, []*dwarf.Var, ir.NameSet) {
- debugInfo := fn.DebugInfo.(*ssa.FuncDebug)
-
- // Produce a DWARF variable entry for each user variable.
- var decls []*ir.Name
- var vars []*dwarf.Var
- var ssaVars ir.NameSet
-
- for varID, dvar := range debugInfo.Vars {
- n := dvar
- ssaVars.Add(n)
- for _, slot := range debugInfo.VarSlots[varID] {
- ssaVars.Add(debugInfo.Slots[slot].N)
- }
-
- if dvar := createComplexVar(fnsym, fn, ssa.VarID(varID), closureVars); dvar != nil {
- decls = append(decls, n)
- vars = append(vars, dvar)
- }
- }
-
- return decls, vars, ssaVars
-}
-
// createComplexVar builds a single DWARF variable entry and location list.
func createComplexVar(fnsym *obj.LSym, fn *ir.Func, varID ssa.VarID, closureVars map[*ir.Name]int64) *dwarf.Var {
debug := fn.DebugInfo.(*ssa.FuncDebug)
@@ -573,7 +565,7 @@ func createComplexVar(fnsym *obj.LSym, fn *ir.Func, varID ssa.VarID, closureVars
// createHeapDerefLocationList creates a location list for a heap-escaped variable
// that describes "dereference pointer at stack offset"
-func createHeapDerefLocationList(n *ir.Name, entryID ssa.ID) []byte {
+func createHeapDerefLocationList(n *ir.Name, entryID ssa.ID) []ssa.LocListEntry {
// Get the stack offset where the heap pointer is stored
heapPtrOffset := n.Heapaddr.FrameOffset()
if base.Ctxt.Arch.FixedFrameSize == 0 {
@@ -584,14 +576,18 @@ func createHeapDerefLocationList(n *ir.Name, entryID ssa.ID) []byte {
}
// Create a location expression: DW_OP_fbreg DW_OP_deref
- var locExpr []byte
- var sizeIdx int
- locExpr, sizeIdx = ssa.SetupLocList(base.Ctxt, entryID, locExpr, ssa.BlockStart.ID, ssa.FuncEnd.ID)
- locExpr = append(locExpr, dwarf.DW_OP_fbreg)
- locExpr = dwarf.AppendSleb128(locExpr, heapPtrOffset)
- locExpr = append(locExpr, dwarf.DW_OP_deref)
- base.Ctxt.Arch.ByteOrder.PutUint16(locExpr[sizeIdx:], uint16(len(locExpr)-sizeIdx-2))
- return locExpr
+ var expr []byte
+ expr = append(expr, dwarf.DW_OP_fbreg)
+ expr = dwarf.AppendSleb128(expr, heapPtrOffset)
+ expr = append(expr, dwarf.DW_OP_deref)
+
+ return []ssa.LocListEntry{{
+ StartBlock: entryID,
+ StartValue: ssa.BlockStart.ID,
+ EndBlock: entryID,
+ EndValue: ssa.FuncEnd.ID,
+ Expr: expr,
+ }}
}
// RecordFlags records the specified command-line flags to be placed
@@ -667,6 +663,34 @@ func RecordPackageName() {
s.P = []byte(types.LocalPkg.Name)
}
+// shouldEmitDwarfVar reports whether n should have a DWARF variable entry.
+// This consolidates filtering that was previously spread across IR (AutoTemp),
+// SSA (IsVarWantedForDebug), and dwarfgen (symbol name checks).
+func shouldEmitDwarfVar(n *ir.Name) bool {
+ if ir.IsAutoTmp(n) {
+ return false
+ }
+ return shouldEmitDwarfVarSafe(n)
+}
+
+// shouldEmitDwarfVarSafe is like shouldEmitDwarfVar but omits the ir.IsAutoTmp
+// check, making it safe to call during parallel compilation on shared ir.Name
+// nodes (e.g., in preInliningDcls). ir.IsAutoTmp reads the mutable flags bitset,
+// which can race with other goroutines writing different flags during compilation.
+// Auto temps have names starting with "." so callers must filter those separately.
+func shouldEmitDwarfVarSafe(n *ir.Name) bool {
+ if !ssa.IsVarWantedForDebug(n) {
+ return false
+ }
+ if n.Sym().Name == "_" {
+ return false
+ }
+ if n.Type().IsUntyped() {
+ return false
+ }
+ return true
+}
+
func closureOffset(n *ir.Name, closureVars map[*ir.Name]int64) int64 {
return closureVars[n]
}
diff --git a/src/cmd/compile/internal/dwarfgen/marker.go b/src/cmd/compile/internal/dwarfgen/marker.go
index ec6ce45a900bc0..dc622899d9c88e 100644
--- a/src/cmd/compile/internal/dwarfgen/marker.go
+++ b/src/cmd/compile/internal/dwarfgen/marker.go
@@ -20,7 +20,7 @@ type ScopeMarker struct {
// checkPos validates the given position and returns the current scope.
func (m *ScopeMarker) checkPos(pos src.XPos) ir.ScopeID {
if !pos.IsKnown() {
- base.Fatalf("unknown scope position")
+ base.Fatalf("unknown scope position, pos=%v", pos)
}
if len(m.marks) == 0 {
diff --git a/src/cmd/compile/internal/escape/alias.go b/src/cmd/compile/internal/escape/alias.go
new file mode 100644
index 00000000000000..f0351e8f671441
--- /dev/null
+++ b/src/cmd/compile/internal/escape/alias.go
@@ -0,0 +1,528 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package escape
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/ir"
+ "cmd/internal/src"
+ "fmt"
+ "maps"
+ "path/filepath"
+)
+
+type aliasAnalysis struct {
+ // fn is the function being analyzed.
+ fn *ir.Func
+
+ // candidateSlices are declared slices that
+ // start unaliased and might still be unaliased.
+ candidateSlices map[*ir.Name]candidateSlice
+
+ // noAliasAppends are appends that have been
+ // proven to use an unaliased slice.
+ noAliasAppends []*ir.CallExpr
+
+ // loops is a stack of observed loops,
+ // each with a list of candidate appends.
+ loops [][]candidateAppend
+
+ // State for optional validation checking (doubleCheck mode):
+ processed map[ir.Node]int // count of times each node was processed, for doubleCheck mode
+ doubleCheck bool // whether to do doubleCheck mode
+}
+
+// candidateSlice tracks information about a declared slice
+// that might be unaliased.
+type candidateSlice struct {
+ loopDepth int // depth of loop when slice was declared
+}
+
+// candidateAppend tracks information about an OAPPEND that
+// might be using an unaliased slice.
+type candidateAppend struct {
+ s *ir.Name // the slice argument in 's = append(s, ...)'
+ call *ir.CallExpr // the append call
+}
+
+// aliasAnalysis looks for specific patterns of slice usage and proves
+// that certain appends are operating on non-aliased slices.
+//
+// This allows us to emit calls to free the backing arrays for certain
+// non-aliased slices at runtime when we know the memory is logically dead.
+//
+// The analysis is conservative, giving up on any operation we do not
+// explicitly understand.
+func (aa *aliasAnalysis) analyze(fn *ir.Func) {
+ // Walk the function body to discover slice declarations, their uses,
+ // and any append that we can prove is using an unaliased slice.
+ //
+ // An example is:
+ //
+ // var s []T
+ // for _, v := range input {
+ // f()
+ // s = append(s, g(v)) // s cannot be aliased here
+ // h()
+ // }
+ // return s
+ //
+ // Here, we can prove that the append to s is operating on an unaliased slice,
+ // and that conclusion is unaffected by s later being returned and escaping.
+ //
+ // In contrast, in this example, the aliasing of s in the loop body means the
+ // append can be operating on an aliased slice, so we do not record s as unaliased:
+ //
+ // var s []T
+ // var alias []T
+ // for _, v := range input {
+ // s = append(s, v) // s is aliased on second pass through loop body
+ // alias = s
+ // }
+ //
+ // Arbitrary uses of s after an append do not affect the aliasing conclusion
+ // for that append, but only if the append cannot be revisited at execution time
+ // via a loop or goto.
+ //
+ // We track the loop depth when a slice was declared and verify all uses of a slice
+ // are non-aliasing until we return to that depth. In other words, we make sure
+ // we have processed any possible execution-time revisiting of the slice prior
+ // to making our final determination.
+ //
+ // This approach helps for example with nested loops, such as:
+ //
+ // var s []int
+ // for range 10 {
+ // for range 10 {
+ // s = append(s, 0) // s is proven as non-aliased here
+ // }
+ // }
+ // alias = s // both loops are complete
+ //
+ // Or in contrast:
+ //
+ // var s []int
+ // for range 10 {
+ // for range 10 {
+ // s = append(s, 0) // s is treated as aliased here
+ // }
+ // alias = s // aliased, and outermost loop cycles back
+ // }
+ //
+ // As we walk the function, we look for things like:
+ //
+ // 1. Slice declarations (currently supporting 'var s []T', 's := make([]T, ...)',
+ // and 's := []T{...}').
+ // 2. Appends to a slice of the form 's = append(s, ...)'.
+ // 3. Other uses of the slice, which we treat as potential aliasing outside
+ // of a few known safe cases.
+ // 4. A start of a loop, which we track in a stack so that
+ // any uses of a slice within a loop body are treated as potential
+ // aliasing, including statements in the loop body after an append.
+ // Candidate appends are stored in the loop stack at the loop depth of their
+ // corresponding slice declaration (rather than the loop depth of the append),
+ // which essentially postpones a decision about the candidate append.
+ // 5. An end of a loop, which pops the loop stack and allows us to
+ // conclusively treat candidate appends from the loop body based
+ // on the loop depth of the slice declaration.
+ //
+ // Note that as we pop a candidate append at the end of a loop, we know
+ // its corresponding slice was unaliased throughout the loop being popped
+ // if the slice is still in the candidate slice map (without having been
+ // removed for potential aliasing), and we know we can make a final decision
+ // about a candidate append if we have returned to the loop depth
+ // where its slice was declared. In other words, there is no unanalyzed
+ // control flow that could take us back at execution-time to the
+ // candidate append in the now analyzed loop. This helps for example
+ // with nested loops, such as in our examples just above.
+ //
+ // We give up on a particular candidate slice if we see any use of it
+ // that we don't explicitly understand, and we give up on all of
+ // our candidate slices if we see any goto or label, which could be
+ // unstructured control flow. (TODO(thepudds): we remove the goto/label
+ // restriction in a subsequent CL.)
+ //
+ // Note that the intended use is to indicate that a slice is safe to pass
+ // to runtime.freegc, which currently requires that the passed pointer
+ // point to the base of its heap object.
+ //
+ // Therefore, we currently do not allow any re-slicing of the slice, though we could
+ // potentially allow s[0:x] or s[:x] or similar. (Slice expressions that alter
+ // the capacity might be possible to allow with freegc changes, though they are
+ // currently disallowed here like all slice expressions).
+ //
+ // TODO(thepudds): we could support the slice being used as non-escaping function call parameter
+ // but to do that, we need to verify any creation of specials via user code triggers an escape,
+ // or mail better runtime.freegc support for specials, or have a temporary compile-time solution
+ // for specials. (Currently, this analysis side-steps specials because any use of a slice
+ // that might cause a user-created special will cause it to be treated as aliased, and
+ // separately, runtime.freegc handles profiling-related specials).
+
+ // Initialize.
+ aa.fn = fn
+ aa.candidateSlices = make(map[*ir.Name]candidateSlice) // slices that might be unaliased
+
+ // doubleCheck controls whether we do a sanity check of our processing logic
+ // by counting each node visited in our main pass, and then comparing those counts
+ // against a simple walk at the end. The main intent is to help catch missing
+ // any nodes squirreled away in some spot we forgot to examine in our main pass.
+ aa.doubleCheck = base.Debug.EscapeAliasCheck > 0
+ aa.processed = make(map[ir.Node]int)
+
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(fn.Pos(), fn, "====== starting func", "======")
+ }
+
+ ir.DoChildren(fn, aa.visit)
+
+ for _, call := range aa.noAliasAppends {
+ if base.Debug.EscapeAlias >= 1 {
+ base.WarnfAt(call.Pos(), "alias analysis: append using non-aliased slice: %v in func %v",
+ call, fn)
+ }
+ if base.Debug.FreeAppend > 0 {
+ call.AppendNoAlias = true
+ }
+ }
+
+ if aa.doubleCheck {
+ doubleCheckProcessed(fn, aa.processed)
+ }
+}
+
+func (aa *aliasAnalysis) visit(n ir.Node) bool {
+ if n == nil {
+ return false
+ }
+
+ if base.Debug.EscapeAlias >= 3 {
+ fmt.Printf("%-25s alias analysis: visiting node: %12s %-18T %v\n",
+ fmtPosShort(n.Pos())+":", n.Op().String(), n, n)
+ }
+
+ // As we visit nodes, we want to ensure we handle all children
+ // without missing any (through ignorance or future changes).
+ // We do this by counting nodes as we visit them or otherwise
+ // declare a node to be fully processed.
+ //
+ // In particular, we want to ensure we don't miss the use
+ // of a slice in some expression that might be an aliasing usage.
+ //
+ // When doubleCheck is enabled, we compare the counts
+ // accumulated in our analysis against counts from a trivial walk,
+ // failing if there is any mismatch.
+ //
+ // This call here counts that we have visited this node n
+ // via our main visit method. (In contrast, some nodes won't
+ // be visited by the main visit method, but instead will be
+ // manually marked via countProcessed when we believe we have fully
+ // dealt with the node).
+ aa.countProcessed(n)
+
+ switch n.Op() {
+ case ir.ODCL:
+ decl := n.(*ir.Decl)
+
+ if decl.X != nil && decl.X.Type().IsSlice() && decl.X.Class == ir.PAUTO {
+ s := decl.X
+ if _, ok := aa.candidateSlices[s]; ok {
+ base.FatalfAt(n.Pos(), "candidate slice already tracked as candidate: %v", s)
+ }
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), s, "adding candidate slice", "(loop depth: %d)", len(aa.loops))
+ }
+ aa.candidateSlices[s] = candidateSlice{loopDepth: len(aa.loops)}
+ }
+ // No children aside from the declared ONAME.
+ aa.countProcessed(decl.X)
+ return false
+
+ case ir.ONAME:
+
+ // We are seeing a name we have not already handled in another case,
+ // so remove any corresponding candidate slice.
+ if n.Type().IsSlice() {
+ name := n.(*ir.Name)
+ _, ok := aa.candidateSlices[name]
+ if ok {
+ delete(aa.candidateSlices, name)
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), name, "removing candidate slice", "")
+ }
+ }
+ }
+ // No children.
+ return false
+
+ case ir.OAS2:
+ n := n.(*ir.AssignListStmt)
+ aa.analyzeAssign(n, n.Lhs, n.Rhs)
+ return false
+
+ case ir.OAS:
+ assign := n.(*ir.AssignStmt)
+ aa.analyzeAssign(n, []ir.Node{assign.X}, []ir.Node{assign.Y})
+ return false
+
+ case ir.OFOR, ir.ORANGE:
+ aa.visitList(n.Init())
+
+ if n.Op() == ir.ORANGE {
+ // TODO(thepudds): previously we visited this range expression
+ // in the switch just below, after pushing the loop. This current placement
+ // is more correct, but generate a test or find an example in stdlib or similar
+ // where it matters. (Our current tests do not complain.)
+ aa.visit(n.(*ir.RangeStmt).X)
+ }
+
+ // Push a new loop.
+ aa.loops = append(aa.loops, nil)
+
+ // Process the loop.
+ switch n.Op() {
+ case ir.OFOR:
+ forstmt := n.(*ir.ForStmt)
+ aa.visit(forstmt.Cond)
+ aa.visitList(forstmt.Body)
+ aa.visit(forstmt.Post)
+ case ir.ORANGE:
+ rangestmt := n.(*ir.RangeStmt)
+ aa.visit(rangestmt.Key)
+ aa.visit(rangestmt.Value)
+ aa.visitList(rangestmt.Body)
+ default:
+ base.Fatalf("loop not OFOR or ORANGE: %v", n)
+ }
+
+ // Pop the loop.
+ var candidateAppends []candidateAppend
+ candidateAppends, aa.loops = aa.loops[len(aa.loops)-1], aa.loops[:len(aa.loops)-1]
+ for _, a := range candidateAppends {
+ // We are done with the loop, so we can validate any candidate appends
+ // that have not had their slice removed yet. We know a slice is unaliased
+ // throughout the loop if the slice is still in the candidate slice map.
+ if cs, ok := aa.candidateSlices[a.s]; ok {
+ if cs.loopDepth == len(aa.loops) {
+ // We've returned to the loop depth where the slice was declared and
+ // hence made it all the way through any loops that started after
+ // that declaration.
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), a.s, "proved non-aliased append",
+ "(completed loop, decl at depth: %d)", cs.loopDepth)
+ }
+ aa.noAliasAppends = append(aa.noAliasAppends, a.call)
+ } else if cs.loopDepth < len(aa.loops) {
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), a.s, "cannot prove non-aliased append",
+ "(completed loop, decl at depth: %d)", cs.loopDepth)
+ }
+ } else {
+ panic("impossible: candidate slice loopDepth > current loop depth")
+ }
+ }
+ }
+ return false
+
+ case ir.OLEN, ir.OCAP:
+ n := n.(*ir.UnaryExpr)
+ if n.X.Op() == ir.ONAME {
+ // This does not disqualify a candidate slice.
+ aa.visitList(n.Init())
+ aa.countProcessed(n.X)
+ } else {
+ ir.DoChildren(n, aa.visit)
+ }
+ return false
+
+ case ir.OCLOSURE:
+ // Give up on all our in-progress slices.
+ closure := n.(*ir.ClosureExpr)
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), closure.Func, "clearing all in-progress slices due to OCLOSURE",
+ "(was %d in-progress slices)", len(aa.candidateSlices))
+ }
+ clear(aa.candidateSlices)
+ return ir.DoChildren(n, aa.visit)
+
+ case ir.OLABEL, ir.OGOTO:
+ // Give up on all our in-progress slices.
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(n.Pos(), n, "clearing all in-progress slices due to label or goto",
+ "(was %d in-progress slices)", len(aa.candidateSlices))
+ }
+ clear(aa.candidateSlices)
+ return false
+
+ default:
+ return ir.DoChildren(n, aa.visit)
+ }
+}
+
+func (aa *aliasAnalysis) visitList(nodes []ir.Node) {
+ for _, n := range nodes {
+ aa.visit(n)
+ }
+}
+
+// analyzeAssign evaluates the assignment dsts... = srcs...
+//
+// assign is an *ir.AssignStmt or *ir.AssignListStmt.
+func (aa *aliasAnalysis) analyzeAssign(assign ir.Node, dsts, srcs []ir.Node) {
+ aa.visitList(assign.Init())
+ for i := range dsts {
+ dst := dsts[i]
+ src := srcs[i]
+
+ if dst.Op() != ir.ONAME || !dst.Type().IsSlice() {
+ // Nothing for us to do aside from visiting the remaining children.
+ aa.visit(dst)
+ aa.visit(src)
+ continue
+ }
+
+ // We have a slice being assigned to an ONAME.
+
+ // Check for simple zero value assignments to an ONAME, which we ignore.
+ if src == nil {
+ aa.countProcessed(dst)
+ continue
+ }
+
+ if base.Debug.EscapeAlias >= 4 {
+ srcfn := ""
+ if src.Op() == ir.ONAME {
+ srcfn = fmt.Sprintf("%v.", src.Name().Curfn)
+ }
+ aa.diag(assign.Pos(), assign, "visiting slice assignment", "%v.%v = %s%v (%s %T = %s %T)",
+ dst.Name().Curfn, dst, srcfn, src, dst.Op().String(), dst, src.Op().String(), src)
+ }
+
+ // Now check what we have on the RHS.
+ switch src.Op() {
+ // Cases:
+
+ // Check for s := make([]T, ...) or s := []T{...}, along with the '=' version
+ // of those which does not alias s as long as s is not used in the make.
+ //
+ // TODO(thepudds): we need to be sure that 's := []T{1,2,3}' does not end up backed by a
+ // global static. Ad-hoc testing indicates that example and similar seem to be
+ // stack allocated, but that was not exhaustive testing. We do have runtime.freegc
+ // able to throw if it finds a global static, but should test more.
+ //
+ // TODO(thepudds): could also possibly allow 's := append([]T(nil), ...)'
+ // and 's := append([]T{}, ...)'.
+ case ir.OMAKESLICE, ir.OSLICELIT:
+ name := dst.(*ir.Name)
+ if name.Class == ir.PAUTO {
+ if base.Debug.EscapeAlias > 1 {
+ aa.diag(assign.Pos(), assign, "assignment from make or slice literal", "")
+ }
+ // If this is Def=true, the ODCL in the init will causes this to be tracked
+ // as a candidate slice. We walk the init and RHS but avoid visiting the name
+ // in the LHS, which would remove the slice from the candidate list after it
+ // was just added.
+ aa.visit(src)
+ aa.countProcessed(name)
+ continue
+ }
+
+ // Check for s = append(s, <...>).
+ case ir.OAPPEND:
+ s := dst.(*ir.Name)
+ call := src.(*ir.CallExpr)
+ if call.Args[0] == s {
+ // Matches s = append(s, <...>).
+ // First visit other arguments in case they use s.
+ aa.visitList(call.Args[1:])
+ // Mark the call as processed, and s twice.
+ aa.countProcessed(s, call, s)
+
+ // We have now examined all non-ONAME children of assign.
+
+ // This is now the heart of the analysis.
+ // Check to see if this slice is a live candidate.
+ cs, ok := aa.candidateSlices[s]
+ if ok {
+ if cs.loopDepth == len(aa.loops) {
+ // No new loop has started after the declaration of s,
+ // so this is definitive.
+ if base.Debug.EscapeAlias >= 2 {
+ aa.diag(assign.Pos(), assign, "proved non-aliased append",
+ "(loop depth: %d, equals decl depth)", len(aa.loops))
+ }
+ aa.noAliasAppends = append(aa.noAliasAppends, call)
+ } else if cs.loopDepth < len(aa.loops) {
+ // A new loop has started since the declaration of s,
+ // so we can't validate this append yet, but
+ // remember it in case we can validate it later when
+ // all loops using s are done.
+ aa.loops[cs.loopDepth] = append(aa.loops[cs.loopDepth],
+ candidateAppend{s: s, call: call})
+ } else {
+ panic("impossible: candidate slice loopDepth > current loop depth")
+ }
+ }
+ continue
+ }
+ } // End of switch on src.Op().
+
+ // Reached bottom of the loop over assignments.
+ // If we get here, we need to visit the dst and src normally.
+ aa.visit(dst)
+ aa.visit(src)
+ }
+}
+
+func (aa *aliasAnalysis) countProcessed(nodes ...ir.Node) {
+ if aa.doubleCheck {
+ for _, n := range nodes {
+ aa.processed[n]++
+ }
+ }
+}
+
+func (aa *aliasAnalysis) diag(pos src.XPos, n ir.Node, what string, format string, args ...any) {
+ fmt.Printf("%-25s alias analysis: %-30s %-20s %s\n",
+ fmtPosShort(pos)+":",
+ what+":",
+ fmt.Sprintf("%v", n),
+ fmt.Sprintf(format, args...))
+}
+
+// doubleCheckProcessed does a sanity check for missed nodes in our visit.
+func doubleCheckProcessed(fn *ir.Func, processed map[ir.Node]int) {
+ // Do a trivial walk while counting the nodes
+ // to compare against the counts in processed.
+
+ observed := make(map[ir.Node]int)
+ var walk func(n ir.Node) bool
+ walk = func(n ir.Node) bool {
+ observed[n]++
+ return ir.DoChildren(n, walk)
+ }
+ ir.DoChildren(fn, walk)
+
+ if !maps.Equal(processed, observed) {
+ // The most likely mistake might be something was missed while building processed,
+ // so print extra details in that direction.
+ for n, observedCount := range observed {
+ processedCount, ok := processed[n]
+ if processedCount != observedCount || !ok {
+ base.WarnfAt(n.Pos(),
+ "alias analysis: mismatch for %T: %v: processed %d times, observed %d times",
+ n, n, processedCount, observedCount)
+ }
+ }
+ base.FatalfAt(fn.Pos(), "alias analysis: mismatch in visited nodes")
+ }
+}
+
+func fmtPosShort(xpos src.XPos) string {
+ // TODO(thepudds): I think I did this a simpler way a while ago? Or maybe add base.FmtPosShort
+ // or similar? Or maybe just use base.FmtPos and give up on nicely aligned log messages?
+ pos := base.Ctxt.PosTable.Pos(xpos)
+ shortLine := filepath.Base(pos.AbsFilename()) + ":" + pos.LineNumber()
+ return shortLine
+}
diff --git a/src/cmd/compile/internal/escape/assign.go b/src/cmd/compile/internal/escape/assign.go
index 6af53886831a1e..27f13ca5aa2d39 100644
--- a/src/cmd/compile/internal/escape/assign.go
+++ b/src/cmd/compile/internal/escape/assign.go
@@ -50,6 +50,8 @@ func (e *escape) addr(n ir.Node) hole {
case ir.OINDEXMAP:
n := n.(*ir.IndexExpr)
e.discard(n.X)
+ // Keys used in map assignments must escape.
+ // See "Hashing Pointers" doc in internal/runtime/maps/map.go.
e.assignHeap(n.Index, "key of map put", n)
}
diff --git a/src/cmd/compile/internal/escape/call.go b/src/cmd/compile/internal/escape/call.go
index f9351de975fdab..8619e6010b0baa 100644
--- a/src/cmd/compile/internal/escape/call.go
+++ b/src/cmd/compile/internal/escape/call.go
@@ -31,28 +31,39 @@ func (e *escape) call(ks []hole, call ir.Node) {
call := call.(*ir.CallExpr)
typecheck.AssertFixedCall(call)
- // Pick out the function callee, if statically known.
+ // Pick out the function callee(s), if statically known.
+ // fns collects all known callees; for a single static callee
+ // it has one element. For unknown callees fns is nil.
//
- // TODO(mdempsky): Change fn from *ir.Name to *ir.Func, but some
- // functions (e.g., runtime builtins, method wrappers, generated
- // eq/hash functions) don't have it set. Investigate whether
- // that's a concern.
- var fn *ir.Name
+ // TODO(mdempsky): Change fns from []*ir.Name to []*ir.Func,
+ // but some functions (e.g., runtime builtins, method wrappers,
+ // generated eq/hash functions) don't have it set. Investigate
+ // whether that's a concern.
+ var fns []*ir.Name
switch call.Op() {
case ir.OCALLFUNC:
- // TODO(thepudds): use an ir.ReassignOracle here.
- v := ir.StaticValue(call.Fun)
- fn = ir.StaticCalleeName(v)
+ ro := e.reassignOracle(e.curfn)
+ v := ro.StaticValue(call.Fun)
+ if fn := ir.StaticCalleeName(v); fn != nil {
+ fns = []*ir.Name{fn}
+ } else if name, ok := v.(*ir.Name); ok {
+ fns = resolveAssignedCallees(ro.FuncAssignments(name.Canonical()))
+ }
}
fntype := call.Fun.Type()
- if fn != nil {
- fntype = fn.Type()
+ if len(fns) == 1 {
+ fntype = fns[0].Type()
}
- if ks != nil && fn != nil && e.inMutualBatch(fn) {
- for i, result := range fn.Type().Results() {
- e.expr(ks[i], result.Nname.(*ir.Name))
+ // Wire result flows for in-batch callees.
+ if ks != nil {
+ for _, f := range fns {
+ if e.inMutualBatch(f) {
+ for i, result := range f.Type().Results() {
+ e.expr(ks[i], result.Nname.(*ir.Name))
+ }
+ }
}
}
@@ -60,7 +71,7 @@ func (e *escape) call(ks []hole, call ir.Node) {
if call.Op() == ir.OCALLFUNC {
// Evaluate callee function expression.
calleeK := e.discardHole()
- if fn == nil { // unknown callee
+ if len(fns) == 0 { // unknown callee
for _, k := range ks {
if k.dst != &e.blankLoc {
// The results flow somewhere, but we don't statically
@@ -77,53 +88,48 @@ func (e *escape) call(ks []hole, call ir.Node) {
recvArg = call.Fun.(*ir.SelectorExpr).X
}
- // argumentParam handles escape analysis of assigning a call
- // argument to its corresponding parameter.
- argumentParam := func(param *types.Field, arg ir.Node) {
- e.rewriteArgument(arg, call, fn)
- argument(e.tagHole(ks, fn, param), arg)
- }
-
- // internal/abi.EscapeNonString forces its argument to be on
- // the heap, if it contains a non-string pointer.
- // This is used in hash/maphash.Comparable, where we cannot
- // hash pointers to local variables, as the address of the
- // local variable might change on stack growth.
- // Strings are okay as the hash depends on only the content,
- // not the pointer.
- // This is also used in unique.clone, to model the data flow
- // edge on the value with strings excluded, because strings
- // are cloned (by content).
- // The actual call we match is
- // internal/abi.EscapeNonString[go.shape.T](dict, go.shape.T)
- if fn != nil && fn.Sym().Pkg.Path == "internal/abi" && strings.HasPrefix(fn.Sym().Name, "EscapeNonString[") {
- ps := fntype.Params()
- if len(ps) == 2 && ps[1].Type.IsShape() {
- if !hasNonStringPointers(ps[1].Type) {
- argumentParam = func(param *types.Field, arg ir.Node) {
- argument(e.discardHole(), arg)
- }
- } else {
- argumentParam = func(param *types.Field, arg ir.Node) {
- argument(e.heapHole(), arg)
- }
- }
- }
- }
-
args := call.Args
- if recvParam := fntype.Recv(); recvParam != nil {
+ if fntype.Recv() != nil {
if recvArg == nil {
// Function call using method expression. Receiver argument is
// at the front of the regular arguments list.
recvArg, args = args[0], args[1:]
}
-
- argumentParam(recvParam, recvArg)
}
- for i, param := range fntype.Params() {
- argumentParam(param, args[i])
+ if call.IsCompilerVarLive {
+ // Don't escape compiler-inserted KeepAlive.
+ if recvArg != nil {
+ argument(e.discardHole(), recvArg)
+ }
+ for _, arg := range args {
+ argument(e.discardHole(), arg)
+ }
+ } else if isEscapeNonString(fns, fntype) {
+ // internal/abi.EscapeNonString forces its argument to
+ // the heap if it contains a non-string pointer. This is
+ // used in hash/maphash.Comparable (where we cannot hash
+ // pointers to locals whose address may change on stack
+ // growth) and unique.clone (to model the data flow edge
+ // with strings excluded, because strings are cloned by
+ // content). The actual call we match is:
+ // internal/abi.EscapeNonString[go.shape.T](dict, go.shape.T)
+ k := e.heapHole()
+ if !hasNonStringPointers(fntype.Params()[1].Type) {
+ k = e.discardHole()
+ }
+ for _, arg := range args {
+ argument(k, arg)
+ }
+ } else {
+ if recvArg != nil {
+ e.rewriteArgument(recvArg, call, fns)
+ argument(e.mergedTagHole(ks, fns, -1, len(fntype.Params())), recvArg)
+ }
+ for i := range fntype.Params() {
+ e.rewriteArgument(args[i], call, fns)
+ argument(e.mergedTagHole(ks, fns, i, len(fntype.Params())), args[i])
+ }
}
case ir.OINLCALL:
@@ -198,6 +204,8 @@ func (e *escape) call(ks []hole, call ir.Node) {
e.discard(arg)
}
e.discard(call.RType)
+ // Note: keys used in map deletes do not need to escape.
+ // See "Hashing Pointers" doc in internal/runtime/maps/map.go.
case ir.OMIN, ir.OMAX:
call := call.(*ir.CallExpr)
@@ -262,12 +270,14 @@ func (e *escape) goDeferStmt(n *ir.GoDeferStmt) {
}
// rewriteArgument rewrites the argument arg of the given call expression.
-// fn is the static callee function, if known.
-func (e *escape) rewriteArgument(arg ir.Node, call *ir.CallExpr, fn *ir.Name) {
- if fn == nil || fn.Func == nil {
- return
+// fns is the list of statically known callees, if any.
+func (e *escape) rewriteArgument(arg ir.Node, call *ir.CallExpr, fns []*ir.Name) {
+ var pragma ir.PragmaFlag
+ for _, fn := range fns {
+ if fn.Func != nil {
+ pragma |= fn.Func.Pragma
+ }
}
- pragma := fn.Func.Pragma
if pragma&(ir.UintptrKeepAlive|ir.UintptrEscapes) == 0 {
return
}
@@ -354,6 +364,25 @@ func (e *escape) copyExpr(pos src.XPos, expr ir.Node, init *ir.Nodes) *ir.Name {
return tmp
}
+func (e *escape) mergedTagHole(ks []hole, fns []*ir.Name, paramIdx int, nParams int) hole {
+ if len(fns) == 0 {
+ return e.heapHole()
+ }
+ holes := make([]hole, 0, len(fns))
+ for _, f := range fns {
+ offset := nParams - len(f.Type().Params())
+ j := paramIdx - offset
+ var p *types.Field
+ if j >= 0 {
+ p = f.Type().Params()[j]
+ } else {
+ p = f.Type().Recv()
+ }
+ holes = append(holes, e.tagHole(ks, f, p))
+ }
+ return e.teeHole(holes...)
+}
+
// tagHole returns a hole for evaluating an argument passed to param.
// ks should contain the holes representing where the function
// callee's results flows. fn is the statically-known callee function,
@@ -397,6 +426,34 @@ func (e *escape) tagHole(ks []hole, fn *ir.Name, param *types.Field) hole {
return e.teeHole(tagKs...)
}
+// resolveAssignedCallees resolves all assignment RHS values to static
+// callee names, skipping zero-value assignments since nil panics on
+// call and can't cause escape.
+func resolveAssignedCallees(assigns []*ir.AssignStmt) []*ir.Name {
+ fns := make([]*ir.Name, 0, len(assigns))
+ for _, as := range assigns {
+ if ir.IsZero(as.Y) {
+ continue // zero value panics on call; skip
+ }
+ callee := ir.StaticCalleeName(as.Y)
+ if callee == nil {
+ return nil
+ }
+ if callee.Func != nil && callee.Func.Pragma&(ir.UintptrKeepAlive|ir.UintptrEscapes) != 0 {
+ return nil
+ }
+ fns = append(fns, callee)
+ }
+ return fns
+}
+
+func isEscapeNonString(fns []*ir.Name, fntype *types.Type) bool {
+ return len(fns) == 1 &&
+ fns[0].Sym().Pkg.Path == "internal/abi" &&
+ strings.HasPrefix(fns[0].Sym().Name, "EscapeNonString[") &&
+ len(fntype.Params()) == 2 && fntype.Params()[1].Type.IsShape()
+}
+
func hasNonStringPointers(t *types.Type) bool {
if !t.HasPointers() {
return false
diff --git a/src/cmd/compile/internal/escape/escape.go b/src/cmd/compile/internal/escape/escape.go
index 59250edfef0078..1c8e3824e9bc8e 100644
--- a/src/cmd/compile/internal/escape/escape.go
+++ b/src/cmd/compile/internal/escape/escape.go
@@ -8,6 +8,7 @@ import (
"fmt"
"go/constant"
"go/token"
+ "internal/goexperiment"
"cmd/compile/internal/base"
"cmd/compile/internal/ir"
@@ -369,6 +370,21 @@ func (b *batch) finish(fns []*ir.Func) {
}
}
}
+
+ if goexperiment.RuntimeFreegc {
+ // Look for specific patterns of usage, such as appends
+ // to slices that we can prove are not aliased.
+ for _, fn := range fns {
+ a := aliasAnalysis{}
+ a.analyze(fn)
+ }
+ }
+
+ for _, fn := range fns {
+ if ir.MatchAstDump(fn, "escape") {
+ ir.AstDump(fn, "escape, "+ir.FuncName(fn))
+ }
+ }
}
// inMutualBatch reports whether function fn is in the batch of
diff --git a/src/cmd/compile/internal/escape/expr.go b/src/cmd/compile/internal/escape/expr.go
index 1521c2edd17605..468c4713f5b925 100644
--- a/src/cmd/compile/internal/escape/expr.go
+++ b/src/cmd/compile/internal/escape/expr.go
@@ -90,6 +90,8 @@ func (e *escape) exprSkipInit(k hole, n ir.Node) {
case ir.OINDEXMAP:
n := n.(*ir.IndexExpr)
e.discard(n.X)
+ // Keys used in map lookups do not need to escape.
+ // See "Hashing Pointers" doc in internal/runtime/maps/map.go.
e.discard(n.Index)
case ir.OSLICE, ir.OSLICEARR, ir.OSLICE3, ir.OSLICE3ARR, ir.OSLICESTR:
n := n.(*ir.SliceExpr)
diff --git a/src/cmd/compile/internal/escape/leaks.go b/src/cmd/compile/internal/escape/leaks.go
index 942f87d2a225ff..176bccd8470e7b 100644
--- a/src/cmd/compile/internal/escape/leaks.go
+++ b/src/cmd/compile/internal/escape/leaks.go
@@ -124,3 +124,21 @@ func parseLeaks(s string) leaks {
copy(l[:], s[4:])
return l
}
+
+func ParseLeaks(s string) leaks {
+ return parseLeaks(s)
+}
+
+// Any reports whether the value flows anywhere at all.
+func (l leaks) Any() bool {
+ // TODO: do mutator/callee matter?
+ if l.Heap() >= 0 || l.Mutator() >= 0 || l.Callee() >= 0 {
+ return true
+ }
+ for i := range numEscResults {
+ if l.Result(i) >= 0 {
+ return true
+ }
+ }
+ return false
+}
diff --git a/src/cmd/compile/internal/gc/compile.go b/src/cmd/compile/internal/gc/compile.go
index 1a40df9a84ff04..3d8974ee299b71 100644
--- a/src/cmd/compile/internal/gc/compile.go
+++ b/src/cmd/compile/internal/gc/compile.go
@@ -29,7 +29,7 @@ var (
compilequeue []*ir.Func // functions waiting to be compiled
)
-func enqueueFunc(fn *ir.Func) {
+func enqueueFunc(fn *ir.Func, symABIs *ssagen.SymABIs) {
if ir.CurFunc != nil {
base.FatalfAt(fn.Pos(), "enqueueFunc %v inside %v", fn, ir.CurFunc)
}
@@ -49,22 +49,30 @@ func enqueueFunc(fn *ir.Func) {
}
if len(fn.Body) == 0 {
- // Initialize ABI wrappers if necessary.
- ir.InitLSym(fn, false)
- types.CalcSize(fn.Type())
- a := ssagen.AbiForBodylessFuncStackMap(fn)
- abiInfo := a.ABIAnalyzeFuncType(fn.Type()) // abiInfo has spill/home locations for wrapper
- if fn.ABI == obj.ABI0 {
- // The current args_stackmap generation assumes the function
- // is ABI0, and only ABI0 assembly function can have a FUNCDATA
- // reference to args_stackmap (see cmd/internal/obj/plist.go:Flushplist).
- // So avoid introducing an args_stackmap if the func is not ABI0.
- liveness.WriteFuncMap(fn, abiInfo)
-
- x := ssagen.EmitArgInfo(fn, abiInfo)
- objw.Global(x, int32(len(x.P)), obj.RODATA|obj.LOCAL)
+ if ir.IsIntrinsicSym(fn.Sym()) && fn.Sym().Linkname == "" && !symABIs.HasDef(fn.Sym()) {
+ // Generate the function body for a bodyless intrinsic, in case it
+ // is used in a non-call context (e.g. as a function pointer).
+ // We skip functions defined in assembly, or has a linkname (which
+ // could be defined in another package).
+ ssagen.GenIntrinsicBody(fn)
+ } else {
+ // Initialize ABI wrappers if necessary.
+ ir.InitLSym(fn, false)
+ types.CalcSize(fn.Type())
+ a := ssagen.AbiForBodylessFuncStackMap(fn)
+ abiInfo := a.ABIAnalyzeFuncType(fn.Type()) // abiInfo has spill/home locations for wrapper
+ if fn.ABI == obj.ABI0 {
+ // The current args_stackmap generation assumes the function
+ // is ABI0, and only ABI0 assembly function can have a FUNCDATA
+ // reference to args_stackmap (see cmd/internal/obj/plist.go:Flushplist).
+ // So avoid introducing an args_stackmap if the func is not ABI0.
+ liveness.WriteFuncMap(fn, abiInfo)
+
+ x := ssagen.EmitArgInfo(fn, abiInfo)
+ objw.Global(x, int32(len(x.P)), obj.RODATA|obj.LOCAL)
+ }
+ return
}
- return
}
errorsBefore := base.Errors()
@@ -113,6 +121,9 @@ func prepareFunc(fn *ir.Func) {
ir.CurFunc = fn
walk.Walk(fn)
+ if ir.MatchAstDump(fn, "walk") {
+ ir.AstDump(fn, "walk, "+ir.FuncName(fn))
+ }
ir.CurFunc = nil // enforce no further uses of CurFunc
base.Ctxt.DwTextCount++
@@ -134,73 +145,47 @@ func compileFunctions(profile *pgoir.Profile) {
// Compile the longest functions first,
// since they're most likely to be the slowest.
// This helps avoid stragglers.
+ // Since we remove from the end of the slice queue,
+ // that means shortest to longest.
slices.SortFunc(compilequeue, func(a, b *ir.Func) int {
- return cmp.Compare(len(b.Body), len(a.Body))
+ return cmp.Compare(len(a.Body), len(b.Body))
})
}
- // By default, we perform work right away on the current goroutine
- // as the solo worker.
- queue := func(work func(int)) {
- work(0)
- }
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ mu.Lock()
- if nWorkers := base.Flag.LowerC; nWorkers > 1 {
- // For concurrent builds, we allow the work queue
- // to grow arbitrarily large, but only nWorkers work items
- // can be running concurrently.
- workq := make(chan func(int))
- done := make(chan int)
+ for workerId := range base.Flag.LowerC {
+ // TODO: replace with wg.Go when the oldest bootstrap has it.
+ // With the current policy, that'd be go1.27.
+ wg.Add(1)
go func() {
- ids := make([]int, nWorkers)
- for i := range ids {
- ids[i] = i
- }
- var pending []func(int)
+ defer wg.Done()
+ var closures []*ir.Func
for {
- select {
- case work := <-workq:
- pending = append(pending, work)
- case id := <-done:
- ids = append(ids, id)
- }
- for len(pending) > 0 && len(ids) > 0 {
- work := pending[len(pending)-1]
- id := ids[len(ids)-1]
- pending = pending[:len(pending)-1]
- ids = ids[:len(ids)-1]
- go func() {
- work(id)
- done <- id
- }()
+ mu.Lock()
+ compilequeue = append(compilequeue, closures...)
+ remaining := len(compilequeue)
+ if remaining == 0 {
+ mu.Unlock()
+ return
}
+ fn := compilequeue[len(compilequeue)-1]
+ compilequeue = compilequeue[:len(compilequeue)-1]
+ mu.Unlock()
+ ssagen.Compile(fn, workerId, profile)
+ closures = fn.Closures
}
}()
- queue = func(work func(int)) {
- workq <- work
- }
- }
-
- var wg sync.WaitGroup
- var compile func([]*ir.Func)
- compile = func(fns []*ir.Func) {
- wg.Add(len(fns))
- for _, fn := range fns {
- fn := fn
- queue(func(worker int) {
- ssagen.Compile(fn, worker, profile)
- compile(fn.Closures)
- wg.Done()
- })
- }
}
types.CalcSizeDisabled = true // not safe to calculate sizes concurrently
base.Ctxt.InParallel = true
- compile(compilequeue)
- compilequeue = nil
+ mu.Unlock()
wg.Wait()
+ compilequeue = nil
base.Ctxt.InParallel = false
types.CalcSizeDisabled = false
diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go
index 42e2afaee4fc24..2b8ecda9d16c0f 100644
--- a/src/cmd/compile/internal/gc/main.go
+++ b/src/cmd/compile/internal/gc/main.go
@@ -8,6 +8,7 @@ import (
"bufio"
"bytes"
"cmd/compile/internal/base"
+ "cmd/compile/internal/bloop"
"cmd/compile/internal/coverage"
"cmd/compile/internal/deadlocals"
"cmd/compile/internal/dwarfgen"
@@ -22,6 +23,7 @@ import (
"cmd/compile/internal/pkginit"
"cmd/compile/internal/reflectdata"
"cmd/compile/internal/rttype"
+ "cmd/compile/internal/slice"
"cmd/compile/internal/ssa"
"cmd/compile/internal/ssagen"
"cmd/compile/internal/staticinit"
@@ -45,6 +47,8 @@ import (
// already been some compiler errors). It may also be invoked from the explicit panic in
// hcrash(), in which case, we pass the panic on through.
func handlePanic() {
+ ir.CloseHTMLWriters()
+ noder.CloseHTMLWriters()
if err := recover(); err != nil {
if err == "-h" {
// Force real panic now with -h option (hcrash) - the error
@@ -81,10 +85,13 @@ func Main(archInit func(*ssagen.ArchInfo)) {
base.DebugSSA = ssa.PhaseOption
base.ParseFlags()
- if os.Getenv("GOGC") == "" { // GOGC set disables starting heap adjustment
- // More processors will use more heap, but assume that more memory is available.
- // So 1 processor -> 40MB, 4 -> 64MB, 12 -> 128MB
- base.AdjustStartingHeap(uint64(32+8*base.Flag.LowerC) << 20)
+ if flagGCStart := base.Debug.GCStart; flagGCStart > 0 || // explicit flags overrides environment variable disable of GC boost
+ os.Getenv("GOGC") == "" && os.Getenv("GOMEMLIMIT") == "" && base.Flag.LowerC != 1 { // explicit GC knobs or no concurrency implies default heap
+ startHeapMB := int64(128)
+ if flagGCStart > 0 {
+ startHeapMB = int64(flagGCStart)
+ }
+ base.AdjustStartingHeap(uint64(startHeapMB)<<20, 0, 0, 0, base.Debug.GCAdjust == 1)
}
types.LocalPkg = types.NewPkg(base.Ctxt.Pkgpath, "")
@@ -186,9 +193,9 @@ func Main(archInit func(*ssagen.ArchInfo)) {
ir.EscFmt = escape.Fmt
ir.IsIntrinsicCall = ssagen.IsIntrinsicCall
+ ir.IsIntrinsicSym = ssagen.IsIntrinsicSym
inline.SSADumpInline = ssagen.DumpInline
ssagen.InitEnv()
- ssagen.InitTables()
types.PtrSize = ssagen.Arch.LinkArch.PtrSize
types.RegSize = ssagen.Arch.LinkArch.RegSize
@@ -202,6 +209,11 @@ func Main(archInit func(*ssagen.ArchInfo)) {
typecheck.InitRuntime()
rttype.Init()
+ // Some intrinsics (notably, the simd intrinsics) mention
+ // types "eagerly", thus ssagen must be initialized AFTER
+ // the type system is ready.
+ ssagen.InitTables()
+
// Parse and typecheck input.
noder.LoadPackage(flag.Args())
@@ -233,10 +245,25 @@ func Main(archInit func(*ssagen.ArchInfo)) {
}
}
+ for _, fn := range typecheck.Target.Funcs {
+ if ir.MatchAstDump(fn, "start") {
+ ir.AstDump(fn, "start, "+ir.FuncName(fn))
+ }
+ }
+
+ // Apply bloop markings.
+ bloop.Walk(typecheck.Target)
+
// Interleaved devirtualization and inlining.
base.Timer.Start("fe", "devirtualize-and-inline")
interleaved.DevirtualizeAndInlinePackage(typecheck.Target, profile)
+ for _, fn := range typecheck.Target.Funcs {
+ if ir.MatchAstDump(fn, "devirtualize-and-inline") {
+ ir.AstDump(fn, "devirtualize-and-inline, "+ir.FuncName(fn))
+ }
+ }
+
noder.MakeWrappers(typecheck.Target) // must happen after inlining
// Get variable capture right in for loops.
@@ -266,6 +293,8 @@ func Main(archInit func(*ssagen.ArchInfo)) {
base.Timer.Start("fe", "escapes")
escape.Funcs(typecheck.Target.Funcs)
+ slice.Funcs(typecheck.Target.Funcs)
+
loopvar.LogTransformations(transformed)
// Collect information for go:nowritebarrierrec
@@ -302,7 +331,7 @@ func Main(archInit func(*ssagen.ArchInfo)) {
}
if nextFunc < len(typecheck.Target.Funcs) {
- enqueueFunc(typecheck.Target.Funcs[nextFunc])
+ enqueueFunc(typecheck.Target.Funcs[nextFunc], symABIs)
nextFunc++
continue
}
diff --git a/src/cmd/compile/internal/importer/gcimporter_test.go b/src/cmd/compile/internal/importer/gcimporter_test.go
index 11e4ee6b583b6a..e81a71f266eb90 100644
--- a/src/cmd/compile/internal/importer/gcimporter_test.go
+++ b/src/cmd/compile/internal/importer/gcimporter_test.go
@@ -720,3 +720,71 @@ var _ issue63285.A[issue63285.B[any]]
t.Errorf("Check failed: %v", err)
}
}
+
+func TestGenMeth(t *testing.T) {
+ testenv.MustHaveGoBuild(t)
+
+ // This package only handles gc export data.
+ if runtime.Compiler != "gc" {
+ t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+ }
+
+ tmpdir := t.TempDir()
+ testoutdir := filepath.Join(tmpdir, "testdata")
+ if err := os.Mkdir(testoutdir, 0700); err != nil {
+ t.Fatalf("making output dir: %v", err)
+ }
+
+ compile(t, "testdata", "genmeth.go", testoutdir, nil)
+
+ genmeth, err := Import(make(map[string]*types2.Package), "./testdata/genmeth", tmpdir, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ check := func(pkgname, src string, imports importMap) (*types2.Package, error) {
+ f, err := syntax.Parse(syntax.NewFileBase(pkgname), strings.NewReader(src), nil, nil, 0)
+ if err != nil {
+ return nil, err
+ }
+ config := &types2.Config{
+ Importer: imports,
+ }
+ return config.Check(pkgname, []*syntax.File{f}, nil)
+ }
+
+ const pSrc = `package p
+
+import "genmeth"
+
+func _() {
+ var ex func(int) genmeth.List[int]
+ var fl func(genmeth.List[int]) genmeth.List[int]
+
+ var l genmeth.List[int]
+ l = l.Map(ex).FlatMap(fl)
+
+ var bl genmeth.BiList[int, any]
+ bl = bl.MapKeys(ex).Flip().FlatMapValues(fl).Flip()
+
+ var id func(int) int
+
+ var op genmeth.Option[int]
+ var _ int = op.MapIfPresent(id).Get()
+
+ var ol genmeth.OrderedList[int]
+ var _ int = ol.Min().Get()
+
+ var b genmeth.Box[int]
+ b.Set(42)
+ var _ int = b.Get()
+}
+`
+
+ importer := importMap{
+ "genmeth": genmeth,
+ }
+ if _, err := check("p", pSrc, importer); err != nil {
+ t.Errorf("Check failed: %v", err)
+ }
+}
diff --git a/src/cmd/compile/internal/importer/testdata/genmeth.go b/src/cmd/compile/internal/importer/testdata/genmeth.go
new file mode 100644
index 00000000000000..0b5aa7e5a35762
--- /dev/null
+++ b/src/cmd/compile/internal/importer/testdata/genmeth.go
@@ -0,0 +1,49 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is used to generate an object file which
+// serves as test file for gcimporter_test.go.
+//
+// Function bodies are never read, so panics are used
+// for brevity. Struct contents are also never filled
+// in.
+
+package genmeth
+
+// monads
+type Either[A, B any] struct {}
+
+func (Either[A, B]) A() A { panic(42) }
+func (Either[A, B]) B() B { panic(42) }
+
+type Option[T any] struct {}
+
+func (Option[T]) Get() T { panic(42) }
+func (Option[T]) GetOrDefault[U any](u U) Either[T, U] { panic(42) }
+func (Option[T]) MapIfPresent[U any](f func(T) U) Option[U] { panic(42) }
+
+// pointer receiver
+type Box[P any] struct {}
+
+func (*Box[P]) Get() P { panic(42) }
+func (*Box[P]) Set(p P) { panic(42) }
+
+// streaming
+type List[E any] []E
+func (List[E]) Map[R any](f func(E) R) List[R] { panic(42) }
+func (List[E]) FlatMap[R any](f func(E) List[R]) List[R] { panic(42) }
+func (List[E1]) Zip[E2, R any](l List[E2], f func(E1, E2) R) List[R] { panic(42) }
+
+type Pair[A, B any] struct {}
+type BiList[A, B any] List[Pair[A, B]]
+
+func (BiList[A, B]) Flip() BiList[B, A] { panic(42) }
+func (BiList[A, B]) MapKeys[R any](f func(A) R) BiList[R, B] { panic(42) }
+func (BiList[A, B]) MapValues[R any](f func(B) R) BiList[A, R] { panic(42) }
+func (BiList[A, B]) FlatMapValues[R any](f func(B) List[R]) BiList[A, R] { panic(42) }
+func (BiList[A, B]) MapEntries[K, V any](f func(Pair[A, B]) Pair[K, V]) BiList[K, V] { panic(42) }
+
+type OrderedList[E comparable] []E
+func (OrderedList[E]) Min() Option[E] { panic(42) }
+func (OrderedList[E]) Max() Option[E] { panic(42) }
diff --git a/src/cmd/compile/internal/importer/ureader.go b/src/cmd/compile/internal/importer/ureader.go
index f6df56b70e665c..fbe275be04c62b 100644
--- a/src/cmd/compile/internal/importer/ureader.go
+++ b/src/cmd/compile/internal/importer/ureader.go
@@ -6,6 +6,9 @@
package importer
import (
+ "cmp"
+ "slices"
+
"cmd/compile/internal/base"
"cmd/compile/internal/syntax"
"cmd/compile/internal/types2"
@@ -72,8 +75,10 @@ type reader struct {
}
type readerDict struct {
- bounds []typeInfo
+ rtbounds []typeInfo
+ rtparams []*types2.TypeParam
+ tbounds []typeInfo
tparams []*types2.TypeParam
derived []derivedInfo
@@ -262,7 +267,11 @@ func (r *reader) doTyp() (res types2.Type) {
return name.Type()
case pkgbits.TypeTypeParam:
- return r.dict.tparams[r.Len()]
+ n := r.Len()
+ if n < len(r.dict.rtbounds) {
+ return r.dict.rtparams[n]
+ }
+ return r.dict.tparams[n-len(r.dict.rtbounds)]
case pkgbits.TypeArray:
len := int64(r.Uint64())
@@ -421,7 +430,7 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types2.Package, string) {
pos := r.pos()
var tparams []*types2.TypeParam
if r.Version().Has(pkgbits.AliasTypeParamNames) {
- tparams = r.typeParamNames(false)
+ tparams = r.typeParamNames(false, false)
}
typ := r.typ()
return newAliasTypeName(pr.enableAlias, pos, objPkg, objName, typ, tparams)
@@ -434,7 +443,10 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types2.Package, string) {
case pkgbits.ObjFunc:
pos := r.pos()
- tparams := r.typeParamNames(false)
+ if r.Version().Has(pkgbits.GenericMethods) {
+ assert(!r.Bool()) // generic methods are read in their defining type
+ }
+ tparams := r.typeParamNames(false, false)
sig := r.signature(nil, nil, tparams)
return types2.NewFunc(pos, objPkg, objName, sig)
@@ -442,7 +454,7 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types2.Package, string) {
pos := r.pos()
return types2.NewTypeNameLazy(pos, objPkg, objName, func(_ *types2.Named) ([]*types2.TypeParam, types2.Type, []*types2.Func, []func()) {
- tparams := r.typeParamNames(true)
+ tparams := r.typeParamNames(true, false)
// TODO(mdempsky): Rewrite receiver types to underlying is an
// Interface? The go/types importer does this (I think because
@@ -450,12 +462,70 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types2.Package, string) {
// about it, so maybe we can avoid worrying about that here.
underlying := r.typ().Underlying()
- methods := make([]*types2.Func, r.Len())
- for i := range methods {
- methods[i] = r.method(true)
+ type indexedMethod struct {
+ index int // (or -1 in V4)
+ fn *types2.Func
+ }
+ var methods []indexedMethod
+
+ if r.Version().Has(pkgbits.GenericMethods) {
+ // V4 (go1.27.0) emitted all non-generic methods
+ // before all generic ones, discarding source
+ // order: a bug (go.dev/issue/81188).
+ // V5 (go1.27.x) fixes it by emitting an explicit
+ // index along with each method.
+
+ // ordinary methods
+ for range r.Len() {
+ idx, m := r.method(true)
+ methods = append(methods, indexedMethod{idx, m})
+ }
+
+ // generic methods
+ for range r.Len() {
+ // Careful: objIdx is used to read in package-scoped declarations, which
+ // methods are not. Instead, decode it here. This makes it easier to
+ // associate it with the type and avoids the main objIdx loop.
+ idx := r.Reloc(pkgbits.SectionObj)
+
+ t := pr.tempReader(pkgbits.SectionObj, idx, pkgbits.SyncObject1)
+ t.dict = pr.objDictIdx(idx)
+
+ pos := t.pos()
+ assert(t.Bool()) // generic method
+ pkg, name := t.selector()
+ rtparams := t.typeParamNames(true, true)
+ recv := t.param()
+ methodIdx := -1
+ if r.Version().Has(pkgbits.PreserveMethodOrder) {
+ methodIdx = t.Len()
+ }
+ tparams := t.typeParamNames(true, false)
+ sig := t.signature(recv, rtparams, tparams)
+
+ r.delayed = append(r.delayed, t.delayed...) // propagate before retiring
+ pr.retireReader(t)
+ methods = append(methods, indexedMethod{methodIdx, types2.NewFunc(pos, pkg, name, sig)})
+ }
+
+ if r.Version().Has(pkgbits.PreserveMethodOrder) {
+ slices.SortFunc(methods, func(a, b indexedMethod) int {
+ return cmp.Compare(a.index, b.index)
+ })
+ }
+ } else {
+ for range r.Len() {
+ _, m := r.method(true)
+ methods = append(methods, indexedMethod{-1, m})
+ }
+ }
+
+ funcs := make([]*types2.Func, len(methods))
+ for i, m := range methods {
+ funcs[i] = m.fn
}
- return tparams, underlying, methods, r.delayed
+ return tparams, underlying, funcs, r.delayed
})
case pkgbits.ObjVar:
@@ -477,9 +547,20 @@ func (pr *pkgReader) objDictIdx(idx pkgbits.Index) *readerDict {
base.Fatalf("unexpected object with %v implicit type parameter(s)", implicits)
}
- dict.bounds = make([]typeInfo, r.Len())
- for i := range dict.bounds {
- dict.bounds[i] = r.typInfo()
+ nreceivers := 0
+ if r.Version().Has(pkgbits.GenericMethods) {
+ nreceivers = r.Len()
+ }
+ nexplicits := r.Len()
+
+ dict.rtbounds = make([]typeInfo, nreceivers)
+ for i := range dict.rtbounds {
+ dict.rtbounds[i] = r.typInfo()
+ }
+
+ dict.tbounds = make([]typeInfo, nexplicits)
+ for i := range dict.tbounds {
+ dict.tbounds[i] = r.typInfo()
}
dict.derived = make([]derivedInfo, r.Len())
@@ -498,15 +579,24 @@ func (pr *pkgReader) objDictIdx(idx pkgbits.Index) *readerDict {
return &dict
}
-func (r *reader) typeParamNames(isLazy bool) []*types2.TypeParam {
+func (r *reader) typeParamNames(isLazy bool, isGenMeth bool) []*types2.TypeParam {
r.Sync(pkgbits.SyncTypeParamNames)
- // Note: This code assumes it only processes objects without
- // implement type parameters. This is currently fine, because
- // reader is only used to read in exported declarations, which are
- // always package scoped.
+ // Note: This code assumes there are no implicit type parameters.
+ // This is fine since it only reads exported declarations, which
+ // never have implicits.
- if len(r.dict.bounds) == 0 {
+ var in []typeInfo
+ var out *[]*types2.TypeParam
+ if isGenMeth {
+ in = r.dict.rtbounds
+ out = &r.dict.rtparams
+ } else {
+ in = r.dict.tbounds
+ out = &r.dict.tparams
+ }
+
+ if len(in) == 0 {
return nil
}
@@ -515,51 +605,57 @@ func (r *reader) typeParamNames(isLazy bool) []*types2.TypeParam {
// create all the TypeNames and TypeParams, then we construct and
// set the bound type.
- r.dict.tparams = make([]*types2.TypeParam, len(r.dict.bounds))
- for i := range r.dict.bounds {
+ // We have to save tparams outside of the closure, because typeParamNames
+ // can be called multiple times with the same dictionary instance.
+ tparams := make([]*types2.TypeParam, len(in))
+ *out = tparams
+
+ for i := range in {
pos := r.pos()
pkg, name := r.localIdent()
tname := types2.NewTypeName(pos, pkg, name, nil)
- r.dict.tparams[i] = types2.NewTypeParam(tname, nil)
+ tparams[i] = types2.NewTypeParam(tname, nil)
}
// Type parameters that are read by lazy loaders cannot have their
// constraints set eagerly; do them after loading (go.dev/issue/63285).
if isLazy {
// The reader dictionary will continue mutating before we have time
- // to call delayed functions; must make a local copy of both the type
- // parameters and their (unexpanded) constraints.
- bounds := make([]types2.Type, len(r.dict.bounds))
- for i, bound := range r.dict.bounds {
- bounds[i] = r.p.typIdx(bound, r.dict)
+ // to call delayed functions; make a local copy of the constraints.
+ types := make([]types2.Type, len(in))
+ for i, info := range in {
+ types[i] = r.p.typIdx(info, r.dict)
}
- tparams := r.dict.tparams
r.delayed = append(r.delayed, func() {
- for i, bound := range bounds {
- tparams[i].SetConstraint(bound)
+ for i, typ := range types {
+ tparams[i].SetConstraint(typ)
}
})
} else {
- for i, bound := range r.dict.bounds {
- r.dict.tparams[i].SetConstraint(r.p.typIdx(bound, r.dict))
+ for i, info := range in {
+ tparams[i].SetConstraint(r.p.typIdx(info, r.dict))
}
}
- return r.dict.tparams
+ return tparams
}
-func (r *reader) method(isLazy bool) *types2.Func {
+func (r *reader) method(isLazy bool) (int, *types2.Func) {
r.Sync(pkgbits.SyncMethod)
+ idx := -1
+ if r.Version().Has(pkgbits.PreserveMethodOrder) {
+ idx = r.Len()
+ }
pos := r.pos()
pkg, name := r.selector()
- rtparams := r.typeParamNames(isLazy)
+ rtparams := r.typeParamNames(isLazy, false)
sig := r.signature(r.param(), rtparams, nil)
_ = r.pos() // TODO(mdempsky): Remove; this is a hacker for linker.go.
- return types2.NewFunc(pos, pkg, name, sig)
+ return idx, types2.NewFunc(pos, pkg, name, sig)
}
func (r *reader) qualifiedIdent() (*types2.Package, string) { return r.ident(pkgbits.SyncSym) }
diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index 3a9243e0291f5e..bd8c3c1a23f482 100644
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -179,6 +179,25 @@ func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {
})
}
+func simdCreditMultiplier(fn *ir.Func) int32 {
+ for _, field := range fn.Type().RecvParamsResults() {
+ if field.Type.IsSIMD() {
+ return 3
+ }
+ }
+ // Sometimes code uses closures, that do not take simd
+ // parameters, to perform repetitive SIMD operations.
+ // fn. These really need to be inlined, or the anticipated
+ // awesome SIMD performance will be missed.
+ for _, v := range fn.ClosureVars {
+ if v.Type().IsSIMD() {
+ return 16 // 11 16 ought to be enough.
+ }
+ }
+
+ return 1
+}
+
// inlineBudget determines the max budget for function 'fn' prior to
// analyzing the hairiness of the body of 'fn'. We pass in the pgo
// profile if available (which can change the budget), also a
@@ -186,9 +205,14 @@ func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {
// possibility that a call to the function might have its score
// adjusted downwards. If 'verbose' is set, then print a remark where
// we boost the budget due to PGO.
+// Note that inlineCostOk has the final say on whether an inline will
+// happen; changes here merely make inlines possible.
func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 {
// Update the budget for profile-guided inlining.
budget := int32(inlineMaxBudget)
+
+ budget *= simdCreditMultiplier(fn)
+
if IsPgoHotFunc(fn, profile) {
budget = inlineHotMaxBudget
if verbose {
@@ -202,6 +226,7 @@ func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose boo
// be very liberal here, if the closure is only called once, the budget is large
budget = max(budget, inlineClosureCalledOnceCost)
}
+
return budget
}
@@ -261,17 +286,27 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) {
// locals, and we use this map to produce a pruned Inline.Dcl
// list. See issue 25459 for more context.
+ dbg := ir.MatchAstDump(fn, "inline")
+
visitor := hairyVisitor{
curFunc: fn,
+ debug: isDebugFn(fn),
isBigFunc: IsBigFunc(fn),
budget: budget,
maxBudget: budget,
extraCallCost: cc,
profile: profile,
+ dbg: dbg, // Useful for downstream debugging
}
+
if visitor.tooHairy(fn) {
reason = visitor.reason
+ if dbg {
+ ir.AstDump(fn, "inline, too hairy because "+visitor.reason+", "+ir.FuncName(fn))
+ }
return
+ } else if dbg {
+ ir.AstDump(fn, "inline, OK, "+ir.FuncName(fn))
}
n.Func.Inl = &ir.Inline{
@@ -289,9 +324,9 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) {
// function is inlinable.
func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
if base.Flag.LowerM > 1 {
- fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, cost, fn.Type(), fn.Body)
+ fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n.DiagName(), cost, fn.Type(), fn.Body)
} else if base.Flag.LowerM != 0 {
- fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
+ fmt.Printf("%v: can inline %v\n", ir.Line(fn), n.DiagName())
}
// JSON optimization log output.
if logopt.Enabled() {
@@ -407,6 +442,7 @@ type hairyVisitor struct {
// This is needed to access the current caller in the doNode function.
curFunc *ir.Func
isBigFunc bool
+ debug bool
budget int32
maxBudget int32
reason string
@@ -414,6 +450,17 @@ type hairyVisitor struct {
usedLocals ir.NameSet
do func(ir.Node) bool
profile *pgoir.Profile
+ dbg bool
+}
+
+func isDebugFn(fn *ir.Func) bool {
+ // if n := fn.Nname; n != nil {
+ // if n.Sym().Name == "Int32x8.Transpose8" && n.Sym().Pkg.Path == "simd/archsimd" {
+ // fmt.Printf("isDebugFn '%s' DOT '%s'\n", n.Sym().Pkg.Path, n.Sym().Name)
+ // return true
+ // }
+ // }
+ return false
}
func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
@@ -434,6 +481,9 @@ func (v *hairyVisitor) doNode(n ir.Node) bool {
if n == nil {
return false
}
+ if v.debug {
+ fmt.Printf("%v: doNode %v budget is %d\n", ir.Line(n), n.Op(), v.budget)
+ }
opSwitch:
switch n.Op() {
// Call is okay if inlinable and we have the budget for the body.
@@ -476,6 +526,9 @@ opSwitch:
break opSwitch
case "panicrangestate":
cheap = true
+ case "deferrangefunc":
+ v.reason = "defer call in range func"
+ return true
}
}
}
@@ -551,12 +604,19 @@ opSwitch:
}
if cheap {
+ if v.debug {
+ if ir.IsIntrinsicCall(n) {
+ fmt.Printf("%v: cheap call is also intrinsic, %v\n", ir.Line(n), n)
+ }
+ }
break // treat like any other node, that is, cost of 1
}
if ir.IsIntrinsicCall(n) {
- // Treat like any other node.
- break
+ if v.debug {
+ fmt.Printf("%v: intrinsic call, %v\n", ir.Line(n), n)
+ }
+ break // Treat like any other node.
}
if callee := inlCallee(v.curFunc, n.Fun, v.profile, false); callee != nil && typecheck.HaveInlineBody(callee) {
@@ -583,6 +643,10 @@ opSwitch:
}
}
+ if v.debug {
+ fmt.Printf("%v: costly OCALLFUNC %v\n", ir.Line(n), n)
+ }
+
// Call cost for non-leaf inlining.
v.budget -= extraCost
@@ -592,6 +656,9 @@ opSwitch:
// Things that are too hairy, irrespective of the budget
case ir.OCALL, ir.OCALLINTER:
// Call cost for non-leaf inlining.
+ if v.debug {
+ fmt.Printf("%v: costly OCALL %v\n", ir.Line(n), n)
+ }
v.budget -= v.extraCallCost
case ir.OPANIC:
@@ -754,7 +821,7 @@ opSwitch:
v.budget--
// When debugging, don't stop early, to get full cost of inlining this function
- if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() {
+ if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() && !v.debug {
v.reason = "too expensive"
return true
}
@@ -845,7 +912,7 @@ func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile
}
if fn := inlCallee(callerfn, call.Fun, profile, false); fn != nil && typecheck.HaveInlineBody(fn) {
- return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce)
+ return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce, profile)
}
return nil
}
@@ -893,7 +960,7 @@ var SSADumpInline = func(*ir.Func) {}
// InlineCall allows the inliner implementation to be overridden.
// If it returns nil, the function will not be inlined.
-var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
+var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int, profile *pgoir.Profile) *ir.InlinedCallExpr {
base.Fatalf("inline.InlineCall not overridden")
panic("unreachable")
}
@@ -914,6 +981,8 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal
maxCost = inlineBigFunctionMaxCost
}
+ simdMaxCost := simdCreditMultiplier(callee) * maxCost
+
if callee.ClosureParent != nil {
maxCost *= 2 // favor inlining closures
if closureCalledOnce { // really favor inlining the one call to this closure
@@ -921,6 +990,8 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal
}
}
+ maxCost = max(maxCost, simdMaxCost)
+
metric := callee.Inl.Cost
if inlheur.Enabled() {
score, ok := inlheur.GetCallSiteScore(caller, n)
@@ -1084,7 +1155,7 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
// The result of mkinlcall MUST be assigned back to n, e.g.
//
// n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool) *ir.InlinedCallExpr {
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool, profile *pgoir.Profile) *ir.InlinedCallExpr {
ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, closureCalledOnce, true)
if !ok {
return nil
@@ -1156,19 +1227,19 @@ func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closur
if base.Flag.LowerM != 0 {
if buildcfg.Experiment.NewInliner {
fmt.Printf("%v: inlining call to %v with score %d\n",
- ir.Line(n), fn, score)
+ ir.Line(n), fn.Nname.DiagName(), score)
} else {
- fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
+ fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn.Nname.DiagName())
}
}
if base.Flag.LowerM > 2 {
fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
}
- res := InlineCall(callerfn, n, fn, inlIndex)
+ res := InlineCall(callerfn, n, fn, inlIndex, profile)
if res == nil {
- base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
+ base.FatalfAt(n.Pos(), "inlining call to %v failed", fn.Nname.DiagName())
}
if base.Flag.LowerM > 2 {
diff --git a/src/cmd/compile/internal/inline/inlheur/analyze_func_params.go b/src/cmd/compile/internal/inline/inlheur/analyze_func_params.go
index f6bd84c3f507f1..abfcb3ab328997 100644
--- a/src/cmd/compile/internal/inline/inlheur/analyze_func_params.go
+++ b/src/cmd/compile/internal/inline/inlheur/analyze_func_params.go
@@ -122,9 +122,9 @@ func (pa *paramsAnalyzer) findParamIdx(n *ir.Name) int {
type testfType func(x ir.Node, param *ir.Name, idx int) (bool, bool)
-// paramsAnalyzer invokes function 'testf' on the specified expression
-// 'x' for each parameter, and if the result is TRUE, or's 'flag' into
-// the flags for that param.
+// checkParams invokes function 'testf' on the specified expression 'x'
+// for each parameter. If the result is TRUE, it OR's either 'flag' or 'mayflag'
+// into the flags for that param, depending on whether we are in a conditional context.
func (pa *paramsAnalyzer) checkParams(x ir.Node, flag ParamPropBits, mayflag ParamPropBits, testf testfType) {
for idx, p := range pa.params {
if !pa.top[idx] && pa.values[idx] == ParamNoInfo {
diff --git a/src/cmd/compile/internal/inline/interleaved/interleaved.go b/src/cmd/compile/internal/inline/interleaved/interleaved.go
index c83bbdb718df56..d66f1a84cc9906 100644
--- a/src/cmd/compile/internal/inline/interleaved/interleaved.go
+++ b/src/cmd/compile/internal/inline/interleaved/interleaved.go
@@ -20,6 +20,13 @@ import (
// DevirtualizeAndInlinePackage interleaves devirtualization and inlining on
// all functions within pkg.
func DevirtualizeAndInlinePackage(pkg *ir.Package, profile *pgoir.Profile) {
+ if base.Flag.W > 1 {
+ for _, fn := range typecheck.Target.Funcs {
+ s := fmt.Sprintf("\nbefore devirtualize-and-inline %v", fn.Sym())
+ ir.DumpList(s, fn.Body)
+ }
+ }
+
if profile != nil && base.Debug.PGODevirtualize > 0 {
// TODO(mdempsky): Integrate into DevirtualizeAndInlineFunc below.
ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) {
@@ -254,28 +261,6 @@ func (s *inlClosureState) mark(n ir.Node) ir.Node {
return n // already visited n.X before wrapping
}
- if isTestingBLoop(n) {
- // No inlining nor devirtualization performed on b.Loop body
- if base.Flag.LowerM > 0 {
- fmt.Printf("%v: skip inlining within testing.B.loop for %v\n", ir.Line(n), n)
- }
- // We still want to explore inlining opportunities in other parts of ForStmt.
- nFor, _ := n.(*ir.ForStmt)
- nForInit := nFor.Init()
- for i, x := range nForInit {
- if x != nil {
- nForInit[i] = s.mark(x)
- }
- }
- if nFor.Cond != nil {
- nFor.Cond = s.mark(nFor.Cond)
- }
- if nFor.Post != nil {
- nFor.Post = s.mark(nFor.Post)
- }
- return n
- }
-
if p != nil {
n = p.X // in this case p was copied in from a (marked) inlined function, this is a new unvisited node.
}
@@ -371,29 +356,3 @@ func match(n ir.Node) bool {
}
return false
}
-
-// isTestingBLoop returns true if it matches the node as a
-// testing.(*B).Loop. See issue #61515.
-func isTestingBLoop(t ir.Node) bool {
- if t.Op() != ir.OFOR {
- return false
- }
- nFor, ok := t.(*ir.ForStmt)
- if !ok || nFor.Cond == nil || nFor.Cond.Op() != ir.OCALLFUNC {
- return false
- }
- n, ok := nFor.Cond.(*ir.CallExpr)
- if !ok || n.Fun == nil || n.Fun.Op() != ir.OMETHEXPR {
- return false
- }
- name := ir.MethodExprName(n.Fun)
- if name == nil {
- return false
- }
- if fSym := name.Sym(); fSym != nil && name.Class == ir.PFUNC && fSym.Pkg != nil &&
- fSym.Name == "(*B).Loop" && fSym.Pkg.Path == "testing" {
- // Attempting to match a function call to testing.(*B).Loop
- return true
- }
- return false
-}
diff --git a/src/cmd/compile/internal/ir/class_string.go b/src/cmd/compile/internal/ir/class_string.go
index 11a94c004701ba..bd14c417b2b639 100644
--- a/src/cmd/compile/internal/ir/class_string.go
+++ b/src/cmd/compile/internal/ir/class_string.go
@@ -23,8 +23,9 @@ const _Class_name = "PxxxPEXTERNPAUTOPAUTOHEAPPPARAMPPARAMOUTPTYPEPARAMPFUNC"
var _Class_index = [...]uint8{0, 4, 11, 16, 25, 31, 40, 50, 55}
func (i Class) String() string {
- if i >= Class(len(_Class_index)-1) {
+ idx := int(i) - 0
+ if i < 0 || idx >= len(_Class_index)-1 {
return "Class(" + strconv.FormatInt(int64(i), 10) + ")"
}
- return _Class_name[_Class_index[i]:_Class_index[i+1]]
+ return _Class_name[_Class_index[idx]:_Class_index[idx+1]]
}
diff --git a/src/cmd/compile/internal/ir/dump.go b/src/cmd/compile/internal/ir/dump.go
index 4c218682ea6c75..9dfadc7f250af1 100644
--- a/src/cmd/compile/internal/ir/dump.go
+++ b/src/cmd/compile/internal/ir/dump.go
@@ -9,11 +9,16 @@
package ir
import (
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
"io"
+ "net/url"
"os"
"reflect"
"regexp"
+ "strings"
+ "sync"
"cmd/compile/internal/base"
"cmd/compile/internal/types"
@@ -21,7 +26,7 @@ import (
)
// DumpAny is like FDumpAny but prints to stderr.
-func DumpAny(root interface{}, filter string, depth int) {
+func DumpAny(root any, filter string, depth int) {
FDumpAny(os.Stderr, root, filter, depth)
}
@@ -42,7 +47,7 @@ func DumpAny(root interface{}, filter string, depth int) {
// rather than their type; struct fields with zero values or
// non-matching field names are omitted, and "…" means recursion
// depth has been reached or struct fields have been omitted.
-func FDumpAny(w io.Writer, root interface{}, filter string, depth int) {
+func FDumpAny(w io.Writer, root any, filter string, depth int) {
if root == nil {
fmt.Fprintln(w, "nil")
return
@@ -63,6 +68,167 @@ func FDumpAny(w io.Writer, root interface{}, filter string, depth int) {
p.printf("\n")
}
+// MatchAstDump returns true if the fn matches the value
+// of the astdump debug flag. Fn matches in the following
+// cases:
+//
+// - astdump == name(fn)
+// - astdump == pkgname(fn).name(fn)
+// - astdump == afterslash(pkgname(fn)).name(fn)
+// - astdump begins with a "~" and what follows "~" is a
+// regular expression matching pkgname(fn).name(fn)
+//
+// If MatchAstDump returns true, it also prints to os.Stderr
+//
+// \nir.Match(, ) for \n
+func MatchAstDump(fn *Func, where string) bool {
+ if len(base.Debug.AstDump) == 0 {
+ return false
+ }
+ return matchForDump(fn, base.Ctxt.Pkgpath, where)
+}
+
+// matchForDump is marked noinline to ensure that the exported
+// function MatchAstDump IS inlineable and is also small, because
+// common case is AstDump is not set.
+//
+//go:noinline
+func matchForDump(fn *Func, pkgPath, where string) bool {
+ return MatchPkgFn(pkgPath, FuncName(fn), base.Debug.AstDump)
+}
+
+// MatchPkgFn returns true if pkg and fnName "match" toMatch.
+// "~REGEXP" matches REGEXP against pkgName + "." + fnName
+// "aFunc" matches "aFunc" (in any package)
+// "aPkg.aFunc" matches "aPkg.aFunc"
+// "aPkg/subPkg.aFunc" matches "subPkg.aFunc"
+func MatchPkgFn(pkgName, fnName, toMatch string) bool {
+ if toMatch[0] == '~' {
+ dbgRE := regexp.MustCompile(toMatch[1:])
+ return dbgRE.MatchString(pkgName + "." + fnName)
+ }
+ if fnName == toMatch {
+ return true
+ }
+ matchPkgDotName := func(pkg string) bool {
+ // Allocation-free equality check for toMatch == base.Ctxt.Pkgpath + "." + fnName
+ return len(toMatch) == len(pkg)+1+len(fnName) &&
+ strings.HasPrefix(toMatch, pkg) && toMatch[len(pkg)] == '.' && strings.HasSuffix(toMatch, fnName)
+ }
+ if matchPkgDotName(pkgName) {
+ return true
+ }
+ if l := strings.LastIndexByte(pkgName, '/'); l > 0 && matchPkgDotName(pkgName[l+1:]) {
+ return true
+ }
+
+ return false
+}
+
+// AstDump appends the ast dump for fn to the ast dump file for fn.
+// The generated file name is
+//
+// url.PathEscape(PkgFuncName(fn)) + ".ast"
+//
+// It also prints
+//
+// Writing ast output to \n
+//
+// to os.Stderr.
+func AstDump(fn *Func, why string) {
+ err := withLockAndFile(
+ fn,
+ func(w io.Writer) {
+ FDump(w, why, fn)
+ },
+ )
+ // strip text following comma, for phase names.
+ comma := strings.Index(why, ",")
+ if comma > 0 {
+ why = why[:comma]
+ }
+ DumpNodeHTML(fn, why, fn)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Dump returned error %v\n", err)
+ }
+}
+
+var mu sync.Mutex
+var astDumpFiles = make(map[string]bool)
+
+func escapedFileName(fn *Func, suffix string) string {
+ return EscapedFileName(PkgFuncName(fn), suffix)
+}
+
+// EscapedFileName constructs a file name from fn and suffix,
+// url-path-escaping the function part of the name and replacing it
+// with a hash if it is too long. The suffix is neither escaped
+// nor including in the length calculation, so an excessively
+// creative suffix will result in problems.
+func EscapedFileName(fn, suffix string) string {
+ name := url.PathEscape(fn)
+ if len(name) > 125 { // arbitrary limit on file names, as if anyone types these in by hand
+ hash := sha256.Sum256([]byte(name))
+ name = hex.EncodeToString(hash[:8])
+ }
+ return name + suffix
+}
+
+// withLockAndFile manages ast dump files for various function names
+// and invokes a dumping function to write output, under a lock.
+func withLockAndFile(fn *Func, dump func(io.Writer)) (err error) {
+ name := escapedFileName(fn, ".ast")
+
+ // Ensure that debugging output is not scrambled and is written promptly
+ mu.Lock()
+ defer mu.Unlock()
+ mode := os.O_APPEND | os.O_RDWR
+ if !astDumpFiles[name] {
+ astDumpFiles[name] = true
+ mode = os.O_CREATE | os.O_TRUNC | os.O_RDWR
+ fmt.Fprintf(os.Stderr, "Writing text ast output for %s to %s\n", PkgFuncName(fn), name)
+ }
+
+ fi, err := os.OpenFile(name, mode, 0777)
+ if err != nil {
+ return err
+ }
+ defer func() { err = fi.Close() }()
+ dump(fi)
+ return
+}
+
+var htmlWriters = make(map[*Func]*HTMLWriter)
+var orderedFuncs = []*Func{}
+
+// DumpNodeHTML dumps the node n to the HTML writer for fn.
+// It uses the same phase name as the text dump.
+func DumpNodeHTML(fn *Func, why string, n Node) {
+ mu.Lock()
+ defer mu.Unlock()
+ w, ok := htmlWriters[fn]
+ if !ok {
+ name := escapedFileName(fn, ".html")
+ w = NewHTMLWriter(name, fn, "")
+ htmlWriters[fn] = w
+ orderedFuncs = append(orderedFuncs, fn)
+ }
+ w.WritePhase(why, why)
+}
+
+// CloseHTMLWriters closes the HTML writer for fn, if one exists.
+func CloseHTMLWriters() {
+ mu.Lock()
+ defer mu.Unlock()
+ for _, fn := range orderedFuncs {
+ if w, ok := htmlWriters[fn]; ok {
+ w.Close("Writing html ast output for %s to %s\n", PkgFuncName(w.Func), w.path)
+ delete(htmlWriters, fn)
+ }
+ }
+ orderedFuncs = nil
+}
+
type dumper struct {
output io.Writer
fieldrx *regexp.Regexp // field name filter
@@ -110,7 +276,7 @@ func (p *dumper) Write(data []byte) (n int, err error) {
}
// printf is a convenience wrapper.
-func (p *dumper) printf(format string, args ...interface{}) {
+func (p *dumper) printf(format string, args ...any) {
if _, err := fmt.Fprintf(p, format, args...); err != nil {
panic(err)
}
diff --git a/src/cmd/compile/internal/ir/dump_test.go b/src/cmd/compile/internal/ir/dump_test.go
new file mode 100644
index 00000000000000..6b607062c47e77
--- /dev/null
+++ b/src/cmd/compile/internal/ir/dump_test.go
@@ -0,0 +1,44 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+ "testing"
+)
+
+func testMatch(t *testing.T, pkgName, fnName, toMatch string, match bool) {
+ if MatchPkgFn(pkgName, fnName, toMatch) != match {
+ t.Errorf("%v != matchPkgFn(%s, %s, %s)", match, pkgName, fnName, toMatch)
+ }
+}
+
+func TestMatchPkgFn(t *testing.T) {
+ // "aFunc" matches "aFunc" (in any package)
+ // "aPkg.aFunc" matches "aPkg.aFunc"
+ // "aPkg/subPkg.aFunc" matches "subPkg.aFunc"
+
+ match := func(pkgName, fnName, toMatch string) {
+ if !MatchPkgFn(pkgName, fnName, toMatch) {
+ t.Errorf("matchPkgFn(%s, %s, %s) did not match", pkgName, fnName, toMatch)
+ }
+ }
+ match("aPkg", "AFunc", "AFunc")
+ match("aPkg", "AFunc", "AFunc")
+ match("aPkg", "AFunc", "aPkg.AFunc")
+ match("aPkg/sPkg", "AFunc", "aPkg/sPkg.AFunc")
+ match("aPkg/sPkg", "AFunc", "sPkg.AFunc")
+
+ notmatch := func(pkgName, fnName, toMatch string) {
+ if MatchPkgFn(pkgName, fnName, toMatch) {
+ t.Errorf("matchPkgFn(%s, %s, %s) should not match", pkgName, fnName, toMatch)
+ }
+ }
+ notmatch("aPkg", "AFunc", "BFunc")
+ notmatch("aPkg", "AFunc", "aPkg.BFunc")
+ notmatch("aPkg", "AFunc", "bPkg.AFunc")
+ notmatch("aPkg", "AFunc", "aPkg_AFunc")
+ notmatch("aPkg/sPkg", "AFunc", "aPkg/ssPkg.AFunc")
+ notmatch("aPkg/sPkg", "AFunc", "XPkg.AFunc")
+}
diff --git a/src/cmd/compile/internal/ir/expr.go b/src/cmd/compile/internal/ir/expr.go
index 7a75ff40f2d609..9b4577bd37f332 100644
--- a/src/cmd/compile/internal/ir/expr.go
+++ b/src/cmd/compile/internal/ir/expr.go
@@ -184,14 +184,20 @@ func (n *BinaryExpr) SetOp(op Op) {
// A CallExpr is a function call Fun(Args).
type CallExpr struct {
miniExpr
- Fun Node
- Args Nodes
- DeferAt Node
- RType Node `mknode:"-"` // see reflectdata/helpers.go
- KeepAlive []*Name // vars to be kept alive until call returns
- IsDDD bool
- GoDefer bool // whether this call is part of a go or defer statement
- NoInline bool // whether this call must not be inlined
+ Fun Node
+ Args Nodes
+ DeferAt Node
+ RType Node `mknode:"-"` // see reflectdata/helpers.go
+ KeepAlive []*Name // vars to be kept alive until call returns
+ IsDDD bool
+ GoDefer bool // whether this call is part of a go or defer statement
+ NoInline bool // whether this call must not be inlined
+ UseBuf bool // use stack buffer for backing store (OAPPEND only)
+ AppendNoAlias bool // backing store proven to be unaliased (OAPPEND only)
+ // whether it's a runtime.KeepAlive call the compiler generates to
+ // keep a variable alive. See #73137.
+ IsCompilerVarLive bool
+ Reshape bool
}
func NewCallExpr(pos src.XPos, op Op, fun Node, args []Node) *CallExpr {
@@ -371,6 +377,7 @@ type InlinedCallExpr struct {
miniExpr
Body Nodes
ReturnVars Nodes // must be side-effect free
+ Reshape bool
}
func NewInlinedCallExpr(pos src.XPos, body, retvars []Node) *InlinedCallExpr {
@@ -386,10 +393,16 @@ func (n *InlinedCallExpr) SingleResult() Node {
if have := len(n.ReturnVars); have != 1 {
base.FatalfAt(n.Pos(), "inlined call has %v results, expected 1", have)
}
- if !n.Type().HasShape() && n.ReturnVars[0].Type().HasShape() {
- // If the type of the call is not a shape, but the type of the return value
- // is a shape, we need to do an implicit conversion, so the real type
- // of n is maintained.
+
+ // If the type of the call is not a shape, but the type of the return value
+ // is a shape, we need to do an implicit conversion, so the real type
+ // of n is maintained.
+ needImplicitConv := !n.Type().HasShape() && n.ReturnVars[0].Type().HasShape()
+ if n.Reshape { // or if the inlined call expr needs reshaping.
+ needImplicitConv = true
+ }
+
+ if needImplicitConv {
r := NewConvExpr(n.Pos(), OCONVNOP, n.Type(), n.ReturnVars[0])
r.SetTypecheck(1)
return r
@@ -680,7 +693,7 @@ type TypeAssertExpr struct {
// When set to true, if this assert would panic, then use a nil pointer panic
// instead of an interface conversion panic.
- // It must not be set for type asserts using the commaok form.
+ // It must not be set for type assertions using the commaok form.
UseNilPanic bool
}
@@ -1027,6 +1040,9 @@ func StaticCalleeName(n Node) *Name {
// IsIntrinsicCall reports whether the compiler back end will treat the call as an intrinsic operation.
var IsIntrinsicCall = func(*CallExpr) bool { return false }
+// IsIntrinsicSym reports whether the compiler back end will treat a call to this symbol as an intrinsic operation.
+var IsIntrinsicSym = func(*types.Sym) bool { return false }
+
// SameSafeExpr checks whether it is safe to reuse one of l and r
// instead of computing both. SameSafeExpr assumes that l and r are
// used in the same statement or expression. In order for it to be
@@ -1145,6 +1161,14 @@ func ParamNames(ft *types.Type) []Node {
return args
}
+func RecvParamNames(ft *types.Type) []Node {
+ args := make([]Node, ft.NumRecvs()+ft.NumParams())
+ for i, f := range ft.RecvParams() {
+ args[i] = f.Nname.(*Name)
+ }
+ return args
+}
+
// MethodSym returns the method symbol representing a method name
// associated with a specific receiver type.
//
@@ -1269,3 +1293,28 @@ func MethodExprFunc(n Node) *types.Field {
base.Fatalf("unexpected node: %v (%v)", n, n.Op())
panic("unreachable")
}
+
+// A MoveToHeapExpr takes a slice as input and moves it to the
+// heap (by copying the backing store if it is not already
+// on the heap).
+type MoveToHeapExpr struct {
+ miniExpr
+ Slice Node
+ // An expression that evaluates to a *runtime._type
+ // that represents the slice element type.
+ RType Node
+ // If PreserveCapacity is true, the capacity of
+ // the resulting slice, and all of the elements in
+ // [len:cap], must be preserved.
+ // If PreserveCapacity is false, the resulting
+ // slice may have any capacity >= len, with any
+ // elements in the resulting [len:cap] range zeroed.
+ PreserveCapacity bool
+}
+
+func NewMoveToHeapExpr(pos src.XPos, slice Node) *MoveToHeapExpr {
+ n := &MoveToHeapExpr{Slice: slice}
+ n.pos = pos
+ n.op = OMOVE2HEAP
+ return n
+}
diff --git a/src/cmd/compile/internal/ir/fmt.go b/src/cmd/compile/internal/ir/fmt.go
index ae4ff62652eac0..00bc1af0ffbd39 100644
--- a/src/cmd/compile/internal/ir/fmt.go
+++ b/src/cmd/compile/internal/ir/fmt.go
@@ -574,7 +574,7 @@ func exprFmt(n Node, s fmt.State, prec int) {
// Special case for rune constants.
if typ == types.RuneType || typ == types.UntypedRune {
if x, ok := constant.Uint64Val(val); ok && x <= utf8.MaxRune {
- fmt.Fprintf(s, "%q", x)
+ fmt.Fprintf(s, "%q", rune(x))
return
}
}
@@ -897,11 +897,19 @@ func (l Nodes) Format(s fmt.State, verb rune) {
// Dump
// Dump prints the message s followed by a debug dump of n.
+// This includes all the recursive structure under n.
func Dump(s string, n Node) {
fmt.Printf("%s%+v\n", s, n)
}
+// FDump prints to w the message s followed by a debug dump of n.
+// This includes all the recursive structure under n.
+func FDump(w io.Writer, s string, n Node) {
+ fmt.Fprintf(w, "%s%+v\n", s, n)
+}
+
// DumpList prints the message s followed by a debug dump of each node in the list.
+// This includes all the recursive structure under each node in the list.
func DumpList(s string, list Nodes) {
var buf bytes.Buffer
FDumpList(&buf, s, list)
@@ -909,6 +917,7 @@ func DumpList(s string, list Nodes) {
}
// FDumpList prints to w the message s followed by a debug dump of each node in the list.
+// This includes all the recursive structure under each node in the list.
func FDumpList(w io.Writer, s string, list Nodes) {
io.WriteString(w, s)
dumpNodes(w, list, 1)
diff --git a/src/cmd/compile/internal/ir/func.go b/src/cmd/compile/internal/ir/func.go
index 668537c90e6a40..e3879276d979d5 100644
--- a/src/cmd/compile/internal/ir/func.go
+++ b/src/cmd/compile/internal/ir/func.go
@@ -7,11 +7,12 @@ package ir
import (
"cmd/compile/internal/base"
"cmd/compile/internal/types"
+ "cmd/internal/hash"
"cmd/internal/obj"
"cmd/internal/objabi"
"cmd/internal/src"
+ "encoding/base64"
"fmt"
- "strings"
"unicode/utf8"
)
@@ -90,7 +91,7 @@ type Func struct {
Marks []Mark
FieldTrack map[*obj.LSym]struct{}
- DebugInfo interface{}
+ DebugInfo any
LSym *obj.LSym // Linker object in this function's native ABI (Func.ABI)
Inl *Inline
@@ -148,7 +149,7 @@ type Func struct {
// a WebAssembly function import.
WasmImport *WasmImport
// WasmExport is used by the //go:wasmexport directive to store info about
- // a WebAssembly function import.
+ // a WebAssembly function export.
WasmExport *WasmExport
}
@@ -315,7 +316,9 @@ func PkgFuncName(f *Func) string {
}
s := f.Sym()
pkg := s.Pkg
-
+ if pkg == nil {
+ return "." + s.Name
+ }
return pkg.Path + "." + s.Name
}
@@ -428,10 +431,9 @@ func ClosureDebugRuntimeCheck(clo *ClosureExpr) {
var globClosgen int32
// closureName generates a new unique name for a closure within outerfn at pos.
-func closureName(outerfn *Func, pos src.XPos, why Op) *types.Sym {
- if outerfn.OClosure != nil && outerfn.OClosure.Func.RangeParent != nil {
- outerfn = outerfn.OClosure.Func.RangeParent
- }
+// gen is an optional counter for the closure name. If it is 0, the counter
+// will be computed based on outerfn.
+func closureName(outerfn *Func, pos src.XPos, why Op, gen int) *types.Sym {
pkg := types.LocalPkg
outer := "glob."
var suffix string = "."
@@ -449,7 +451,6 @@ func closureName(outerfn *Func, pos src.XPos, why Op) *types.Sym {
case ODEFER:
suffix = ".deferwrap"
}
- gen := &globClosgen
// There may be multiple functions named "_". In those
// cases, we can't use their individual Closgens as it
@@ -457,30 +458,59 @@ func closureName(outerfn *Func, pos src.XPos, why Op) *types.Sym {
if !IsBlank(outerfn.Nname) {
pkg = outerfn.Sym().Pkg
outer = FuncName(outerfn)
-
- switch why {
- case OCLOSURE:
- gen = &outerfn.funcLitGen
- case ORANGE:
- gen = &outerfn.rangeLitGen
- default:
- gen = &outerfn.goDeferGen
- }
}
- // If this closure was created due to inlining, then incorporate any
- // inlined functions' names into the closure's linker symbol name
- // too (#60324).
+ // If this closure was created due to inlining, find the original
+ // outer function's name for the closure (#60324).
+ var inlHash string
if inlIndex := base.Ctxt.InnermostPos(pos).Base().InliningIndex(); inlIndex >= 0 {
- names := []string{outer}
+ // The compiler doesn't like multiple symbols with the same
+ // name. We make a unique suffix temporarily for the
+ // compiler, and strip it during object file writing, so
+ // it will not be the linker symbol name. For linking,
+ // we use a content hash to disambiguate instead.
+ // We choose the suffix as a hash of the inline call stack.
+ h := hash.New32()
+ fmt.Fprint(h, inlIndex)
base.Ctxt.InlTree.AllParents(inlIndex, func(call obj.InlinedCall) {
- names = append(names, call.Name)
+ if call.Parent >= 0 {
+ fmt.Fprint(h, " ", call.Parent)
+ }
})
- outer = strings.Join(names, ".")
+ inlHash = base64.StdEncoding.EncodeToString(h.Sum(nil)[:8])
+
+ outer = base.Ctxt.InlTree.InlinedFuncName(inlIndex)
+ if pkgPath := base.Ctxt.InlTree.InlinedFuncPkg(inlIndex); pkgPath != "" {
+ pkg = types.NewPkg(pkgPath, "")
+ }
}
- *gen++
- return pkg.Lookup(fmt.Sprintf("%s%s%d", outer, suffix, *gen))
+ if gen == 0 {
+ p := &globClosgen
+ if !IsBlank(outerfn.Nname) {
+ switch why {
+ case OCLOSURE:
+ p = &outerfn.funcLitGen
+ case ORANGE:
+ p = &outerfn.rangeLitGen
+ default:
+ p = &outerfn.goDeferGen
+ }
+ }
+ *p++
+ gen = int(*p)
+ }
+
+ name := fmt.Sprintf("%s%s%d", outer, suffix, gen)
+ if inlHash != "" {
+ // Attach the inline hash (see the comment above).
+ // If it already has a hash, trim it, so we don't include
+ // two hashes for nested closures. The new hash should be
+ // enough to disambiguate.
+ name = obj.TrimInlineHash(name) + "#" + inlHash + "#"
+ }
+
+ return pkg.Lookup(name)
}
// NewClosureFunc creates a new Func to represent a function literal
@@ -498,14 +528,19 @@ func closureName(outerfn *Func, pos src.XPos, why Op) *types.Sym {
// why is the reason we're generating this Func. It can be OCLOSURE
// (for a normal function literal) or OGO or ODEFER (for wrapping a
// call expression that has parameters or results).
-func NewClosureFunc(fpos, cpos src.XPos, why Op, typ *types.Type, outerfn *Func, pkg *Package) *Func {
+//
+// gen is an optional counter for the closure name. If it is 0,
+// the counter will be computed based on outerfn.
+func NewClosureFunc(fpos, cpos src.XPos, why Op, typ *types.Type, outerfn *Func, pkg *Package, gen int) *Func {
if outerfn == nil {
base.FatalfAt(fpos, "outerfn is nil")
}
- fn := NewFunc(fpos, fpos, closureName(outerfn, cpos, why), typ)
+ fn := NewFunc(fpos, fpos, closureName(outerfn, cpos, why, gen), typ)
fn.SetDupok(outerfn.Dupok()) // if the outer function is dupok, so is the closure
+ fn.Linksym().Set(obj.AttrContentAddressable, true)
+
clo := &ClosureExpr{Func: fn}
clo.op = OCLOSURE
clo.pos = cpos
diff --git a/src/cmd/compile/internal/ir/html.go b/src/cmd/compile/internal/ir/html.go
new file mode 100644
index 00000000000000..0f766ae15fe800
--- /dev/null
+++ b/src/cmd/compile/internal/ir/html.go
@@ -0,0 +1,1086 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+ "bufio"
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/types"
+ "cmd/internal/src"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "html"
+ "io"
+ "os"
+ "path/filepath"
+ "reflect"
+ "regexp"
+ "strings"
+)
+
+// An HTMLWriter dumps IR to multicolumn HTML, similar to what the
+// ssa backend does for GOSSAFUNC. This is not the format used for
+// the ast column in GOSSAFUNC output.
+type HTMLWriter struct {
+ HTMLWriterBase
+ Func *Func
+}
+
+type HTMLWriterBase struct {
+ w *BufferedWriterCloser
+ canonIdMap map[any]int
+ prevCanonId int
+ path string
+ prevHash []byte
+ pendingPhases []string
+ pendingTitles []string
+ doDump func(string) func()
+}
+
+func (h *HTMLWriterBase) Init(out io.WriteCloser, reportPath string, doDump func(string) func()) {
+ h.w = NewBufferedWriterCloser(out)
+ h.canonIdMap = make(map[any]int)
+ h.path = reportPath
+ h.doDump = doDump
+}
+
+// BufferedWriterCloser is here to help avoid pre-buffering the whole
+// rendered HTML in memory, which can cause problems for large inputs.
+type BufferedWriterCloser struct {
+ file io.Closer
+ w *bufio.Writer
+}
+
+func (b *BufferedWriterCloser) Write(p []byte) (n int, err error) {
+ return b.w.Write(p)
+}
+
+func (b *BufferedWriterCloser) Close() error {
+ b.w.Flush()
+ b.w = nil
+ return b.file.Close()
+}
+
+func NewBufferedWriterCloser(f io.WriteCloser) *BufferedWriterCloser {
+ return &BufferedWriterCloser{file: f, w: bufio.NewWriter(f)}
+}
+
+func NewHTMLWriter(path string, f *Func, cfgMask string) *HTMLWriter {
+ path = strings.ReplaceAll(path, "/", string(filepath.Separator))
+ out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+ if err != nil {
+ base.Fatalf("%v", err)
+ }
+ reportPath := path
+ if !filepath.IsAbs(reportPath) {
+ pwd, err := os.Getwd()
+ if err != nil {
+ base.Fatalf("%v", err)
+ }
+ reportPath = filepath.Join(pwd, path)
+ }
+ h := HTMLWriter{
+ Func: f,
+ }
+ h.Init(out, reportPath, h.FuncHTML)
+ h.start()
+ return &h
+}
+
+func (h *HTMLWriterBase) Path() string {
+ return h.path
+}
+
+// CanonId assigns indices to nodes based on pointer identity.
+// this helps ensure that output html files don't gratuitously
+// differ from run to run.
+func (h *HTMLWriterBase) CanonId(n any) int {
+ if id := h.canonIdMap[n]; id > 0 {
+ return id
+ }
+ h.prevCanonId++
+ h.canonIdMap[n] = h.prevCanonId
+ return h.prevCanonId
+}
+
+// Fatalf reports an error and exits.
+func (w *HTMLWriterBase) Fatalf(msg string, args ...any) {
+ base.FatalfAt(src.NoXPos, msg, args...)
+}
+
+const (
+ RightArrow = "►" // \u25BA click-to-open (is closed)
+ DownArrow = "▼" // \u25BC click-to-close (is open)
+)
+
+func (w *HTMLWriter) start() {
+ if w == nil {
+ return
+ }
+ escName := html.EscapeString(PkgFuncName(w.Func))
+ w.Print("")
+ w.Print("")
+ w.Printf(`
+AST display for %s
+`, escName, CSS, JS("bloop", "loopvar", "escape", "slice", "walk"), escName)
+ w.Print("")
+ w.Print("")
+ w.Print(html.EscapeString(w.Func.Sym().Name))
+ w.Print(" ")
+ w.Print(`
+help
+
+
+
+Click anywhere on a node (with "cell" cursor) to outline a node and all of its subtrees.
+
+
+Click on a name (with "crosshair" cursor) to highlight every occurrence of a name.
+(Note that all the name nodes are the same node, so those also all outline together).
+
+
+Click on a file, line, or column (with "crosshair" cursor) to highlight positions
+in that file, at that file:line, or at that file:line:column, respectively.
+
+Click on a ` + DownArrow + ` to collapse a subtree, or on a ` + RightArrow + ` to expand a subtree.
+
+
+
+
darkmode
+")
+ w.Print("")
+}
+
+func (w *HTMLWriterBase) Close(format string, args ...any) {
+ if w == nil {
+ return
+ }
+ w.Print(" ")
+ w.Print("
")
+ w.Print("")
+ w.Print("\n")
+ w.w.Close()
+ fmt.Fprintf(os.Stderr, format, args...)
+}
+
+// WritePhase writes f in a column headed by title.
+// phase is used for collapsing columns and should be unique across the table.
+func (w *HTMLWriterBase) WritePhase(phase, title string) {
+ if w == nil {
+ return // avoid generating HTML just to discard it
+ }
+ w.pendingPhases = append(w.pendingPhases, phase)
+ w.pendingTitles = append(w.pendingTitles, title)
+ w.flushPhases()
+}
+
+// flushPhases collects any pending phases and titles, writes them to the html, and resets the pending slices.
+func (w *HTMLWriterBase) flushPhases() {
+ phaseLen := len(w.pendingPhases)
+ if phaseLen == 0 {
+ return
+ }
+ phases := strings.Join(w.pendingPhases, " + ")
+ w.WriteMultiTitleColumn(
+ phases,
+ w.pendingTitles,
+ "allow-x-scroll",
+ w.doDump(w.pendingPhases[phaseLen-1]),
+ )
+ w.pendingPhases = w.pendingPhases[:0]
+ w.pendingTitles = w.pendingTitles[:0]
+}
+
+func (w *HTMLWriterBase) WriteMultiTitleColumn(phase string, titles []string, class string, writeContent func()) {
+ if w == nil {
+ return
+ }
+ id := strings.ReplaceAll(phase, " ", "-")
+ // collapsed column
+ w.Printf("%v
", id)
+ } else {
+ w.Printf(" ", id, class)
+ }
+ for _, title := range titles {
+ w.Print("" + title + " ")
+ }
+ writeContent()
+ w.Print("
")
+ w.Print(" \n")
+}
+
+func (w *HTMLWriterBase) Printf(msg string, v ...any) {
+ if _, err := fmt.Fprintf(w.w, msg, v...); err != nil {
+ w.Fatalf("%v", err)
+ }
+}
+
+func (w *HTMLWriterBase) Print(s string) {
+ if _, err := fmt.Fprint(w.w, s); err != nil {
+ w.Fatalf("%v", err)
+ }
+}
+
+func (w *HTMLWriterBase) indent(n int) {
+ indent(w.w, n)
+}
+
+func (w *HTMLWriter) FuncHTML(phase string) func() {
+ return func() {
+ w.Print("") // use pre for formatting to preserve indentation
+ w.dumpNodesHTML(w.Func.Body, 1)
+ w.Print(" ")
+ }
+}
+
+func (h *HTMLWriter) dumpNodesHTML(list Nodes, depth int) {
+ if len(list) == 0 {
+ h.Print(" ")
+ return
+ }
+
+ for _, n := range list {
+ h.dumpNodeHTML(n, depth)
+ }
+}
+
+const indentString = ". "
+
+// indent prints indentation to w.
+func (h *HTMLWriterBase) indentForToggle(depth int, hasChildren bool) {
+ h.Print("\n")
+ if depth == 0 {
+ return
+ }
+ for i := 0; i < depth-1; i++ {
+ h.Print(indentString)
+ }
+ if hasChildren {
+ // Remove 2 spaces, which have similar rendered width to
+ // leading ir.DownArrow and trailing space.
+ h.Print(indentString[:len(indentString)-2])
+ } else {
+ h.Print(indentString)
+ }
+}
+
+func (h *HTMLWriter) dumpNodeHTML(n Node, depth int) {
+ hasChildren := nodeHasChildren(n)
+ h.indentForToggle(depth, hasChildren)
+
+ if depth > 40 {
+ h.Print("...")
+ return
+ }
+
+ if n == nil {
+ h.Print("NilIrNode")
+ return
+ }
+
+ // For HTML, we want to wrap the node and its details in a span that can be highlighted
+ // across all occurrences of the span in all columns, so it has to be linked to the node ID,
+ // which is its address. Canonicalize the address to a counter so that repeated compiler
+ // runs yield the same html.
+ //
+ // JS Equivalence logic:
+ // var c = elem.classList.item(0);
+ // var x = document.getElementsByClassName(c);
+ //
+ // Tag each class with its canonicalized index.
+
+ h.Printf("", h.CanonId(n))
+ defer h.Printf(" ")
+
+ if hasChildren {
+ h.Print(`` + DownArrow + ` `) // NOTE TRAILING SPACE after !
+ }
+
+ if len(n.Init()) != 0 {
+ h.Print(``)
+ h.Printf("%+v-init", n.Op())
+ h.dumpNodesHTML(n.Init(), depth+1)
+ h.indent(depth)
+ h.Print(` `)
+ }
+
+ switch n.Op() {
+ default:
+ h.Printf("%+v", n.Op())
+ h.dumpNodeHeaderHTML(n)
+
+ case OLITERAL:
+ h.Printf("%+v-%v", n.Op(), html.EscapeString(fmt.Sprintf("%v", n.Val())))
+ h.dumpNodeHeaderHTML(n)
+ return
+
+ case ONAME, ONONAME:
+ if n.Sym() != nil {
+ // Name highlighting:
+ // Create a hash for the symbol name to use as a class
+ // We use the same irValueClicked logic which uses the first class as the identifier
+ name := fmt.Sprintf("%v", n.Sym())
+ hash := sha256.Sum256([]byte(name))
+ symID := "sym-" + hex.EncodeToString(hash[:6])
+ h.Printf("%+v-%+v ", n.Op(), symID, html.EscapeString(name))
+ } else {
+ h.Printf("%+v", n.Op())
+ }
+ h.dumpNodeHeaderHTML(n)
+ return
+
+ case OLINKSYMOFFSET:
+ n := n.(*LinksymOffsetExpr)
+ h.Printf("%+v-%v", n.Op(), html.EscapeString(fmt.Sprintf("%v", n.Linksym)))
+ if n.Offset_ != 0 {
+ h.Printf("%+v", n.Offset_)
+ }
+ h.dumpNodeHeaderHTML(n)
+
+ case OASOP:
+ n := n.(*AssignOpStmt)
+ h.Printf("%+v-%+v", n.Op(), n.AsOp)
+ h.dumpNodeHeaderHTML(n)
+
+ case OTYPE:
+ h.Printf("%+v %+v", n.Op(), html.EscapeString(fmt.Sprintf("%v", n.Sym())))
+ h.dumpNodeHeaderHTML(n)
+ return
+
+ case OCLOSURE:
+ h.Printf("%+v", n.Op())
+ h.dumpNodeHeaderHTML(n)
+
+ case ODCLFUNC:
+ n := n.(*Func)
+ h.Printf("%+v", n.Op())
+ h.dumpNodeHeaderHTML(n)
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+ }
+ fn := n
+ if len(fn.Dcl) > 0 {
+ h.indent(depth)
+ h.Printf("%+v-Dcl", n.Op())
+ for _, dcl := range n.Dcl {
+ h.dumpNodeHTML(dcl, depth+1)
+ }
+ }
+ if len(fn.ClosureVars) > 0 {
+ h.indent(depth)
+ h.Printf("%+v-ClosureVars", n.Op())
+ for _, cv := range fn.ClosureVars {
+ h.dumpNodeHTML(cv, depth+1)
+ }
+ }
+ if len(fn.Body) > 0 {
+ h.indent(depth)
+ h.Printf("%+v-body", n.Op())
+ h.dumpNodesHTML(fn.Body, depth+1)
+ }
+ return
+ }
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+ }
+
+ v := reflect.ValueOf(n).Elem()
+ t := reflect.TypeOf(n).Elem()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ vf := v.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ switch tf.Type.Kind() {
+ case reflect.Interface, reflect.Ptr, reflect.Slice:
+ if vf.IsNil() {
+ continue
+ }
+ }
+ name := strings.TrimSuffix(tf.Name, "_")
+ switch name {
+ case "X", "Y", "Index", "Chan", "Value", "Call":
+ name = ""
+ }
+ switch val := vf.Interface().(type) {
+ case Node:
+ if name != "" {
+ h.indent(depth)
+ h.Printf("%+v-%s", n.Op(), name)
+ }
+ h.dumpNodeHTML(val, depth+1)
+ case Nodes:
+ if len(val) == 0 {
+ continue
+ }
+ if name != "" {
+ h.indent(depth)
+ h.Printf("%+v-%s", n.Op(), name)
+ }
+ h.dumpNodesHTML(val, depth+1)
+ default:
+ if vf.Kind() == reflect.Slice && vf.Type().Elem().Implements(nodeType) {
+ if vf.Len() == 0 {
+ continue
+ }
+ if name != "" {
+ h.indent(depth)
+ h.Printf("%+v-%s", n.Op(), name)
+ }
+ for i, n := 0, vf.Len(); i < n; i++ {
+ h.dumpNodeHTML(vf.Index(i).Interface().(Node), depth+1)
+ }
+ }
+ }
+ }
+}
+
+func nodeHasChildren(n Node) bool {
+ if n == nil {
+ return false
+ }
+ if len(n.Init()) != 0 {
+ return true
+ }
+ switch n.Op() {
+ case OLITERAL, ONAME, ONONAME, OTYPE:
+ return false
+ case ODCLFUNC:
+ n := n.(*Func)
+ return len(n.Dcl) > 0 || len(n.ClosureVars) > 0 || len(n.Body) > 0
+ }
+
+ v := reflect.ValueOf(n).Elem()
+ t := reflect.TypeOf(n).Elem()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ vf := v.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ switch tf.Type.Kind() {
+ case reflect.Interface, reflect.Ptr, reflect.Slice:
+ if vf.IsNil() {
+ continue
+ }
+ }
+ switch val := vf.Interface().(type) {
+ case Node:
+ return true
+ case Nodes:
+ if len(val) > 0 {
+ return true
+ }
+ default:
+ if vf.Kind() == reflect.Slice && vf.Type().Elem().Implements(nodeType) {
+ if vf.Len() > 0 {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+func (h *HTMLWriter) dumpNodeHeaderHTML(n Node) {
+ // print pointer to be able to see identical nodes
+ if base.Debug.DumpPtrs != 0 {
+ h.Printf(" p(%p)", n)
+ }
+
+ if base.Debug.DumpPtrs != 0 && n.Name() != nil && n.Name().Defn != nil {
+ h.Printf(" defn(%p)", n.Name().Defn)
+ }
+
+ if base.Debug.DumpPtrs != 0 && n.Name() != nil && n.Name().Curfn != nil {
+ h.Printf(" curfn(%p)", n.Name().Curfn)
+ }
+ if base.Debug.DumpPtrs != 0 && n.Name() != nil && n.Name().Outer != nil {
+ h.Printf(" outer(%p)", n.Name().Outer)
+ }
+
+ if EscFmt != nil {
+ if esc := EscFmt(n); esc != "" {
+ h.Printf(" %s", html.EscapeString(esc))
+ }
+ }
+
+ if n.Sym() != nil && n.Op() != ONAME && n.Op() != ONONAME && n.Op() != OTYPE {
+ h.Printf(" %+v", html.EscapeString(fmt.Sprintf("%v", n.Sym())))
+ }
+
+ v := reflect.ValueOf(n).Elem()
+ t := v.Type()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ k := tf.Type.Kind()
+ if reflect.Bool <= k && k <= reflect.Complex128 {
+ name := strings.TrimSuffix(tf.Name, "_")
+ vf := v.Field(i)
+ vfi := vf.Interface()
+ if name == "Offset" && vfi == types.BADWIDTH || name != "Offset" && vf.IsZero() {
+ continue
+ }
+ if vfi == true {
+ h.Printf(" %s", name)
+ } else {
+ h.Printf(" %s:%+v", name, html.EscapeString(fmt.Sprintf("%v", vf.Interface())))
+ }
+ }
+ }
+
+ v = reflect.ValueOf(n)
+ t = v.Type()
+ nm := t.NumMethod()
+ for i := 0; i < nm; i++ {
+ tm := t.Method(i)
+ if tm.PkgPath != "" {
+ continue
+ }
+ m := v.Method(i)
+ mt := m.Type()
+ if mt.NumIn() == 0 && mt.NumOut() == 1 && mt.Out(0).Kind() == reflect.Bool {
+ func() {
+ defer func() { recover() }()
+ if m.Call(nil)[0].Bool() {
+ name := strings.TrimSuffix(tm.Name, "_")
+ h.Printf(" %s", name)
+ }
+ }()
+ }
+ }
+
+ if n.Op() == OCLOSURE {
+ n := n.(*ClosureExpr)
+ if fn := n.Func; fn != nil && fn.Nname.Sym() != nil {
+ h.Printf(" fnName(%+v)", html.EscapeString(fmt.Sprintf("%v", fn.Nname.Sym())))
+ }
+ }
+
+ if n.Type() != nil {
+ if n.Op() == OTYPE {
+ h.Printf(" type")
+ }
+ h.Printf(" %+v", html.EscapeString(fmt.Sprintf("%v", n.Type())))
+ }
+ if n.Typecheck() != 0 {
+ h.Printf(" tc(%d)", n.Typecheck())
+ }
+
+ if n.Pos().IsKnown() {
+ h.Print(" ")
+ switch n.Pos().IsStmt() {
+ case src.PosNotStmt:
+ h.Print("_")
+ case src.PosIsStmt:
+ h.Print("+")
+ }
+ sep := ""
+ base.Ctxt.AllPos(n.Pos(), func(pos src.Pos) {
+ h.Print(sep)
+ sep = " "
+ // Hierarchical highlighting:
+ // Click file -> highlight all ranges in this file
+ // Click line -> highlight all ranges at this line (in this file)
+ // Click col -> highlight this specific range
+
+ file := pos.Filename()
+ // Create a hash for the filename to use as a class
+ hash := sha256.Sum256([]byte(file))
+ fileID := "loc-" + hex.EncodeToString(hash[:6])
+ lineID := fmt.Sprintf("%s-L%d", fileID, pos.Line())
+ colID := fmt.Sprintf("%s-C%d", lineID, pos.Col())
+
+ // File part: triggers fileID
+ h.Printf("%s :", fileID, html.EscapeString(filepath.Base(file)))
+ // Line part: triggers lineID (and fileID via class list)
+ h.Printf("%d :", lineID, fileID, pos.Line())
+ // Col part: triggers colID (and lineID, fileID)
+ h.Printf("%d ", colID, lineID, fileID, pos.Col())
+ })
+ h.Print(" ")
+ }
+}
+
+const CSS = `
+`
+
+// safePhaseNameString is a very conservative limit on phase names
+// that can safely be encoded as JavaScript strings by wrapping with
+// double-quotes.
+var safePhaseNameString = regexp.MustCompile("^[a-zA-Z0-9_ .]+$")
+
+func JS(opened ...string) string {
+ var middle strings.Builder
+
+ // "bloop",
+ // "loopvar",
+ // "escape",
+ // "slice",
+ // "walk",
+
+ // This is only for default-display purposes, and the expected strings
+ // are the names of compiler phases. If a wonky name is rejected, the
+ // "harm" is that a pane in the debugging display is not pre-opened.
+ for _, s := range opened {
+ if !safePhaseNameString.MatchString(s) {
+ continue
+ }
+ middle.WriteString("\t\"")
+ middle.WriteString(s)
+ middle.WriteString("\",\n")
+ }
+ return JS1 + middle.String() + JS2
+}
+
+const (
+ JS1 = `
+`
+)
diff --git a/src/cmd/compile/internal/ir/html_test.go b/src/cmd/compile/internal/ir/html_test.go
new file mode 100644
index 00000000000000..c15adcc9acf0a9
--- /dev/null
+++ b/src/cmd/compile/internal/ir/html_test.go
@@ -0,0 +1,104 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/types"
+ "cmd/internal/obj"
+ "cmd/internal/src"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestHTMLWriter(t *testing.T) {
+ // Initialize base.Ctxt to avoid panics
+ base.Ctxt = new(obj.Link)
+
+ // Setup a temporary directory for output
+ tmpDir := t.TempDir()
+
+ // Mock func
+ fn := &Func{
+ Nname: &Name{
+ sym: &types.Sym{Name: "TestFunc"},
+ },
+ }
+ // Func embeds miniExpr, so we might need to set op if checked
+ fn.op = ODCLFUNC
+
+ // Create HTMLWriter
+ outFile := filepath.Join(tmpDir, "test.html")
+ w := NewHTMLWriter(outFile, fn, "")
+ if w == nil {
+ t.Fatalf("Failed to create HTMLWriter")
+ }
+
+ // Write a phase
+ w.WritePhase("phase1", "Phase 1")
+
+ // Register a file/line
+ posBase := src.NewFileBase("test.go", "test.go")
+ // base.Ctxt.PosTable.Register(posBase) -- Not needed/doesn't exist
+ pos := src.MakePos(posBase, 10, 1)
+
+ // Create a dummy node
+ n := &Name{
+ sym: &types.Sym{Name: "VarX"},
+ Class: PAUTO,
+ }
+ n.op = ONAME
+ n.pos = base.Ctxt.PosTable.XPos(pos)
+
+ // Add another phase which actually dumps something interesting
+ fn.Body = []Node{n}
+ w.WritePhase("phase2", "Phase 2")
+
+ // Test escaping
+ n2 := &Name{
+ sym: &types.Sym{Name: ""},
+ Class: PAUTO,
+ }
+ n2.op = ONAME
+ fn.Body = []Node{n2}
+ w.WritePhase("phase3", "Phase 3")
+
+ w.Close("Writing html ast output for %s to %s\n", PkgFuncName(w.Func), w.path)
+
+ // Verify file exists and has content
+ content, err := os.ReadFile(outFile)
+ if err != nil {
+ t.Fatalf("Failed to read output file: %v", err)
+ }
+
+ s := string(content)
+ if len(s) == 0 {
+ t.Errorf("Output file is empty")
+ }
+
+ // Check for Expected strings
+ expected := []string{
+ "",
+ "Phase 1",
+ "Phase 2",
+ "Phase 2",
+ "VarX",
+ "NAME",
+ "<Bad>",
+ "resizer",
+ "loc-",
+ "line-number",
+ "sym-",
+ "variable-name",
+ }
+
+ for _, e := range expected {
+ if !strings.Contains(s, e) {
+ t.Errorf("Output missing %q", e)
+ }
+ }
+}
diff --git a/src/cmd/compile/internal/ir/name.go b/src/cmd/compile/internal/ir/name.go
index 6f8d0a7fcc1989..cf5ec8ec487c5a 100644
--- a/src/cmd/compile/internal/ir/name.go
+++ b/src/cmd/compile/internal/ir/name.go
@@ -43,8 +43,8 @@ type Name struct {
Func *Func // TODO(austin): nil for I.M
Offset_ int64
val constant.Value
- Opt interface{} // for use by escape analysis
- Embed *[]Embed // list of embedded files, for ONAME var
+ Opt any // for use by escape or slice analysis
+ Embed *[]Embed // list of embedded files, for ONAME var
// For a local variable (not param) or extern, the initializing assignment (OAS or OAS2).
// For a closure var, the ONAME node of the original (outermost) captured variable.
@@ -167,6 +167,10 @@ func (*Name) CanBeNtype() {}
func (*Name) CanBeAnSSASym() {}
func (*Name) CanBeAnSSAAux() {}
+// DiagName returns the symbol name for diagnostics.
+// XXX should it be part of the formatter?
+func (n *Name) DiagName() string { return obj.TrimInlineHash(fmt.Sprint(n.Sym())) }
+
// Pragma returns the PragmaFlag for p, which must be for an OTYPE.
func (n *Name) Pragma() PragmaFlag { return n.pragma }
diff --git a/src/cmd/compile/internal/ir/node.go b/src/cmd/compile/internal/ir/node.go
index 8c61bb6ed5a481..509a47e3c970b6 100644
--- a/src/cmd/compile/internal/ir/node.go
+++ b/src/cmd/compile/internal/ir/node.go
@@ -18,6 +18,9 @@ import (
// A Node is the abstract interface to an IR node.
type Node interface {
// Formatting
+ // For debugging output, use one of
+ // Dump/FDump/DumpList/FDumplist (in fmt.go)
+ // DumpAny/FDumpAny (in dump.go)
Format(s fmt.State, verb rune)
// Source position.
@@ -293,6 +296,7 @@ const (
OLINKSYMOFFSET // offset within a name
OJUMPTABLE // A jump table structure for implementing dense expression switches
OINTERFACESWITCH // A type switch with interface cases
+ OMOVE2HEAP // Promote a stack-backed slice to heap
// opcodes for generics
ODYNAMICDOTTYPE // x = i.(T) where T is a type parameter (or derived from a type parameter)
diff --git a/src/cmd/compile/internal/ir/node_gen.go b/src/cmd/compile/internal/ir/node_gen.go
index 2221045c93d215..2ae0a43e49617c 100644
--- a/src/cmd/compile/internal/ir/node_gen.go
+++ b/src/cmd/compile/internal/ir/node_gen.go
@@ -1175,6 +1175,40 @@ func (n *MakeExpr) editChildrenWithHidden(edit func(Node) Node) {
}
}
+func (n *MoveToHeapExpr) Format(s fmt.State, verb rune) { fmtNode(n, s, verb) }
+func (n *MoveToHeapExpr) copy() Node {
+ c := *n
+ c.init = copyNodes(c.init)
+ return &c
+}
+func (n *MoveToHeapExpr) doChildren(do func(Node) bool) bool {
+ if doNodes(n.init, do) {
+ return true
+ }
+ if n.Slice != nil && do(n.Slice) {
+ return true
+ }
+ if n.RType != nil && do(n.RType) {
+ return true
+ }
+ return false
+}
+func (n *MoveToHeapExpr) doChildrenWithHidden(do func(Node) bool) bool {
+ return n.doChildren(do)
+}
+func (n *MoveToHeapExpr) editChildren(edit func(Node) Node) {
+ editNodes(n.init, edit)
+ if n.Slice != nil {
+ n.Slice = edit(n.Slice).(Node)
+ }
+ if n.RType != nil {
+ n.RType = edit(n.RType).(Node)
+ }
+}
+func (n *MoveToHeapExpr) editChildrenWithHidden(edit func(Node) Node) {
+ n.editChildren(edit)
+}
+
func (n *Name) Format(s fmt.State, verb rune) { fmtNode(n, s, verb) }
func (n *NilExpr) Format(s fmt.State, verb rune) { fmtNode(n, s, verb) }
diff --git a/src/cmd/compile/internal/ir/op_string.go b/src/cmd/compile/internal/ir/op_string.go
index 7494beee4c58e0..6c1c99a35d840f 100644
--- a/src/cmd/compile/internal/ir/op_string.go
+++ b/src/cmd/compile/internal/ir/op_string.go
@@ -151,22 +151,24 @@ func _() {
_ = x[OLINKSYMOFFSET-140]
_ = x[OJUMPTABLE-141]
_ = x[OINTERFACESWITCH-142]
- _ = x[ODYNAMICDOTTYPE-143]
- _ = x[ODYNAMICDOTTYPE2-144]
- _ = x[ODYNAMICTYPE-145]
- _ = x[OTAILCALL-146]
- _ = x[OGETG-147]
- _ = x[OGETCALLERSP-148]
- _ = x[OEND-149]
+ _ = x[OMOVE2HEAP-143]
+ _ = x[ODYNAMICDOTTYPE-144]
+ _ = x[ODYNAMICDOTTYPE2-145]
+ _ = x[ODYNAMICTYPE-146]
+ _ = x[OTAILCALL-147]
+ _ = x[OGETG-148]
+ _ = x[OGETCALLERSP-149]
+ _ = x[OEND-150]
}
-const _Op_name = "XXXNAMENONAMETYPELITERALNILADDSUBORXORADDSTRADDRANDANDAPPENDBYTES2STRBYTES2STRTMPRUNES2STRSTR2BYTESSTR2BYTESTMPSTR2RUNESSLICE2ARRSLICE2ARRPTRASAS2AS2DOTTYPEAS2FUNCAS2MAPRAS2RECVASOPCALLCALLFUNCCALLMETHCALLINTERCAPCLEARCLOSECLOSURECOMPLITMAPLITSTRUCTLITARRAYLITSLICELITPTRLITCONVCONVIFACECONVNOPCOPYDCLDCLFUNCDELETEDOTDOTPTRDOTMETHDOTINTERXDOTDOTTYPEDOTTYPE2EQNELTLEGEGTDEREFINDEXINDEXMAPKEYSTRUCTKEYLENMAKEMAKECHANMAKEMAPMAKESLICEMAKESLICECOPYMULDIVMODLSHRSHANDANDNOTNEWNOTBITNOTPLUSNEGORORPANICPRINTPRINTLNPARENSENDSLICESLICEARRSLICESTRSLICE3SLICE3ARRSLICEHEADERSTRINGHEADERRECOVERRECVRUNESTRSELRECV2MINMAXREALIMAGCOMPLEXUNSAFEADDUNSAFESLICEUNSAFESLICEDATAUNSAFESTRINGUNSAFESTRINGDATAMETHEXPRMETHVALUEBLOCKBREAKCASECONTINUEDEFERFALLFORGOTOIFLABELGORANGERETURNSELECTSWITCHTYPESWINLCALLMAKEFACEITABIDATASPTRCFUNCCHECKNILRESULTINLMARKLINKSYMOFFSETJUMPTABLEINTERFACESWITCHDYNAMICDOTTYPEDYNAMICDOTTYPE2DYNAMICTYPETAILCALLGETGGETCALLERSPEND"
+const _Op_name = "XXXNAMENONAMETYPELITERALNILADDSUBORXORADDSTRADDRANDANDAPPENDBYTES2STRBYTES2STRTMPRUNES2STRSTR2BYTESSTR2BYTESTMPSTR2RUNESSLICE2ARRSLICE2ARRPTRASAS2AS2DOTTYPEAS2FUNCAS2MAPRAS2RECVASOPCALLCALLFUNCCALLMETHCALLINTERCAPCLEARCLOSECLOSURECOMPLITMAPLITSTRUCTLITARRAYLITSLICELITPTRLITCONVCONVIFACECONVNOPCOPYDCLDCLFUNCDELETEDOTDOTPTRDOTMETHDOTINTERXDOTDOTTYPEDOTTYPE2EQNELTLEGEGTDEREFINDEXINDEXMAPKEYSTRUCTKEYLENMAKEMAKECHANMAKEMAPMAKESLICEMAKESLICECOPYMULDIVMODLSHRSHANDANDNOTNEWNOTBITNOTPLUSNEGORORPANICPRINTPRINTLNPARENSENDSLICESLICEARRSLICESTRSLICE3SLICE3ARRSLICEHEADERSTRINGHEADERRECOVERRECVRUNESTRSELRECV2MINMAXREALIMAGCOMPLEXUNSAFEADDUNSAFESLICEUNSAFESLICEDATAUNSAFESTRINGUNSAFESTRINGDATAMETHEXPRMETHVALUEBLOCKBREAKCASECONTINUEDEFERFALLFORGOTOIFLABELGORANGERETURNSELECTSWITCHTYPESWINLCALLMAKEFACEITABIDATASPTRCFUNCCHECKNILRESULTINLMARKLINKSYMOFFSETJUMPTABLEINTERFACESWITCHMOVE2HEAPDYNAMICDOTTYPEDYNAMICDOTTYPE2DYNAMICTYPETAILCALLGETGGETCALLERSPEND"
-var _Op_index = [...]uint16{0, 3, 7, 13, 17, 24, 27, 30, 33, 35, 38, 44, 48, 54, 60, 69, 81, 90, 99, 111, 120, 129, 141, 143, 146, 156, 163, 170, 177, 181, 185, 193, 201, 210, 213, 218, 223, 230, 237, 243, 252, 260, 268, 274, 278, 287, 294, 298, 301, 308, 314, 317, 323, 330, 338, 342, 349, 357, 359, 361, 363, 365, 367, 369, 374, 379, 387, 390, 399, 402, 406, 414, 421, 430, 443, 446, 449, 452, 455, 458, 461, 467, 470, 473, 479, 483, 486, 490, 495, 500, 507, 512, 516, 521, 529, 537, 543, 552, 563, 575, 582, 586, 593, 601, 604, 607, 611, 615, 622, 631, 642, 657, 669, 685, 693, 702, 707, 712, 716, 724, 729, 733, 736, 740, 742, 747, 749, 754, 760, 766, 772, 778, 785, 793, 797, 802, 806, 811, 819, 825, 832, 845, 854, 869, 883, 898, 909, 917, 921, 932, 935}
+var _Op_index = [...]uint16{0, 3, 7, 13, 17, 24, 27, 30, 33, 35, 38, 44, 48, 54, 60, 69, 81, 90, 99, 111, 120, 129, 141, 143, 146, 156, 163, 170, 177, 181, 185, 193, 201, 210, 213, 218, 223, 230, 237, 243, 252, 260, 268, 274, 278, 287, 294, 298, 301, 308, 314, 317, 323, 330, 338, 342, 349, 357, 359, 361, 363, 365, 367, 369, 374, 379, 387, 390, 399, 402, 406, 414, 421, 430, 443, 446, 449, 452, 455, 458, 461, 467, 470, 473, 479, 483, 486, 490, 495, 500, 507, 512, 516, 521, 529, 537, 543, 552, 563, 575, 582, 586, 593, 601, 604, 607, 611, 615, 622, 631, 642, 657, 669, 685, 693, 702, 707, 712, 716, 724, 729, 733, 736, 740, 742, 747, 749, 754, 760, 766, 772, 778, 785, 793, 797, 802, 806, 811, 819, 825, 832, 845, 854, 869, 878, 892, 907, 918, 926, 930, 941, 944}
func (i Op) String() string {
- if i >= Op(len(_Op_index)-1) {
+ idx := int(i) - 0
+ if i < 0 || idx >= len(_Op_index)-1 {
return "Op(" + strconv.FormatInt(int64(i), 10) + ")"
}
- return _Op_name[_Op_index[i]:_Op_index[i+1]]
+ return _Op_name[_Op_index[idx]:_Op_index[idx+1]]
}
diff --git a/src/cmd/compile/internal/ir/reassignment.go b/src/cmd/compile/internal/ir/reassignment.go
index ba14d078a2ff26..ff3d08cd5b1bbc 100644
--- a/src/cmd/compile/internal/ir/reassignment.go
+++ b/src/cmd/compile/internal/ir/reassignment.go
@@ -6,6 +6,7 @@ package ir
import (
"cmd/compile/internal/base"
+ "cmd/compile/internal/types"
)
// A ReassignOracle efficiently answers queries about whether local
@@ -21,6 +22,15 @@ type ReassignOracle struct {
// maps candidate name to its defining assignment (or
// for params, defining func).
singleDef map[*Name]Node
+
+ // funcAssigns tracks all known simple assignments (OAS) to
+ // func-typed PAUTO variables. Only func-typed variables are
+ // tracked because this data is used exclusively for callee
+ // resolution in escape analysis. Deletion means the candidate was
+ // invalidated (e.g., addr-taken, non-simple assignment form, or too
+ // many assignments). Assignments inside nested closures are accepted
+ // because the only alternative value is nil, which panics on call.
+ funcAssigns map[*Name][]*AssignStmt
}
// Init initializes the oracle based on the IR in function fn, laying
@@ -33,6 +43,7 @@ func (ro *ReassignOracle) Init(fn *Func) {
// Collect candidate map. Start by adding function parameters
// explicitly.
ro.singleDef = make(map[*Name]Node)
+ ro.funcAssigns = make(map[*Name][]*AssignStmt)
sig := fn.Type()
numParams := sig.NumRecvs() + sig.NumParams()
for _, param := range fn.Dcl[:numParams] {
@@ -48,8 +59,21 @@ func (ro *ReassignOracle) Init(fn *Func) {
var findLocals func(n Node) bool
findLocals = func(n Node) bool {
if nn, ok := n.(*Name); ok {
- if nn.Defn != nil && !nn.Addrtaken() && nn.Class == PAUTO {
- ro.singleDef[nn] = nn.Defn
+ if nn.Class == PAUTO && !nn.Addrtaken() {
+ isFunc := nn.Type().Kind() == types.TFUNC
+ if nn.Defn == nil {
+ // Bare declaration (e.g., "var f func()").
+ if isFunc {
+ ro.funcAssigns[nn] = nil
+ }
+ } else if _, ok := nn.Defn.(*AssignStmt); ok {
+ ro.singleDef[nn] = nn.Defn
+ if isFunc {
+ ro.funcAssigns[nn] = nil
+ }
+ } else {
+ ro.singleDef[nn] = nn.Defn
+ }
}
} else if nn, ok := n.(*ClosureExpr); ok {
Any(nn.Func, findLocals)
@@ -71,26 +95,35 @@ func (ro *ReassignOracle) Init(fn *Func) {
// pruneIfNeeded examines node nn appearing on the left hand side
// of assignment statement asn to see if it contains a reassignment
- // to any nodes in our candidate map ro.singleDef; if a reassignment
- // is found, the corresponding name is deleted from singleDef.
+ // to any nodes in our candidate maps; if a reassignment is found,
+ // the corresponding name is deleted.
pruneIfNeeded := func(nn Node, asn Node) {
oname := outerName(nn)
if oname == nil {
return
}
- defn, ok := ro.singleDef[oname]
- if !ok {
- return
+ if defn, ok := ro.singleDef[oname]; ok {
+ // any assignment to a param invalidates the entry.
+ paramAssigned := oname.Class == PPARAM
+ // assignment to local ok iff assignment is its orig def.
+ localAssigned := (oname.Class == PAUTO && asn != defn)
+ if paramAssigned || localAssigned {
+ // We found an assignment to name N that doesn't
+ // correspond to its original definition; remove
+ // from candidates.
+ delete(ro.singleDef, oname)
+ }
}
- // any assignment to a param invalidates the entry.
- paramAssigned := oname.Class == PPARAM
- // assignment to local ok iff assignment is its orig def.
- localAssigned := (oname.Class == PAUTO && asn != defn)
- if paramAssigned || localAssigned {
- // We found an assignment to name N that doesn't
- // correspond to its original definition; remove
- // from candidates.
- delete(ro.singleDef, oname)
+ if _, ok := ro.funcAssigns[oname]; ok {
+ as, isOAS := asn.(*AssignStmt)
+ if isOAS && isNilAssign(as) {
+ // Zero-value assignment (nil, bare decl), skip.
+ } else if !isOAS {
+ // Not a simple assignment: invalidate.
+ delete(ro.funcAssigns, oname)
+ } else {
+ ro.funcAssigns[oname] = append(ro.funcAssigns[oname], as)
+ }
}
}
@@ -139,6 +172,11 @@ func (ro *ReassignOracle) StaticValue(n Node) Node {
continue
}
+ if n.Op() == OPAREN {
+ n = n.(*ParenExpr).X
+ continue
+ }
+
n1 := ro.staticValue1(n)
if n1 == nil {
if consistencyCheckEnabled {
@@ -203,3 +241,26 @@ func (ro *ReassignOracle) Reassigned(n *Name) bool {
}
return result
}
+
+// FuncAssignments returns all known simple assignments to a func-typed
+// variable. For variables defined with := and a non-zero value, the
+// defining assignment is included. Returns nil if the variable is not
+// func-typed, was invalidated (addr-taken, non-simple assignment,
+// too many assignments), or has no tracked assignments. Assignments
+// inside nested closures are accepted because the only alternative
+// value is nil, which panics on call.
+func (ro *ReassignOracle) FuncAssignments(name *Name) []*AssignStmt {
+ return ro.funcAssigns[name.Canonical()]
+}
+
+// isNilAssign reports whether as has a nil or absent RHS.
+func isNilAssign(as *AssignStmt) bool {
+ if as.Y == nil {
+ return true
+ }
+ y := as.Y
+ for y.Op() == OCONVNOP {
+ y = y.(*ConvExpr).X
+ }
+ return IsNil(y)
+}
diff --git a/src/cmd/compile/internal/ir/sizeof_test.go b/src/cmd/compile/internal/ir/sizeof_test.go
index 14b6b4f3cd4750..b805155e6e3b79 100644
--- a/src/cmd/compile/internal/ir/sizeof_test.go
+++ b/src/cmd/compile/internal/ir/sizeof_test.go
@@ -16,9 +16,9 @@ func TestSizeof(t *testing.T) {
const _64bit = unsafe.Sizeof(uintptr(0)) == 8
var tests = []struct {
- val interface{} // type as a value
- _32bit uintptr // size on 32bit platforms
- _64bit uintptr // size on 64bit platforms
+ val any // type as a value
+ _32bit uintptr // size on 32bit platforms
+ _64bit uintptr // size on 64bit platforms
}{
{Func{}, 184, 312},
{Name{}, 96, 160},
diff --git a/src/cmd/compile/internal/ir/stmt.go b/src/cmd/compile/internal/ir/stmt.go
index 0801ecdd9e8722..affa5f4551e450 100644
--- a/src/cmd/compile/internal/ir/stmt.go
+++ b/src/cmd/compile/internal/ir/stmt.go
@@ -42,6 +42,7 @@ func (*Decl) isStmt() {}
type Stmt interface {
Node
isStmt()
+ PtrInit() *Nodes
}
// A miniStmt is a miniNode with extra fields common to statements.
diff --git a/src/cmd/compile/internal/ir/symtab.go b/src/cmd/compile/internal/ir/symtab.go
index f8eb4578809312..57cde66fb5601c 100644
--- a/src/cmd/compile/internal/ir/symtab.go
+++ b/src/cmd/compile/internal/ir/symtab.go
@@ -29,12 +29,20 @@ type symsStruct struct {
GCWriteBarrier [8]*obj.LSym
Goschedguarded *obj.LSym
Growslice *obj.LSym
+ GrowsliceBuf *obj.LSym
+ GrowsliceBufNoAlias *obj.LSym
+ GrowsliceNoAlias *obj.LSym
+ MoveSlice *obj.LSym
+ MoveSliceNoScan *obj.LSym
+ MoveSliceNoCap *obj.LSym
+ MoveSliceNoCapNoScan *obj.LSym
InterfaceSwitch *obj.LSym
MallocGC *obj.LSym
- MallocGCSmallNoScan [27]*obj.LSym
- MallocGCSmallScanNoHeader [27]*obj.LSym
- MallocGCTiny [16]*obj.LSym
+ MallocGCTiny *obj.LSym
+ MallocGCSmallNoScan [8]*obj.LSym
+ MallocGCSmallScanNoHeader [8]*obj.LSym
Memmove *obj.LSym
+ Memequal *obj.LSym
Msanread *obj.LSym
Msanwrite *obj.LSym
Msanmove *obj.LSym
@@ -48,6 +56,7 @@ type symsStruct struct {
PanicdottypeI *obj.LSym
Panicnildottype *obj.LSym
Panicoverflow *obj.LSym
+ PanicSimdImm *obj.LSym
Racefuncenter *obj.LSym
Racefuncexit *obj.LSym
Raceread *obj.LSym
@@ -58,22 +67,24 @@ type symsStruct struct {
WBZero *obj.LSym
WBMove *obj.LSym
// Wasm
- SigPanic *obj.LSym
- Staticuint64s *obj.LSym
- Typedmemmove *obj.LSym
- Udiv *obj.LSym
- WriteBarrier *obj.LSym
- Zerobase *obj.LSym
- ZeroVal *obj.LSym
- ARM64HasATOMICS *obj.LSym
- ARMHasVFPv4 *obj.LSym
- Loong64HasLAMCAS *obj.LSym
- Loong64HasLAM_BH *obj.LSym
- Loong64HasLSX *obj.LSym
- RISCV64HasZbb *obj.LSym
- X86HasFMA *obj.LSym
- X86HasPOPCNT *obj.LSym
- X86HasSSE41 *obj.LSym
+ SigPanic *obj.LSym
+ Staticuint64s *obj.LSym
+ Typedmemmove *obj.LSym
+ Udiv *obj.LSym
+ WriteBarrier *obj.LSym
+ Zerobase *obj.LSym
+ ZeroVal *obj.LSym
+ ARM64HasATOMICS *obj.LSym
+ ARMHasVFPv4 *obj.LSym
+ Loong64HasLAMCAS *obj.LSym
+ Loong64HasLAM_BH *obj.LSym
+ Loong64HasDBAR_HINTS *obj.LSym
+ Loong64HasLSX *obj.LSym
+ RISCV64HasZbb *obj.LSym
+ X86HasAVX *obj.LSym
+ X86HasFMA *obj.LSym
+ X86HasPOPCNT *obj.LSym
+ X86HasSSE41 *obj.LSym
// Wasm
WasmDiv *obj.LSym
// Wasm
diff --git a/src/cmd/compile/internal/liveness/arg.go b/src/cmd/compile/internal/liveness/arg.go
index 77960f5e15a1f9..69ead6ecc5fdc8 100644
--- a/src/cmd/compile/internal/liveness/arg.go
+++ b/src/cmd/compile/internal/liveness/arg.go
@@ -132,7 +132,7 @@ func ArgLiveness(fn *ir.Func, f *ssa.Func, pp *objw.Progs) (blockIdx, valueIdx m
}
nargs := int32(len(lv.args))
- bulk := bitvec.NewBulk(nargs, int32(len(f.Blocks)*2), fn.Pos())
+ bulk := bitvec.NewBulk(nargs, int32(len(f.Blocks)*2))
for _, b := range f.Blocks {
be := &lv.be[b.ID]
be.livein = bulk.Next()
@@ -316,6 +316,7 @@ func (lv *argLiveness) emit() *obj.LSym {
lsym := base.Ctxt.Lookup(lv.fn.LSym.Name + ".argliveinfo")
lsym.Set(obj.AttrContentAddressable, true)
+ lsym.Align = 1
off := objw.Uint8(lsym, 0, argOffsets[0]) // smallest offset that needs liveness info.
for idx, live := range livenessMaps {
diff --git a/src/cmd/compile/internal/liveness/mergelocals.go b/src/cmd/compile/internal/liveness/mergelocals.go
index 6967ee016ee710..b9be0dd4c9768d 100644
--- a/src/cmd/compile/internal/liveness/mergelocals.go
+++ b/src/cmd/compile/internal/liveness/mergelocals.go
@@ -513,8 +513,26 @@ func (cs *cstate) populateIndirectUseTable(cands []*ir.Name) ([]*ir.Name, []cand
}
}
}
- for _, arg := range v.Args {
+ for idx, arg := range v.Args {
if nc, ok := pendingUses[arg.ID]; ok {
+ if !v.AddrSinkArg(idx) {
+ // If this op may propagate its input address
+ // to somewhere else, we must track where that
+ // somewhere else might be. See issue 80127.
+ if v.Type.IsMemory() {
+ // Might be stored to memory. Give up.
+ continue
+ }
+ // Some sort of address arithmetic.
+ if _, ok := pendingUses[v.ID]; ok {
+ // v has used multiple addresses, which is something
+ // we can't keep track of. Give up.
+ continue
+ }
+ // Treat this op as producing the address of the same variable
+ // that its argument was the address of.
+ pendingUses[v.ID] = nameCount{n: nc.n, count: v.Uses}
+ }
// We found a use of some value that took the
// address of nc.n. Record this inst as a
// potential indirect use.
diff --git a/src/cmd/compile/internal/liveness/plive.go b/src/cmd/compile/internal/liveness/plive.go
index 9b8f15e9c276f7..02643fa352efde 100644
--- a/src/cmd/compile/internal/liveness/plive.go
+++ b/src/cmd/compile/internal/liveness/plive.go
@@ -17,6 +17,7 @@ package liveness
import (
"cmp"
"fmt"
+ "math"
"os"
"slices"
"sort"
@@ -427,7 +428,7 @@ func newliveness(fn *ir.Func, f *ssa.Func, vars []*ir.Name, idx map[*ir.Name]int
nblocks := int32(len(f.Blocks))
nvars := int32(len(vars))
- bulk := bitvec.NewBulk(nvars, nblocks*7, fn.Pos())
+ bulk := bitvec.NewBulk(nvars, nblocks*4)
for _, b := range f.Blocks {
be := lv.blockEffects(b)
@@ -1367,6 +1368,10 @@ func (lv *Liveness) emit() (argsSym, liveSym *obj.LSym) {
loff := objw.Uint32(&liveSymTmp, 0, uint32(len(lv.stackMaps))) // number of bitmaps
loff = objw.Uint32(&liveSymTmp, loff, uint32(locals.N)) // number of bits in each bitmap
+ // Check for overflow before serializing stackmaps
+ checkStackmapOverflow(args, len(lv.stackMaps), lv.fn.Pos())
+ checkStackmapOverflow(locals, len(lv.stackMaps), lv.fn.Pos())
+
for _, live := range lv.stackMaps {
args.Clear()
locals.Clear()
@@ -1473,6 +1478,7 @@ func (lv *Liveness) emitStackObjects() *obj.LSym {
// Format must match runtime/stack.go:stackObjectRecord.
x := base.Ctxt.Lookup(lv.fn.LSym.Name + ".stkobj")
x.Set(obj.AttrContentAddressable, true)
+ x.Align = int16(types.PtrSize) // see https://go.dev/issue/80668
lv.fn.LSym.Func().StackObjects = x
off := 0
off = objw.Uintptr(x, off, uint64(len(vars)))
@@ -1534,6 +1540,9 @@ func isfat(t *types.Type) bool {
}
return true
case types.TSTRUCT:
+ if t.IsSIMD() {
+ return false
+ }
// Struct with 1 field, check if field is fat
if t.NumFields() == 1 {
return isfat(t.Field(0).Type)
@@ -1563,6 +1572,10 @@ func WriteFuncMap(fn *ir.Func, abiInfo *abi.ABIParamResultInfo) {
if fn.Type().NumResults() > 0 {
nbitmap = 2
}
+
+ // defensive check: function arguments can't realistically be large enough for overflow here
+ checkStackmapOverflow(bv, nbitmap, fn.Pos())
+
lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap")
lsym.Set(obj.AttrLinkname, true) // allow args_stackmap referenced from assembly
off := objw.Uint32(lsym, 0, uint32(nbitmap))
@@ -1580,3 +1593,18 @@ func WriteFuncMap(fn *ir.Func, abiInfo *abi.ABIParamResultInfo) {
objw.Global(lsym, int32(off), obj.RODATA|obj.LOCAL)
}
+
+// checkStackmapOverflow checks for potential overflow in runtime stackmap reading.
+// Runtime computes: n * ((nbit+7)/8) using int32 arithmetic.
+// See runtime.stackmapdata implementation.
+func checkStackmapOverflow(bv bitvec.BitVec, count int, pos src.XPos) {
+ if bv.N <= 0 || count <= 0 {
+ return
+ }
+ bytesPerBitVec := (int64(bv.N) + 7) >> 3
+ totalBytes := bytesPerBitVec * int64(count)
+ if totalBytes > math.MaxInt32 {
+ // runtime.stackmap has to support 64-bit values to avoid this restriction, see issue 77170
+ base.FatalfAt(pos, "liveness stackmaps are too large: nbit=%d count=%d totalBytes=%d exceeds MaxInt32", bv.N, count, totalBytes)
+ }
+}
diff --git a/src/cmd/compile/internal/logopt/log_opts.go b/src/cmd/compile/internal/logopt/log_opts.go
index d08f6fb5d6d92f..c47c9ee5afb23b 100644
--- a/src/cmd/compile/internal/logopt/log_opts.go
+++ b/src/cmd/compile/internal/logopt/log_opts.go
@@ -224,12 +224,12 @@ type Diagnostic struct {
// A LoggedOpt is what the compiler produces and accumulates,
// to be converted to JSON for human or IDE consumption.
type LoggedOpt struct {
- pos src.XPos // Source code position at which the event occurred. If it is inlined, outer and all inlined locations will appear in JSON.
- lastPos src.XPos // Usually the same as pos; current exception is for reporting entire range of transformed loops
- compilerPass string // Compiler pass. For human/adhoc consumption; does not appear in JSON (yet)
- functionName string // Function name. For human/adhoc consumption; does not appear in JSON (yet)
- what string // The (non) optimization; "nilcheck", "boundsCheck", "inline", "noInline"
- target []interface{} // Optional target(s) or parameter(s) of "what" -- what was inlined, why it was not, size of copy, etc. 1st is most important/relevant.
+ pos src.XPos // Source code position at which the event occurred. If it is inlined, outer and all inlined locations will appear in JSON.
+ lastPos src.XPos // Usually the same as pos; current exception is for reporting entire range of transformed loops
+ compilerPass string // Compiler pass. For human/adhoc consumption; does not appear in JSON (yet)
+ functionName string // Function name. For human/adhoc consumption; does not appear in JSON (yet)
+ what string // The (non) optimization; "nilcheck", "boundsCheck", "inline", "noInline"
+ target []any // Optional target(s) or parameter(s) of "what" -- what was inlined, why it was not, size of copy, etc. 1st is most important/relevant.
}
type logFormat uint8
@@ -325,7 +325,7 @@ var mu = sync.Mutex{} // mu protects loggedOpts.
// Pos is the source position (including inlining), what is the message, pass is which pass created the message,
// funcName is the name of the function
// A typical use for this to accumulate an explanation for a missed optimization, for example, why did something escape?
-func NewLoggedOpt(pos, lastPos src.XPos, what, pass, funcName string, args ...interface{}) *LoggedOpt {
+func NewLoggedOpt(pos, lastPos src.XPos, what, pass, funcName string, args ...any) *LoggedOpt {
pass = strings.ReplaceAll(pass, " ", "_")
return &LoggedOpt{pos, lastPos, pass, funcName, what, args}
}
@@ -333,7 +333,7 @@ func NewLoggedOpt(pos, lastPos src.XPos, what, pass, funcName string, args ...in
// LogOpt logs information about a (usually missed) optimization performed by the compiler.
// Pos is the source position (including inlining), what is the message, pass is which pass created the message,
// funcName is the name of the function.
-func LogOpt(pos src.XPos, what, pass, funcName string, args ...interface{}) {
+func LogOpt(pos src.XPos, what, pass, funcName string, args ...any) {
if Format == None {
return
}
@@ -346,7 +346,7 @@ func LogOpt(pos src.XPos, what, pass, funcName string, args ...interface{}) {
// LogOptRange is the same as LogOpt, but includes the ability to express a range of positions,
// not just a point.
-func LogOptRange(pos, lastPos src.XPos, what, pass, funcName string, args ...interface{}) {
+func LogOptRange(pos, lastPos src.XPos, what, pass, funcName string, args ...any) {
if Format == None {
return
}
diff --git a/src/cmd/compile/internal/loong64/ssa.go b/src/cmd/compile/internal/loong64/ssa.go
index 5e554282839d2e..090d3d186c53b4 100644
--- a/src/cmd/compile/internal/loong64/ssa.go
+++ b/src/cmd/compile/internal/loong64/ssa.go
@@ -187,6 +187,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpLOONG64DIVD,
ssa.OpLOONG64MULV, ssa.OpLOONG64MULHV, ssa.OpLOONG64MULHVU, ssa.OpLOONG64MULH, ssa.OpLOONG64MULHU,
ssa.OpLOONG64DIVV, ssa.OpLOONG64REMV, ssa.OpLOONG64DIVVU, ssa.OpLOONG64REMVU,
+ ssa.OpLOONG64MULWVW, ssa.OpLOONG64MULWVWU,
ssa.OpLOONG64FCOPYSGD:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
@@ -528,6 +529,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpLOONG64BITREV4B,
ssa.OpLOONG64BITREVW,
ssa.OpLOONG64BITREVV,
+ ssa.OpLOONG64ABSF,
ssa.OpLOONG64ABSD:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
@@ -537,7 +539,11 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpLOONG64VPCNT64,
ssa.OpLOONG64VPCNT32,
- ssa.OpLOONG64VPCNT16:
+ ssa.OpLOONG64VPCNT16,
+ ssa.OpLOONG64FRINTND,
+ ssa.OpLOONG64FRINTZD,
+ ssa.OpLOONG64FRINTPD,
+ ssa.OpLOONG64FRINTMD:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = ((v.Args[0].Reg() - loong64.REG_F0) & 31) + loong64.REG_V0
@@ -574,7 +580,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
}
case ssa.OpLOONG64LoweredZeroLoop:
ptrReg := v.Args[0].Reg()
- countReg := v.RegTmp()
+ endReg := v.RegTmp()
+ flagReg := int16(loong64.REGTMP)
var off int64
n := v.AuxInt
loopSize := int64(64)
@@ -587,58 +594,109 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
// vs
// 16 instuctions in the straightline code
// Might as well use straightline code.
- v.Fatalf("ZeroLoop size tool small %d", n)
+ v.Fatalf("ZeroLoop size too small %d", n)
}
- // Put iteration count in a register.
- // MOVV $n/loopSize, countReg
- p := s.Prog(loong64.AMOVV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = n / loopSize
- p.To.Type = obj.TYPE_REG
- p.To.Reg = countReg
- cntInit := p
+ // ADDV n - n%loopSize, ptrReg, endReg
+ // MOVBU ir.Syms.Loong64HasLSX, flagReg
+ // BNE flagReg, lsxInit
+ // genericLoop:
+ // for off = 0; off < loopSize; off += 8 {
+ // zero8(s, ptrReg, off)
+ // }
+ // ADDV $loopSize, ptrReg
+ // BNE endReg, ptrReg, genericLoop
+ // JMP tail
+ // lsxInit:
+ // VXORV V31, V31, V31
+ // lsxLoop:
+ // for off = 0; off < loopSize; off += 16 {
+ // zero16(s, V31, ptrReg, off)
+ // }
+ // ADDV $loopSize, ptrReg
+ // BNE endReg, ptrReg, lsxLoop
+ // tail:
+ // n %= loopSize
+ // for off = 0; n >= 8; off += 8, n -= 8 {
+ // zero8(s, ptrReg, off)
+ // }
+ //
+ // if n != 0 {
+ // zero8(s, ptrReg, off+n-8)
+ // }
+
+ p1 := s.Prog(loong64.AADDV)
+ p1.From.Type = obj.TYPE_CONST
+ p1.From.Offset = n - n%loopSize
+ p1.Reg = ptrReg
+ p1.To.Type = obj.TYPE_REG
+ p1.To.Reg = endReg
- // Zero loopSize bytes starting at ptrReg.
- for range loopSize / 8 {
- // MOVV ZR, off(ptrReg)
+ p2 := s.Prog(loong64.AMOVBU)
+ p2.From.Type = obj.TYPE_MEM
+ p2.From.Name = obj.NAME_EXTERN
+ p2.From.Sym = ir.Syms.Loong64HasLSX
+ p2.To.Type = obj.TYPE_REG
+ p2.To.Reg = flagReg
+
+ p3 := s.Prog(loong64.ABNE)
+ p3.From.Type = obj.TYPE_REG
+ p3.From.Reg = flagReg
+ p3.To.Type = obj.TYPE_BRANCH
+
+ for off = 0; off < loopSize; off += 8 {
zero8(s, ptrReg, off)
- off += 8
}
- // Increment ptrReg by loopSize.
- // ADDV $loopSize, ptrReg
- p = s.Prog(loong64.AADDV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = loopSize
- p.To.Type = obj.TYPE_REG
- p.To.Reg = ptrReg
+ p4 := s.Prog(loong64.AADDV)
+ p4.From.Type = obj.TYPE_CONST
+ p4.From.Offset = loopSize
+ p4.To.Type = obj.TYPE_REG
+ p4.To.Reg = ptrReg
- // Decrement loop count.
- // SUBV $1, countReg
- p = s.Prog(loong64.ASUBV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = 1
- p.To.Type = obj.TYPE_REG
- p.To.Reg = countReg
+ p5 := s.Prog(loong64.ABNE)
+ p5.From.Type = obj.TYPE_REG
+ p5.From.Reg = endReg
+ p5.Reg = ptrReg
+ p5.To.Type = obj.TYPE_BRANCH
+ p5.To.SetTarget(p3.Link)
- // Jump to loop header if we're not done yet.
- // BNE countReg, loop header
- p = s.Prog(loong64.ABNE)
- p.From.Type = obj.TYPE_REG
- p.From.Reg = countReg
- p.To.Type = obj.TYPE_BRANCH
- p.To.SetTarget(cntInit.Link)
+ p6 := s.Prog(obj.AJMP)
+ p6.To.Type = obj.TYPE_BRANCH
+
+ p7 := s.Prog(loong64.AVXORV)
+ p7.From.Type = obj.TYPE_REG
+ p7.From.Reg = loong64.REG_V31
+ p7.To.Type = obj.TYPE_REG
+ p7.To.Reg = loong64.REG_V31
+ p3.To.SetTarget(p7)
+
+ for off = 0; off < loopSize; off += 16 {
+ zero16(s, loong64.REG_V31, ptrReg, off)
+ }
+
+ p8 := s.Prog(loong64.AADDV)
+ p8.From.Type = obj.TYPE_CONST
+ p8.From.Offset = loopSize
+ p8.To.Type = obj.TYPE_REG
+ p8.To.Reg = ptrReg
+
+ p9 := s.Prog(loong64.ABNE)
+ p9.From.Type = obj.TYPE_REG
+ p9.From.Reg = endReg
+ p9.Reg = ptrReg
+ p9.To.Type = obj.TYPE_BRANCH
+ p9.To.SetTarget(p7.Link)
+
+ p10 := s.Prog(obj.ANOP)
+ p6.To.SetTarget(p10)
// Multiples of the loop size are now done.
n %= loopSize
-
- off = 0
// Write any fractional portion.
- for n >= 8 {
- // MOVV ZR, off(ptrReg)
+ for off = 0; n >= 8; off += 8 {
+ // MOVV ZR, off(ptrReg)
zero8(s, ptrReg, off)
- off += 8
n -= 8
}
@@ -652,7 +710,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
if dstReg == srcReg {
break
}
- tmpReg := int16(loong64.REG_R20)
+ tmpReg := int16(loong64.REG_R23)
n := v.AuxInt
if n < 16 {
v.Fatalf("Move too small %d", n)
@@ -678,8 +736,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
if dstReg == srcReg {
break
}
- countReg := int16(loong64.REG_R20)
- tmpReg := int16(loong64.REG_R21)
+ srcEndReg := int16(loong64.REG_R23)
+ tmpReg := int16(loong64.REG_R24)
var off int64
n := v.AuxInt
loopSize := int64(64)
@@ -692,78 +750,74 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
// vs
// 16 instructions in the straightline code
// Might as well use straightline code.
- v.Fatalf("ZeroLoop size too small %d", n)
+ v.Fatalf("MoveLoop size too small %d", n)
}
- // Put iteration count in a register.
- // MOVV $n/loopSize, countReg
- p := s.Prog(loong64.AMOVV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = n / loopSize
- p.To.Type = obj.TYPE_REG
- p.To.Reg = countReg
- cntInit := p
+ // ADDV n - n%loopSize, srcReg, srcEndReg
+ // Loop8:
+ // for off = 0; off < loopSize; off += 8 {
+ // move8(s, srcReg, dstReg, tmpReg, off)
+ // }
+ // ADDV $loopSize, srcReg
+ // ADDV $loopSize, dstReg
+ // BNE srcEndReg, srcReg, Loop8
+ //
+ // n %= loopSize
+ // for off = 0; n >= 8; off += 8 {
+ // move8(s, srcReg, dstReg, tmpReg, off)
+ // n -= 8
+ // }
+ //
+ // if n != 0 {
+ // move8(s, srcReg, dstReg, tmpReg, off+n-8)
+ // }
+
+ p1 := s.Prog(loong64.AADDV)
+ p1.From.Type = obj.TYPE_CONST
+ p1.From.Offset = n - n%loopSize
+ p1.Reg = srcReg
+ p1.To.Type = obj.TYPE_REG
+ p1.To.Reg = srcEndReg
- // Move loopSize bytes starting at srcReg to dstReg.
- for range loopSize / 8 {
- // MOVV off(srcReg), tmpReg
- // MOVV tmpReg, off(dstReg)
+ for off = 0; off < loopSize; off += 8 {
move8(s, srcReg, dstReg, tmpReg, off)
- off += 8
}
- // Increment srcReg and destReg by loopSize.
- // ADDV $loopSize, srcReg
- p = s.Prog(loong64.AADDV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = loopSize
- p.To.Type = obj.TYPE_REG
- p.To.Reg = srcReg
- // ADDV $loopSize, dstReg
- p = s.Prog(loong64.AADDV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = loopSize
- p.To.Type = obj.TYPE_REG
- p.To.Reg = dstReg
+ p2 := s.Prog(loong64.AADDV)
+ p2.From.Type = obj.TYPE_CONST
+ p2.From.Offset = loopSize
+ p2.To.Type = obj.TYPE_REG
+ p2.To.Reg = srcReg
- // Decrement loop count.
- // SUBV $1, countReg
- p = s.Prog(loong64.ASUBV)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = 1
- p.To.Type = obj.TYPE_REG
- p.To.Reg = countReg
+ p3 := s.Prog(loong64.AADDV)
+ p3.From.Type = obj.TYPE_CONST
+ p3.From.Offset = loopSize
+ p3.To.Type = obj.TYPE_REG
+ p3.To.Reg = dstReg
- // Jump to loop header if we're not done yet.
- // BNE countReg, loop header
- p = s.Prog(loong64.ABNE)
- p.From.Type = obj.TYPE_REG
- p.From.Reg = countReg
- p.To.Type = obj.TYPE_BRANCH
- p.To.SetTarget(cntInit.Link)
+ p4 := s.Prog(loong64.ABNE)
+ p4.From.Type = obj.TYPE_REG
+ p4.From.Reg = srcEndReg
+ p4.Reg = srcReg
+ p4.To.Type = obj.TYPE_BRANCH
+ p4.To.SetTarget(p1.Link)
// Multiples of the loop size are now done.
n %= loopSize
- off = 0
// Copy any fractional portion.
- for n >= 8 {
- // MOVV off(srcReg), tmpReg
- // MOVV tmpReg, off(dstReg)
+ for off = 0; n >= 8; off += 8 {
move8(s, srcReg, dstReg, tmpReg, off)
- off += 8
n -= 8
}
if n != 0 {
- // MOVV off+n-8(srcReg), tmpReg
- // MOVV tmpReg, off+n-8(srcReg)
move8(s, srcReg, dstReg, tmpReg, off+n-8)
}
case ssa.OpLOONG64CALLstatic, ssa.OpLOONG64CALLclosure, ssa.OpLOONG64CALLinter:
s.Call(v)
- case ssa.OpLOONG64CALLtail:
+ case ssa.OpLOONG64CALLtail, ssa.OpLOONG64CALLtailinter:
s.TailCall(v)
case ssa.OpLOONG64LoweredWB:
p := s.Prog(obj.ACALL)
@@ -1333,7 +1387,7 @@ func move8(s *ssagen.State, src, dst, tmp int16, off int64) {
// zero8 zeroes 8 bytes at reg+off.
func zero8(s *ssagen.State, reg int16, off int64) {
- // MOVV ZR, off(reg)
+ // MOVV ZR, off(reg)
p := s.Prog(loong64.AMOVV)
p.From.Type = obj.TYPE_REG
p.From.Reg = loong64.REGZERO
@@ -1341,3 +1395,14 @@ func zero8(s *ssagen.State, reg int16, off int64) {
p.To.Reg = reg
p.To.Offset = off
}
+
+// zero16 zeroes 16 bytes at reg+off.
+func zero16(s *ssagen.State, regZero, regBase int16, off int64) {
+ // VMOVQ regZero, off(regBase)
+ p := s.Prog(loong64.AVMOVQ)
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = regZero
+ p.To.Type = obj.TYPE_MEM
+ p.To.Reg = regBase
+ p.To.Offset = off
+}
diff --git a/src/cmd/compile/internal/loopvar/loopvar.go b/src/cmd/compile/internal/loopvar/loopvar.go
index 5a4590d2998d0c..d2da412ed5263d 100644
--- a/src/cmd/compile/internal/loopvar/loopvar.go
+++ b/src/cmd/compile/internal/loopvar/loopvar.go
@@ -448,6 +448,11 @@ func ForCapture(fn *ir.Func) []VarAndLoop {
}
}
ir.WithFunc(fn, forCapture)
+
+ if ir.MatchAstDump(fn, "loopvar") {
+ ir.AstDump(fn, "loopvar, "+ir.FuncName(fn))
+ }
+
return transformed
}
@@ -557,7 +562,7 @@ func LogTransformations(transformed []VarAndLoop) {
if logopt.Enabled() {
// For automated checking of coverage of this transformation, include this in the JSON information.
- var nString interface{} = n
+ var nString any = n
if inner != outer {
nString = fmt.Sprintf("%v (from inline)", n)
}
diff --git a/src/cmd/compile/internal/loopvar/loopvar_test.go b/src/cmd/compile/internal/loopvar/loopvar_test.go
index 0265dd06eec788..c4ff8f620dc5e6 100644
--- a/src/cmd/compile/internal/loopvar/loopvar_test.go
+++ b/src/cmd/compile/internal/loopvar/loopvar_test.go
@@ -6,6 +6,7 @@ package loopvar_test
import (
"internal/testenv"
+ "os"
"os/exec"
"path/filepath"
"regexp"
@@ -381,3 +382,62 @@ func TestLoopVarVersionDisableGoBuild(t *testing.T) {
t.Errorf("err=%v == nil", err)
}
}
+
+// TestLoopVarLineDirective tests that loopvar version detection works correctly
+// with line directives. This is a regression test for a bug where FileBase() was
+// used instead of Base(), causing incorrect version lookup when line directives
+// were present.
+func TestLoopVarLineDirective(t *testing.T) {
+ switch runtime.GOOS {
+ case "linux", "darwin":
+ default:
+ t.Skipf("Slow test, usually avoid it, os=%s not linux or darwin", runtime.GOOS)
+ }
+ switch runtime.GOARCH {
+ case "amd64", "arm64":
+ default:
+ t.Skipf("Slow test, usually avoid it, arch=%s not amd64 or arm64", runtime.GOARCH)
+ }
+
+ testenv.MustHaveGoBuild(t)
+ gocmd := testenv.GoToolPath(t)
+ tmpdir := t.TempDir()
+ output := filepath.Join(tmpdir, "foo.exe")
+
+ // Create a go.mod file with Go 1.21 to test compatibility behavior.
+ // When building with a higher Go compiler, the loopvar should be created per-loop.
+ gomodPath := filepath.Join(tmpdir, "go.mod")
+ if err := os.WriteFile(gomodPath, []byte("module test\n\ngo 1.21\n"), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ // Copy the test file (with line directive) to the temporary module
+ testFile := "range_esc_closure_linedir.go"
+ srcPath := filepath.Join("testdata", testFile)
+ dstPath := filepath.Join(tmpdir, testFile)
+ src, err := os.ReadFile(srcPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(dstPath, src, 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ // Build the module (not as a single file, so go.mod is respected)
+ cmd := testenv.Command(t, gocmd, "build", "-o", output, ".")
+ cmd.Dir = tmpdir
+ b, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Logf("build output: %s", b)
+ t.Fatal(err)
+ }
+ t.Logf("build output: %s", b)
+
+ cmd = testenv.Command(t, output)
+ b, err = cmd.CombinedOutput()
+ t.Logf("run output: %s", b)
+
+ if err != nil {
+ t.Errorf("expected success (exit code 0), got: %v", err)
+ }
+}
diff --git a/src/cmd/compile/internal/loopvar/testdata/range_esc_closure_linedir.go b/src/cmd/compile/internal/loopvar/testdata/range_esc_closure_linedir.go
new file mode 100644
index 00000000000000..24823e8990f29f
--- /dev/null
+++ b/src/cmd/compile/internal/loopvar/testdata/range_esc_closure_linedir.go
@@ -0,0 +1,24 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//line range_esc_closure_linedir.go:5
+package main
+
+import "fmt"
+
+var is []func() int
+
+func main() {
+ var ints = []int{0, 0, 0}
+ for i := range ints {
+ is = append(is, func() int { return i })
+ }
+
+ for _, f := range is {
+ fmt.Println(f())
+ if f() != 2 {
+ panic("loop variable i: expected shared per-loop, but got distinct per-iteration")
+ }
+ }
+}
diff --git a/src/cmd/compile/internal/midway/analysis.go b/src/cmd/compile/internal/midway/analysis.go
new file mode 100644
index 00000000000000..ab7482161d6895
--- /dev/null
+++ b/src/cmd/compile/internal/midway/analysis.go
@@ -0,0 +1,296 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package midway
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/syntax"
+ "cmd/compile/internal/types2"
+)
+
+// Analyzer holds the state for SIMD dependency analysis
+type Analyzer struct {
+ pkg *types2.Package
+ info *types2.Info
+ isDependentObj map[types2.Object]bool // does an Object depend on a simd type in some way?
+ isDependentMethod map[types2.Object]bool // is this dependent Object also a method? (methods are not renamed, their types are)
+ hasDependentMethod map[types2.Type]bool // is this a type that has a dependent method?
+ visited map[types2.Type]bool // if in map, type has been visited, and value is whether type is dependent
+ inSimd bool // true if the current package is the simd package (which is a special case)
+}
+
+func NewAnalyzer(pkg *types2.Package, info *types2.Info) *Analyzer {
+ return &Analyzer{
+ pkg: pkg,
+ info: info,
+ isDependentObj: make(map[types2.Object]bool),
+ isDependentMethod: make(map[types2.Object]bool),
+ hasDependentMethod: make(map[types2.Type]bool),
+ visited: make(map[types2.Type]bool),
+ inSimd: pkg.Path() == simdPkg,
+ }
+}
+
+// Analyze builds the set of SIMD-dependent objects
+func (a *Analyzer) Analyze(files []*syntax.File) bool {
+ // Phase 1: Seed dependence from types and signatures
+ hdmsize := len(a.hasDependentMethod)
+
+ for {
+ for _, obj := range a.info.Defs {
+ if obj != nil {
+ a.markIfDependent(obj)
+ }
+ }
+ for _, obj := range a.info.Uses {
+ if obj != nil {
+ a.markIfDependent(obj)
+ }
+ }
+ if hdmsize == len(a.hasDependentMethod) {
+ break
+ }
+ if base.Debug.Simd > 0 {
+ base.Warn("hasDependentMethod increased from %d to %d", hdmsize, len(a.hasDependentMethod))
+ }
+ hdmsize = len(a.hasDependentMethod)
+ clear(a.visited)
+ }
+
+ // Phase 2: Transitive closure via function bodies
+ changed := true
+ for changed {
+ changed = false
+ for _, file := range files {
+ for _, decl := range file.DeclList {
+ if fn, ok := decl.(*syntax.FuncDecl); ok {
+ if fn.Name == nil {
+ continue
+ }
+ obj := a.info.Defs[fn.Name]
+ if obj == nil || a.isDependentObj[obj] {
+ continue
+ }
+
+ if a.hasBodyDependency(fn) {
+ a.isDependentObj[obj] = true
+ changed = true
+ }
+ }
+ }
+ }
+ }
+
+ return len(a.isDependentObj) > 0
+}
+
+func (a *Analyzer) hasBodyDependency(fn *syntax.FuncDecl) bool {
+ if fn.Body == nil {
+ return false
+ }
+ // Walk the body and check identifiers
+ // This will also note any variable references that are dependent.
+ found := false
+ syntax.Inspect(fn.Body, func(n syntax.Node) bool {
+ if id, ok := n.(*syntax.Name); ok {
+ obj := a.info.Uses[id]
+ if obj == nil {
+ obj = a.info.Defs[id]
+ }
+ if obj != nil {
+ if _, isFunc := obj.(*types2.Func); !isFunc {
+ if a.isDependentObj[obj] {
+ found = true
+ return false
+ }
+ } else {
+ sig := obj.Type().(*types2.Signature)
+ if a.HasDependentSignature(sig) {
+ found = true
+ return false
+ }
+ }
+ if a.isDependentType(obj.Type()) {
+ // Whatever this is, it makes the outer object dependent.
+ // If this is a package variable with dependent type, mark the
+ // variable as dependent, so that references to it become dependent.
+ if obj, ok := obj.(*types2.Var); ok && obj.Kind() == types2.PackageVar {
+ // everything else is nested within a dependent function/struct/scope
+ // and does not need its own renaming
+ a.isDependentObj[obj] = true
+ }
+ found = true
+ return false
+ }
+ if isBaseSimdTypeObj(obj) {
+ found = true
+ return false
+ }
+ }
+ }
+ return true
+ })
+ return found
+}
+
+func (a *Analyzer) markIfDependent(obj types2.Object) bool {
+ if a.isDependentObj[obj] {
+ return true
+ }
+
+ isDep := false
+ isDepMeth := false
+ switch obj := obj.(type) {
+ case *types2.Var:
+ if obj.Pkg() == a.pkg && obj.Parent() == a.pkg.Scope() {
+ isDep = a.isDependentType(obj.Type())
+ }
+ case *types2.TypeName:
+ isDep = a.isDependentType(obj.Type())
+ case *types2.Func:
+ sig := obj.Type().(*types2.Signature)
+ if a.HasDependentSignature(sig) {
+ // NOT dependent if it is a method of one of the base SIMD types.
+ // TODO: what about aliases of base SIMD types?
+ if rcv := sig.Recv(); rcv == nil {
+ isDep = true
+ } else if named, ok := rcv.Type().(*types2.Named); !ok || !isBaseSimdType(named) {
+ isDep = true
+ t := rcv.Type()
+ if !a.isDependentType(t) {
+ a.markHasMethod(t)
+ }
+ isDepMeth = true
+ }
+ }
+ }
+
+ // Also check if obj name is "simd.Type" (base case)
+ if isBaseSimdTypeObj(obj) {
+ isDep = true
+ }
+
+ if isDep {
+ if base.Debug.Simd > 0 {
+ base.Warn("%s: %v is simd-dependent", obj.Pos().String(), obj)
+ }
+ a.isDependentObj[obj] = true
+ }
+ if isDepMeth {
+ if base.Debug.Simd > 0 {
+ base.Warn("%s: %v is simd-dependent method", obj.Pos().String(), obj)
+ }
+ a.isDependentMethod[obj] = true
+ }
+ return isDep
+}
+
+func (a *Analyzer) isDependentType(t types2.Type) bool {
+ return a.checkTypeRecursive(t)
+}
+
+func (a *Analyzer) checkTypeRecursive(t types2.Type) bool {
+ if t == nil {
+ return false
+ }
+ if a.hasDependentMethod[t] {
+ a.visited[t] = true
+ }
+ if b, ok := a.visited[t]; ok {
+ return b // Break cycles
+ }
+ a.visited[t] = false
+
+ memo := func(b bool) bool {
+ a.visited[t] = b
+ return b
+ }
+
+ // Unwrap aliases
+ if named, ok := t.(*types2.Named); ok {
+ if isBaseSimdType(named) {
+ return memo(true)
+ }
+ if a.checkTypeRecursive(named.Underlying()) {
+ return memo(true)
+ }
+ }
+
+ switch t := t.(type) {
+ case *types2.Basic:
+ return false
+ case *types2.Pointer:
+ return memo(a.checkTypeRecursive(t.Elem()))
+ case *types2.Slice:
+ return memo(a.checkTypeRecursive(t.Elem()))
+ case *types2.Array:
+ return memo(a.checkTypeRecursive(t.Elem()))
+ case *types2.Map:
+ return memo(a.checkTypeRecursive(t.Key()) ||
+ a.checkTypeRecursive(t.Elem()))
+ case *types2.Chan:
+ return memo(a.checkTypeRecursive(t.Elem()))
+ case *types2.Struct:
+ for i := 0; i < t.NumFields(); i++ {
+ if a.checkTypeRecursive(t.Field(i).Type()) {
+ return memo(true)
+ }
+ }
+ case *types2.Signature:
+ return memo(a.HasDependentSignature(t))
+ case *types2.Tuple:
+ for i := 0; i < t.Len(); i++ {
+ if a.checkTypeRecursive(t.At(i).Type()) {
+ return memo(true)
+ }
+ }
+ case *types2.Alias:
+ return memo(a.checkTypeRecursive(types2.Unalias(t)))
+ }
+ return false
+}
+
+// This attempts to mark types that are not otherwise dependent
+// as being dependent, if they have a method with a dependent
+// signature.
+func (a *Analyzer) markHasMethod(t types2.Type) {
+ if t == nil {
+ return
+ }
+ if a.hasDependentMethod[t] {
+ return
+ }
+
+ a.hasDependentMethod[t] = true
+
+ switch t := t.(type) {
+ case *types2.Pointer:
+ a.markHasMethod(t.Elem())
+ case *types2.Alias:
+ a.markHasMethod(t.Rhs())
+ }
+ return
+}
+
+func isBaseSimdType(t *types2.Named) bool {
+ return isBaseSimdTypeObj(t.Obj())
+}
+
+func isBaseSimdTypeObj(obj types2.Object) bool {
+ if obj == nil || obj.Pkg() == nil {
+ return false
+ }
+ if obj.Pkg().Path() != simdPkg {
+ return false
+ }
+ return isSimdTypeName(obj.Name())
+}
+
+func (a *Analyzer) HasDependentSignature(sig *types2.Signature) bool {
+ // TODO what about type parameters? Need to invent a test that provokes that case.
+ return a.isDependentType(sig.Params()) ||
+ a.isDependentType(sig.Results()) ||
+ (sig.Recv() != nil && a.isDependentType(sig.Recv().Type()))
+}
diff --git a/src/cmd/compile/internal/midway/check.go b/src/cmd/compile/internal/midway/check.go
new file mode 100644
index 00000000000000..b9809a7c00dd5a
--- /dev/null
+++ b/src/cmd/compile/internal/midway/check.go
@@ -0,0 +1,27 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package midway
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/syntax"
+)
+
+// CheckPositions checks that all nodes in the files have known positions.
+// This converts lack-of-Pos into an early fatal error instead of a later
+// weird downstream error (e.g., in the linker, in debugging information).
+func CheckPositions(files []*syntax.File, phase string) {
+ for _, file := range files {
+ syntax.Inspect(file, func(n syntax.Node) bool {
+ if n == nil {
+ return true
+ }
+ if !n.Pos().IsKnown() {
+ base.Fatalf("Phase %s, Node without known position: %T\n", phase, n)
+ }
+ return true
+ })
+ }
+}
diff --git a/src/cmd/compile/internal/midway/deepcopy.go b/src/cmd/compile/internal/midway/deepcopy.go
new file mode 100644
index 00000000000000..99f6ef0a6f471f
--- /dev/null
+++ b/src/cmd/compile/internal/midway/deepcopy.go
@@ -0,0 +1,627 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package midway
+
+import (
+ "fmt"
+ "strings"
+
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/syntax"
+ "cmd/compile/internal/types2"
+)
+
+// DeepCopier clones syntax nodes and maintains types2.Info mappings.
+type DeepCopier struct {
+ VecLen int
+ info *types2.Info
+ pkg *types2.Package
+ analyzer *Analyzer
+ suffix string
+
+ vars map[*types2.Var]*types2.Var
+}
+
+func NewDeepCopier(pkg *types2.Package, info *types2.Info, vecLen int, analyzer *Analyzer, suffix string) *DeepCopier {
+ return &DeepCopier{
+ VecLen: vecLen,
+ info: info,
+ pkg: pkg,
+ analyzer: analyzer,
+ suffix: suffix,
+ vars: make(map[*types2.Var]*types2.Var),
+ }
+}
+
+func (c *DeepCopier) registerDef(newName *syntax.Name, oldName *syntax.Name) {
+ if oldName == nil || newName == nil {
+ return
+ }
+ if oldObj := c.info.Defs[oldName]; oldObj != nil {
+ if val, isVar := oldObj.(*types2.Var); isVar {
+ newObj := types2.NewVar(newName.Pos(), c.pkg, newName.Value, val.Type())
+ c.vars[val] = newObj
+ c.info.Defs[newName] = newObj
+ } else {
+ c.info.Defs[newName] = oldObj
+ }
+ }
+}
+
+func (c *DeepCopier) mapUse(newName *syntax.Name, oldName *syntax.Name) {
+ if oldName == nil || newName == nil {
+ return
+ }
+ if oldObj := c.info.Uses[oldName]; oldObj != nil {
+ if val, isVar := oldObj.(*types2.Var); isVar && c.vars[val] != nil {
+ c.info.Uses[newName] = c.vars[val]
+ } else {
+ c.info.Uses[newName] = oldObj
+ }
+ }
+}
+
+// OnName rewrites "dependent" and SIMD names to their architecture-specific version.
+func (c *DeepCopier) OnName(id *syntax.Name) *syntax.Name {
+ obj := c.info.Uses[id]
+ if obj == nil {
+ obj = c.info.Defs[id]
+ }
+ if obj == nil {
+ return nil
+ }
+ // Don't rename methods of dependent types
+ if c.analyzer.isDependentMethod[obj] {
+ return nil
+ }
+
+ if c.analyzer.isDependentObj[obj] || isBaseSimdTypeObj(obj) {
+ newId := syntax.NewName(id.Pos(), id.Value+c.suffix)
+ // Object link will be handled manually in deepcopier Use/Def mapper
+ if base.Debug.Simd > 0 {
+ base.Warn("%s: rewriting name %s to %s", id.Pos().String(), id.Value, newId.Value)
+ }
+ return newId
+ }
+ return nil
+}
+
+// OnNameExpr rewrites references to simd. into
+// ..
+func (c *DeepCopier) OnNameExpr(id *syntax.Name) syntax.Expr {
+ obj := c.info.Uses[id]
+ if obj == nil {
+ obj = c.info.Defs[id]
+ }
+ if obj == nil {
+ return nil
+ }
+
+ if isBaseSimdTypeObj(obj) {
+ // if it is a name, that means that this is in the simd package,
+ // and the name must be replaced with a selector referencing
+ // the architecture-dependent packages.
+ name := id.Value
+ width := nameToElemBitWidth(name)
+ if width > 0 {
+ archsimdId := syntax.NewName(id.Pos(), archPkg)
+ if c.VecLen == 0 {
+ // special case for emulation
+ newSel := &syntax.SelectorExpr{
+ X: archsimdId,
+ Sel: id, // name is unchanged for emulation
+ }
+ newSel.SetPos(id.Pos())
+ return newSel
+ }
+
+ count := c.VecLen / width
+ base := name[:len(name)-1]
+ newName := fmt.Sprintf("%sx%d", base, count)
+ newSelId := syntax.NewName(id.Pos(), newName)
+ newSel := &syntax.SelectorExpr{
+ X: archsimdId,
+ Sel: newSelId,
+ }
+ newSel.SetPos(id.Pos())
+ return newSel
+ }
+ }
+
+ if c.analyzer.isDependentObj[obj] {
+ newId := syntax.NewName(id.Pos(), id.Value+c.suffix)
+ // Object link will be handled manually in deepcopier Use/Def mapper
+ if base.Debug.Simd > 0 {
+ base.Warn("%s: rewriting name %s to %s", id.Pos().String(), id.Value, newId.Value)
+ }
+ return newId
+ }
+ return nil
+}
+
+// OnSelector is looking for simd.Something, to be rewritten into
+// appropriately. Note that this will not work properly within the simd
+// package because there is no "simd." selection there.
+func (c *DeepCopier) OnSelector(se *syntax.SelectorExpr) syntax.Expr {
+ if x, ok := se.X.(*syntax.Name); ok {
+ obj := c.info.Uses[x]
+ if pkgName, isPkg := obj.(*types2.PkgName); isPkg && pkgName.Imported().Path() == simdPkg {
+ // This first little bit detects name = Load-Type-Width-s-{,Part}
+ // and converts the name to Type-Width-s (for nameToWidth), sets isLoad,
+ // and initializes the suffix appropriately.
+ prefix := ""
+ nameSuffix := ""
+ name := se.Sel.Value
+ end := len(name)
+ if strings.HasPrefix(name, "Load") {
+ prefix = "Load"
+ if strings.HasSuffix(name, "Part") {
+ end = strings.Index(name, "Part")
+ nameSuffix = "Part"
+ }
+ name = name[len("Load"):end]
+ }
+ if strings.HasPrefix(name, "Broadcast") {
+ prefix = "Broadcast"
+ name = name[len("Broadcast"):end]
+ }
+
+ width := nameToElemBitWidth(name)
+ if width > 0 {
+ archsimdId := syntax.NewName(se.Pos(), archPkg)
+ if c.VecLen == 0 {
+ // emulated instead, name is unchanged
+ newSel := &syntax.SelectorExpr{
+ X: archsimdId,
+ Sel: se.Sel,
+ }
+ newSel.SetPos(se.Pos())
+ return newSel
+ }
+
+ count := c.VecLen / width
+ base := name[:len(name)-1]
+ newName := fmt.Sprintf("%sx%d", base, count)
+ newName = prefix + newName + nameSuffix
+
+ newSelId := syntax.NewName(se.Sel.Pos(), newName)
+
+ newSel := &syntax.SelectorExpr{
+ X: archsimdId,
+ Sel: newSelId,
+ }
+ newSel.SetPos(se.Pos())
+ return newSel
+ }
+ }
+ }
+ return nil
+}
+
+func (c *DeepCopier) CopyDecl(d syntax.Decl) syntax.Decl {
+ if d == nil {
+ return nil
+ }
+ switch d := d.(type) {
+ case *syntax.FuncDecl:
+ return c.CopyFuncDecl(d)
+ case *syntax.VarDecl:
+ return c.CopyVarDecl(d)
+ case *syntax.TypeDecl:
+ return c.CopyTypeDecl(d)
+ case *syntax.ConstDecl:
+ return c.CopyConstDecl(d)
+ case *syntax.ImportDecl:
+ newD := &syntax.ImportDecl{
+ Group: d.Group,
+ Pragma: d.Pragma,
+ LocalPkgName: c.CopyName(d.LocalPkgName, false),
+ Path: c.CopyExpr(d.Path).(*syntax.BasicLit),
+ }
+ newD.SetPos(d.Pos())
+ return newD
+ default:
+ return d
+ }
+}
+
+func (c *DeepCopier) CopyVarDecl(d *syntax.VarDecl) *syntax.VarDecl {
+ newD := &syntax.VarDecl{
+ Group: d.Group,
+ Pragma: d.Pragma,
+ Type: c.CopyExpr(d.Type),
+ Values: c.CopyExpr(d.Values),
+ }
+ newD.SetPos(d.Pos())
+ for _, n := range d.NameList {
+ newN := c.CopyName(n, true)
+ newD.NameList = append(newD.NameList, newN)
+ }
+ return newD
+}
+
+func (c *DeepCopier) CopyTypeDecl(d *syntax.TypeDecl) *syntax.TypeDecl {
+ newD := &syntax.TypeDecl{
+ Group: d.Group,
+ Pragma: d.Pragma,
+ Name: c.CopyName(d.Name, true),
+ TParamList: c.CopyFieldList(d.TParamList),
+ Alias: d.Alias,
+ Type: c.CopyExpr(d.Type),
+ }
+ newD.SetPos(d.Pos())
+ return newD
+}
+
+func (c *DeepCopier) CopyConstDecl(d *syntax.ConstDecl) *syntax.ConstDecl {
+ newD := &syntax.ConstDecl{
+ Group: d.Group,
+ Pragma: d.Pragma,
+ Type: c.CopyExpr(d.Type),
+ Values: c.CopyExpr(d.Values),
+ }
+ newD.SetPos(d.Pos())
+ for _, n := range d.NameList {
+ newD.NameList = append(newD.NameList, c.CopyName(n, true))
+ }
+ return newD
+}
+
+func (c *DeepCopier) CopyFuncDecl(d *syntax.FuncDecl) *syntax.FuncDecl {
+ newD := &syntax.FuncDecl{
+ Pragma: d.Pragma,
+ Recv: c.CopyField(d.Recv),
+ Name: c.CopyName(d.Name, true),
+ TParamList: c.CopyFieldList(d.TParamList),
+ Type: c.CopyExpr(d.Type).(*syntax.FuncType),
+ }
+ newD.SetPos(d.Pos())
+
+ // Create and register new types2.Func
+ if oldFuncObj, ok := c.info.Defs[d.Name].(*types2.Func); ok {
+ newFuncObj := types2.NewFunc(newD.Name.Pos(), c.pkg, newD.Name.Value, oldFuncObj.Type().(*types2.Signature))
+ c.info.Defs[newD.Name] = newFuncObj
+ }
+
+ newD.Body = c.CopyBlockStmt(d.Body)
+ return newD
+}
+
+func (c *DeepCopier) CopyName(id *syntax.Name, isDef bool) *syntax.Name {
+ if id == nil {
+ return nil
+ }
+ if match := c.OnName(id); match != nil {
+ match.SetPos(id.Pos())
+ if isDef {
+ c.registerDef(match, id)
+ } else {
+ c.mapUse(match, id)
+ }
+ return match
+ }
+ newId := syntax.NewName(id.Pos(), id.Value)
+ if isDef {
+ c.registerDef(newId, id)
+ } else {
+ c.mapUse(newId, id)
+ }
+ return newId
+}
+
+func (c *DeepCopier) CopyNameExpr(id *syntax.Name) syntax.Expr {
+ if !c.analyzer.inSimd {
+ return c.CopyName(id, false)
+ }
+ if id == nil {
+ return nil
+ }
+
+ if match := c.OnNameExpr(id); match != nil {
+ match.SetPos(id.Pos())
+ if n, ok := match.(*syntax.Name); ok {
+ c.mapUse(n, id)
+ }
+ return match
+ }
+
+ newId := syntax.NewName(id.Pos(), id.Value)
+ c.mapUse(newId, id)
+ return newId
+}
+
+func (c *DeepCopier) CopyExpr(e syntax.Expr) syntax.Expr {
+ if e == nil {
+ return nil
+ }
+ var newE syntax.Expr
+ switch e := e.(type) {
+ case *syntax.Name:
+ return c.CopyNameExpr(e)
+ case *syntax.BasicLit:
+ newLit := &syntax.BasicLit{Value: e.Value, Kind: e.Kind, Bad: e.Bad}
+ newE = newLit
+ case *syntax.CompositeLit:
+ newLit := &syntax.CompositeLit{
+ Type: c.CopyExpr(e.Type),
+ NKeys: e.NKeys,
+ Rbrace: e.Rbrace,
+ }
+ for _, el := range e.ElemList {
+ newLit.ElemList = append(newLit.ElemList, c.CopyExpr(el))
+ }
+ newE = newLit
+ case *syntax.KeyValueExpr:
+ newE = &syntax.KeyValueExpr{Key: c.CopyExpr(e.Key), Value: c.CopyExpr(e.Value)}
+ case *syntax.FuncLit:
+ newE = &syntax.FuncLit{Type: c.CopyExpr(e.Type).(*syntax.FuncType), Body: c.CopyBlockStmt(e.Body)}
+ case *syntax.ParenExpr:
+ newE = &syntax.ParenExpr{X: c.CopyExpr(e.X)}
+ case *syntax.SelectorExpr:
+ if sub := c.OnSelector(e); sub != nil {
+ sub.SetPos(e.Pos())
+ if sel := c.info.Selections[e]; sel != nil {
+ c.info.Selections[sub.(*syntax.SelectorExpr)] = sel
+ }
+ return sub
+ }
+ newSel := &syntax.SelectorExpr{X: c.CopyExpr(e.X), Sel: c.CopyName(e.Sel, false)}
+ if sel := c.info.Selections[e]; sel != nil {
+ c.info.Selections[newSel] = sel
+ }
+ newE = newSel
+ case *syntax.IndexExpr:
+ newE = &syntax.IndexExpr{X: c.CopyExpr(e.X), Index: c.CopyExpr(e.Index)}
+ case *syntax.SliceExpr:
+ newE = &syntax.SliceExpr{
+ X: c.CopyExpr(e.X),
+ Index: [3]syntax.Expr{c.CopyExpr(e.Index[0]), c.CopyExpr(e.Index[1]), c.CopyExpr(e.Index[2])},
+ Full: e.Full,
+ }
+ case *syntax.AssertExpr:
+ newE = &syntax.AssertExpr{X: c.CopyExpr(e.X), Type: c.CopyExpr(e.Type)}
+ case *syntax.TypeSwitchGuard:
+ newE = &syntax.TypeSwitchGuard{Lhs: c.CopyName(e.Lhs, true), X: c.CopyExpr(e.X)}
+ case *syntax.Operation:
+ newE = &syntax.Operation{Op: e.Op, X: c.CopyExpr(e.X), Y: c.CopyExpr(e.Y)}
+ case *syntax.CallExpr:
+ newCall := &syntax.CallExpr{
+ Fun: c.CopyExpr(e.Fun),
+ HasDots: e.HasDots,
+ }
+ for _, a := range e.ArgList {
+ newCall.ArgList = append(newCall.ArgList, c.CopyExpr(a))
+ }
+ newE = newCall
+ case *syntax.ListExpr:
+ newList := &syntax.ListExpr{}
+ for _, el := range e.ElemList {
+ newList.ElemList = append(newList.ElemList, c.CopyExpr(el))
+ }
+ newE = newList
+ case *syntax.ArrayType:
+ newE = &syntax.ArrayType{Len: c.CopyExpr(e.Len), Elem: c.CopyExpr(e.Elem)}
+ case *syntax.SliceType:
+ newE = &syntax.SliceType{Elem: c.CopyExpr(e.Elem)}
+ case *syntax.DotsType:
+ newE = &syntax.DotsType{Elem: c.CopyExpr(e.Elem)}
+ case *syntax.StructType:
+ newE = &syntax.StructType{
+ FieldList: c.CopyFieldList(e.FieldList),
+ TagList: e.TagList, // Shallow copy for tags is fine usually
+ }
+ case *syntax.InterfaceType:
+ newE = &syntax.InterfaceType{MethodList: c.CopyFieldList(e.MethodList)}
+ case *syntax.FuncType:
+ newE = &syntax.FuncType{
+ ParamList: c.CopyFieldList(e.ParamList),
+ ResultList: c.CopyFieldList(e.ResultList),
+ }
+ case *syntax.MapType:
+ newE = &syntax.MapType{Key: c.CopyExpr(e.Key), Value: c.CopyExpr(e.Value)}
+ case *syntax.ChanType:
+ newE = &syntax.ChanType{Dir: e.Dir, Elem: c.CopyExpr(e.Elem)}
+ case *syntax.BadExpr:
+ newE = &syntax.BadExpr{}
+ default:
+ newE = e
+ }
+ newE.SetPos(e.Pos())
+ return newE
+}
+
+func (c *DeepCopier) CopyStmt(s syntax.Stmt) syntax.Stmt {
+ if s == nil {
+ return nil
+ }
+ var newS syntax.Stmt
+ switch s := s.(type) {
+ case *syntax.DeclStmt:
+ newDeclList := make([]syntax.Decl, len(s.DeclList))
+ for i, v := range s.DeclList {
+ newDeclList[i] = c.CopyDecl(v)
+ }
+ newS = &syntax.DeclStmt{DeclList: newDeclList}
+ case *syntax.ExprStmt:
+ newS = &syntax.ExprStmt{X: c.CopyExpr(s.X)}
+ case *syntax.SendStmt:
+ newS = &syntax.SendStmt{Chan: c.CopyExpr(s.Chan), Value: c.CopyExpr(s.Value)}
+ case *syntax.AssignStmt:
+ newS = &syntax.AssignStmt{Op: s.Op, Lhs: c.CopyExpr(s.Lhs), Rhs: c.CopyExpr(s.Rhs)}
+ case *syntax.ReturnStmt:
+ newS = &syntax.ReturnStmt{Results: c.CopyExpr(s.Results)}
+ case *syntax.BranchStmt:
+ // TODO this is broken
+ newS = &syntax.BranchStmt{Tok: s.Tok, Label: c.CopyName(s.Label, false), Target: nil} // Targets need fix-up
+ case *syntax.CallStmt:
+ newS = &syntax.CallStmt{Tok: s.Tok, Call: c.CopyExpr(s.Call), DeferAt: c.CopyExpr(s.DeferAt)}
+ case *syntax.IfStmt:
+ newS = &syntax.IfStmt{
+ Init: c.CopySimpleStmt(s.Init),
+ Cond: c.CopyExpr(s.Cond),
+ Then: c.CopyBlockStmt(s.Then),
+ Else: c.CopyStmt(s.Else),
+ }
+ case *syntax.ForStmt:
+ newS = &syntax.ForStmt{
+ Init: c.CopySimpleStmt(s.Init),
+ Cond: c.CopyExpr(s.Cond),
+ Post: c.CopySimpleStmt(s.Post),
+ Body: c.CopyBlockStmt(s.Body),
+ }
+ case *syntax.SwitchStmt:
+ newS = &syntax.SwitchStmt{
+ Init: c.CopySimpleStmt(s.Init),
+ Tag: c.CopyExpr(s.Tag),
+ Body: c.CopyCaseClauses(s.Body),
+ Rbrace: s.Rbrace,
+ }
+ case *syntax.SelectStmt:
+ newS = &syntax.SelectStmt{
+ Body: c.CopyCommClauses(s.Body),
+ Rbrace: s.Rbrace,
+ }
+ case *syntax.EmptyStmt:
+ newS = &syntax.EmptyStmt{}
+ case *syntax.LabeledStmt:
+ newS = &syntax.LabeledStmt{Label: c.CopyName(s.Label, true), Stmt: c.CopyStmt(s.Stmt)} // Labels are defs
+ case *syntax.BlockStmt:
+ return c.CopyBlockStmt(s)
+ default:
+ newS = s
+ }
+ newS.SetPos(s.Pos())
+ return newS
+}
+
+func (c *DeepCopier) CopySimpleStmt(s syntax.SimpleStmt) syntax.SimpleStmt {
+ if s == nil {
+ return nil
+ }
+ switch s := s.(type) {
+ case *syntax.RangeClause:
+ newS := &syntax.RangeClause{
+ Def: s.Def,
+ X: c.CopyExpr(s.X),
+ }
+ // In a range clause, Lhs may contain definitions if Def is true.
+ if list, ok := s.Lhs.(*syntax.ListExpr); ok && s.Def {
+ newList := &syntax.ListExpr{}
+ for _, el := range list.ElemList {
+ if id, ok := el.(*syntax.Name); ok {
+ newList.ElemList = append(newList.ElemList, c.CopyName(id, true))
+ } else {
+ newList.ElemList = append(newList.ElemList, c.CopyExpr(el))
+ }
+ }
+ newS.Lhs = newList
+ } else if id, ok := s.Lhs.(*syntax.Name); ok && s.Def {
+ newS.Lhs = c.CopyName(id, true)
+ } else {
+ newS.Lhs = c.CopyExpr(s.Lhs)
+ }
+ newS.Lhs.SetPos(s.Lhs.Pos())
+ newS.SetPos(s.Pos())
+ return newS
+ case *syntax.AssignStmt:
+ // Check for :=
+ isDef := false
+ if list, ok := s.Lhs.(*syntax.ListExpr); ok {
+ for _, el := range list.ElemList {
+ if id, ok := el.(*syntax.Name); ok && c.info.Defs[id] != nil {
+ isDef = true
+ break
+ }
+ }
+ } else if id, ok := s.Lhs.(*syntax.Name); ok && c.info.Defs[id] != nil {
+ isDef = true
+ }
+
+ newS := &syntax.AssignStmt{Op: s.Op, Rhs: c.CopyExpr(s.Rhs)}
+ if isDef {
+ if list, ok := s.Lhs.(*syntax.ListExpr); ok {
+ newList := &syntax.ListExpr{}
+ for _, el := range list.ElemList {
+ if id, ok := el.(*syntax.Name); ok && c.info.Defs[id] != nil {
+ newList.ElemList = append(newList.ElemList, c.CopyName(id, true))
+ } else {
+ newList.ElemList = append(newList.ElemList, c.CopyExpr(el))
+ }
+ }
+ newS.Lhs = newList
+ } else if id, ok := s.Lhs.(*syntax.Name); ok {
+ newS.Lhs = c.CopyName(id, true)
+ }
+ } else {
+ newS.Lhs = c.CopyExpr(s.Lhs)
+ }
+ newS.Lhs.SetPos(s.Lhs.Pos())
+ newS.SetPos(s.Pos())
+ return newS
+ default:
+ return c.CopyStmt(s).(syntax.SimpleStmt)
+ }
+}
+
+func (c *DeepCopier) CopyCaseClauses(list []*syntax.CaseClause) []*syntax.CaseClause {
+ var newList []*syntax.CaseClause
+ for _, cc := range list {
+ newC := &syntax.CaseClause{Cases: c.CopyExpr(cc.Cases), Colon: cc.Colon}
+ for _, b := range cc.Body {
+ newC.Body = append(newC.Body, c.CopyStmt(b))
+ }
+ newC.SetPos(cc.Pos())
+ newList = append(newList, newC)
+ }
+ return newList
+}
+
+func (c *DeepCopier) CopyCommClauses(list []*syntax.CommClause) []*syntax.CommClause {
+ var newList []*syntax.CommClause
+ for _, cc := range list {
+ newC := &syntax.CommClause{Comm: c.CopySimpleStmt(cc.Comm), Colon: cc.Colon}
+ for _, b := range cc.Body {
+ newC.Body = append(newC.Body, c.CopyStmt(b))
+ }
+ newC.SetPos(cc.Pos())
+ newList = append(newList, newC)
+ }
+ return newList
+}
+
+func (c *DeepCopier) CopyBlockStmt(b *syntax.BlockStmt) *syntax.BlockStmt {
+ if b == nil {
+ return nil
+ }
+ newB := &syntax.BlockStmt{Rbrace: b.Rbrace}
+ for _, s := range b.List {
+ newB.List = append(newB.List, c.CopyStmt(s))
+ }
+ newB.SetPos(b.Pos())
+ return newB
+}
+
+func (c *DeepCopier) CopyFieldList(f []*syntax.Field) []*syntax.Field {
+ if f == nil {
+ return nil
+ }
+ var newF []*syntax.Field
+ for _, field := range f {
+ newF = append(newF, c.CopyField(field))
+ }
+ return newF
+}
+
+func (c *DeepCopier) CopyField(f *syntax.Field) *syntax.Field {
+ if f == nil {
+ return nil
+ }
+ newF := &syntax.Field{
+ Name: c.CopyName(f.Name, true),
+ Type: c.CopyExpr(f.Type),
+ }
+ newF.SetPos(f.Pos())
+ return newF
+}
diff --git a/src/cmd/compile/internal/midway/midway.go b/src/cmd/compile/internal/midway/midway.go
new file mode 100644
index 00000000000000..9adeec8024530c
--- /dev/null
+++ b/src/cmd/compile/internal/midway/midway.go
@@ -0,0 +1,38 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package midway
+
+import (
+ "internal/buildcfg"
+)
+
+func rewriteSizes() []int {
+ switch buildcfg.GOARCH {
+ case "wasm":
+ return []int{0, 128}
+ case "amd64":
+ return []int{0, 128, 256, 512}
+ case "arm64":
+ return []int{0, 128} // this will change for SVE and cannot just be a size-based choice.
+ }
+ return nil
+}
+
+const simdPkg = "simd"
+const archFullPkg = "simd/internal/bridge"
+const archPkg = "bridge"
+const vectorSizeFn = "VectorBitSize"
+const emulatedFn = "Emulated"
+
+func isSimdTypeName(s string) bool {
+ switch s {
+ case "Int8s", "Int16s", "Int32s", "Int64s",
+ "Uint8s", "Uint16s", "Uint32s", "Uint64s",
+ "Mask8s", "Mask16s", "Mask32s", "Mask64s",
+ "Float32s", "Float64s":
+ return true
+ }
+ return false
+}
diff --git a/src/cmd/compile/internal/midway/rewrite.go b/src/cmd/compile/internal/midway/rewrite.go
new file mode 100644
index 00000000000000..e80930bcd79346
--- /dev/null
+++ b/src/cmd/compile/internal/midway/rewrite.go
@@ -0,0 +1,545 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package midway
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/syntax"
+ "cmd/compile/internal/types2"
+ "fmt"
+ "internal/buildcfg"
+ "strings"
+)
+
+// "Midway" rewriting
+//
+// Go attempts to provide a package similar to the the "Highway" library
+// for C++ (https://google.github.io/highway). The library package is "simd"
+// and defines vector types with unspecified widths that are bound to particular
+// machine dependent types as late as program execution. This is accomplished
+// by rewriting code that depends on these types into code that references
+// architecture-specific types, perhaps more than once, and if necessary
+// dynamically choosing which version to execute based on hardware attributes.
+//
+// The rewriting takes place early in the compiler, after type checking but
+// before conversion to "unified" IR. To ensure that types are correctly set
+// on the modified version of the code, type checking information is reset and
+// the type checking phase is re-run. The places some limits on the shape of
+// the rewrites, but it also ensures that the rewritten code is well-formed.
+//
+// Rewritten code does not reference "archsimd" types directly, but instead
+// references types in a "bridge" package that filters the available methods
+// and adds a few more. The package used relies on a builder/compiler hack;
+// the compiler's type checker enforces export naming conventions, but the
+// build system limits visibility to unrelated "internal" packages and can be
+// modified to allow access in special cases (like this one). This allows the
+// rewritten code to reference types, functions, and methods that are not
+// accessible otherwise.
+//
+// The rewrite works in phases. The first is "analysis", to discover functions,
+// types, methods, and variables that depend on "simd" types. "Depend on" means
+// any mention of a simd type, and for types, also includes types that have a
+// simd-dependent method. Dependent functions are split into two categories;
+// those whose dependence includes their signature, and those that do not.
+// The second category forms the boundary between code that depends on simd and
+// code that does not. Notice that there cannot be a boundary method, because
+// (by design) the receiver type is simd-dependent and thus a dependent method
+// also has a dependent type in its signature.
+//
+// The second phase rewrites such "boundary" functions into a "dispatch" version
+// and (later, third phase) "specialized" versions. The dispatch function
+// will choose which specialized version to call based on which simd implementation
+// has been chosen, and forward parameters and results to/from that specialized version
+// of the function. The dispatch version shares the same name as the original function.
+// Note that this applies to functions only, and not methods.
+
+// The third phase specializes dependent functions (both kinds), methods,
+// global variables, and types into size/emulation/feature-specific variants.
+// Except for methods, this is done by adding a suffix beginning with "@" to
+// the name. Because "@" cannot appear in legal Go identifiers this removes
+// the risk of a naming overlap. Methods are specialized, but not renamed,
+// because their receiver type is renamed instead. Not changing method names
+// preserves interface satisfaction, for example in the case of generic interfaces.
+//
+// Non-boundary dependent function and methods are not rewritten into dispatch
+// functions/methods, but remain in the generated code because they must be
+// present in the export data so that other packages that import them will still
+// compile before rewriting. Their bodies are replaced with panic(...) to allow
+// compilation while preventing even worse chaos in the event of a bug either in
+// the compiler or through ambitious use of reflection or assembly language.
+//
+
+/* Example rewrites
+
+// Type alias, global variable, and init function:
+
+// before:
+type MyInt8s = simd.Int8s
+func Generic[T haslen](x int) int {
+ var v T
+ return x + v.Len()
+}
+var VL int
+func init() {
+ VL = Generic[MyInt8s](1)
+}
+// dispatch:
+func init() {
+ switch simd.VectorBitSize() {
+ case
+ 128:
+ init@simd128()
+ return
+ case 256:
+ init@simd256()
+ return
+ case 512:
+ init@simd512()
+ return
+ default:
+ panic("unsupported vector size")
+ }
+}
+// specialized (128)
+type MyInt8s@simd128 = archsimd.Int8x16
+func init@simd128() {
+ VL = Generic[MyInt8s@simd128](1)
+}
+
+
+// structure containing simd fields, and with simd methods
+
+// before
+// A struct dependent on SIMD
+type VectorC struct {
+ Field simd.Float32s
+}
+func (v *VectorC) MethodOfSimd() bool {
+ return false
+}
+func (v VectorC) Data() simd.Float32s {
+ return v.Field
+}
+func (v VectorC) Foo(x VectorC) VectorC {
+ return VectorC{Field: v.Field.Add(x.Field)}
+}
+
+// dispatch
+// technically there is none, but functions with panicking bodies
+// remain because code must pass type checking before rewriting.
+type VectorC struct {
+ Field simd.Float32s
+}
+func (v *VectorC) MethodOfSimd() bool {
+ panic(...)
+}
+func (v VectorC) Data() simd.Float32s {
+ panic(...)
+}
+func (v VectorC) Foo(x VectorC) VectorC {
+ panic(...)
+}
+
+// specialized (128)
+
+// A struct dependent on SIMD
+type VectorC@simd128 struct {
+ Field bridge.Float32x4
+}
+func (v *VectorC@simd128) MethodOfSimd() bool {
+ return false
+}
+func (v VectorC@simd128) Data() bridge.Float32x4 {
+ return v.Field
+}
+func (v VectorC@simd128) Foo(x VectorC@simd128) VectorC@simd128 {
+ return VectorC@simd128{Field: v.Field.Add(x.Field)}
+}
+
+*/
+
+type Rewriter struct {
+ pkg *types2.Package
+ analyzer *Analyzer
+ info *types2.Info
+ sizes []int
+}
+
+func NewRewriter(pkg *types2.Package, info *types2.Info, analyzer *Analyzer, sizes []int) *Rewriter {
+ return &Rewriter{
+ pkg: pkg,
+ info: info,
+ analyzer: analyzer,
+ sizes: sizes,
+ }
+}
+
+func (r *Rewriter) Rewrite(files []*syntax.File) {
+
+ // First duplicate and specialize all dependent functions and variables.
+ for _, fileAST := range files {
+
+ var newDecls []syntax.Decl
+ for _, k := range r.sizes {
+ newDecls = r.generateForSize(fileAST, k, newDecls)
+ }
+
+ // Then replace original functions with dispatchers.
+ // This also edits the DeclList of fileAST.
+ r.generateDispatchers(fileAST)
+
+ fileAST.DeclList = append(fileAST.DeclList, newDecls...)
+ }
+}
+
+func (r *Rewriter) generateDispatchers(fileAST *syntax.File) {
+ var newDecls []syntax.Decl
+
+ change := false
+
+ for _, decl := range fileAST.DeclList {
+ switch d := decl.(type) {
+ case *syntax.FuncDecl:
+ if d.Name == nil {
+ newDecls = append(newDecls, d)
+ continue
+ }
+ obj := r.info.Defs[d.Name]
+ if !r.analyzer.isDependentObj[obj] || r.analyzer.inSimd {
+ newDecls = append(newDecls, d)
+ continue
+ }
+
+ sig, ok := obj.Type().(*types2.Signature)
+ if !ok {
+ newDecls = append(newDecls, d)
+ continue
+ }
+
+ change = true
+ if r.analyzer.HasDependentSignature(sig) {
+ if base.Debug.Simd > 0 {
+ base.Warn("%s: removing body of dependent-sig original function %v", d.Pos().String(), d.Name.Value)
+ }
+ d.Body = r.blockOf(d.Pos(), r.panicStmt(d.Pos(),
+ "unexpected call of original function rewritten to specialized SIMD"))
+ newDecls = append(newDecls, d)
+ continue
+ }
+
+ // Clean signature -> Replace body with dispatcher
+ d.Body = r.createDispatcherBody(d, sig)
+ newDecls = append(newDecls, d)
+
+ case *syntax.VarDecl:
+ // Keep var decls even if rewritten, so that pre-rewrite code parses correctly.
+ // TODO figure out how to deal with side-effects in initializers.
+ newDecls = append(newDecls, d)
+
+ case *syntax.TypeDecl:
+ // Keep all types; we need the untranslated copy if a method referencing it
+ // needs to typecheck pre-translation.
+ newDecls = append(newDecls, d)
+ default:
+ newDecls = append(newDecls, decl)
+ }
+ }
+
+ if !change {
+ return
+ }
+
+ fileAST.DeclList = newDecls
+
+ if !r.analyzer.inSimd {
+ // Inject an import to the bridge package (if not exists)
+ hasArchSimd := false
+ var simdImport *syntax.ImportDecl
+ p := fileAST.Pos()
+ for _, decl := range fileAST.DeclList {
+ if imp, ok := decl.(*syntax.ImportDecl); ok {
+ if imp.Path.Value == `"`+archFullPkg+`"` {
+ hasArchSimd = true
+ if simdImport == nil {
+ p = imp.Pos()
+ }
+ }
+ if imp.Path.Value == `"`+simdPkg+`"` {
+ simdImport = imp
+ p = imp.Pos()
+ }
+ }
+ }
+
+ if !hasArchSimd {
+ r.injectImport(fileAST, archFullPkg, p)
+ }
+
+ // Ensure at least one use of "simd"
+ // var _ = simd.VectorBitLen()
+ fun := &syntax.SelectorExpr{
+ X: syntax.NewName(p, simdPkg), // Assume this is resolvable
+ Sel: syntax.NewName(p, vectorSizeFn),
+ }
+ fun.SetPos(p)
+ call := &syntax.CallExpr{Fun: fun}
+ call.SetPos(p)
+
+ name := syntax.NewName(p, "_")
+
+ varDecl := &syntax.VarDecl{NameList: []*syntax.Name{name}, Values: call}
+ varDecl.SetPos(p)
+ fileAST.DeclList = append(fileAST.DeclList, varDecl)
+ }
+}
+
+func (r *Rewriter) injectImport(fileAST *syntax.File, toImport string, simdImportPos syntax.Pos) {
+ importDecl := &syntax.ImportDecl{
+ Path: &syntax.BasicLit{Value: `"` + toImport + `"`, Kind: syntax.StringLit},
+ }
+ importDecl.Path.SetPos(simdImportPos)
+ importDecl.SetPos(simdImportPos)
+ fileAST.DeclList = append([]syntax.Decl{importDecl}, fileAST.DeclList...)
+}
+
+func (r *Rewriter) createDispatcherBody(d *syntax.FuncDecl, sig *types2.Signature) *syntax.BlockStmt {
+
+ // Build call arguments from the function parameters
+ args := func() []syntax.Expr {
+ var args []syntax.Expr
+ if d.Type.ParamList != nil {
+ for _, field := range d.Type.ParamList {
+ if field.Name != nil {
+ paramName := syntax.NewName(field.Pos(), field.Name.Value)
+ args = append(args, paramName)
+ }
+ }
+ }
+ return args
+ }
+
+ // Slap a pos on an expression
+ pe := func(e syntax.Expr) syntax.Expr {
+ e.SetPos(d.Pos())
+ return e
+ }
+ // Slap a pos on a statement
+ ps := func(e syntax.Stmt) syntax.Stmt {
+ e.SetPos(d.Pos())
+ return e
+ }
+
+ // switch ast node.
+ // the goal is something like (for now, till there are finer-grained choices)
+ // switch simd.VectorSize() {
+ // case 128: if simd.Emulated() { call the specialize-for-emulation-code(args) }
+ // else { call the specialize-for-128-code(args) }
+ // case 256: call the specialize-for-256-code(args)
+ // etc
+ // }
+ //
+ // the cases above deal with the usual `return call(...)` vs `call(...); return`
+ switchStmt := &syntax.SwitchStmt{
+ Tag: pe(&syntax.CallExpr{
+ Fun: pe(&syntax.SelectorExpr{
+ X: syntax.NewName(d.Pos(), simdPkg), // Assume this is resolvable
+ Sel: syntax.NewName(d.Pos(), vectorSizeFn),
+ }),
+ }),
+ Body: []*syntax.CaseClause{},
+ }
+
+ var emulation syntax.Stmt
+
+ for _, k := range r.sizes {
+ fnName := fmt.Sprintf("%s@simd%d", d.Name.Value, k)
+ fnIdent := syntax.NewName(d.Pos(), fnName)
+
+ callExpr := pe(&syntax.CallExpr{
+ Fun: pe(fnIdent),
+ ArgList: args(),
+ })
+
+ // callReturnStmt is either `return call(...)` or `call(...); return`
+ var callReturnStmt syntax.Stmt
+ if d.Type.ResultList != nil && len(d.Type.ResultList) > 0 {
+ callReturnStmt = &syntax.ReturnStmt{Results: callExpr}
+ } else {
+ callReturnStmt = &syntax.BlockStmt{
+ List: []syntax.Stmt{
+ ps(&syntax.ExprStmt{X: callExpr}),
+ ps(&syntax.ReturnStmt{}),
+ },
+ Rbrace: d.Pos(),
+ }
+ }
+ callReturnStmt.SetPos(d.Pos())
+
+ if k == 0 {
+ // emulation == `if simd.Emulated() { callReturnStmt }`
+ // save it for the first part of the 128 case.
+ cond := pe(&syntax.CallExpr{
+ Fun: pe(&syntax.SelectorExpr{
+ X: syntax.NewName(d.Pos(), simdPkg), // Assume this is resolvable
+ Sel: syntax.NewName(d.Pos(), emulatedFn),
+ })})
+
+ blockStmt, ok := callReturnStmt.(*syntax.BlockStmt)
+ if !ok {
+ blockStmt = &syntax.BlockStmt{
+ List: []syntax.Stmt{callReturnStmt},
+ Rbrace: d.Pos(),
+ }
+ blockStmt.SetPos(d.Pos())
+ }
+
+ emulation = ps(&syntax.IfStmt{
+ Cond: cond,
+ Then: blockStmt,
+ })
+ continue
+ }
+
+ var caseBody []syntax.Stmt
+ // assume that 128 is a case; when we do scalable simd, this may change.
+ // For now, if there is emulation, it is 128-bit (only).
+ if emulation != nil && k == 128 {
+ caseBody = append(caseBody, emulation)
+ emulation = nil
+ }
+
+ caseClause := &syntax.CaseClause{
+ Cases: pe(&syntax.BasicLit{Kind: syntax.IntLit, Value: fmt.Sprintf("%d", k)}),
+ Body: append(caseBody, callReturnStmt),
+ }
+ caseClause.SetPos(d.Pos())
+ switchStmt.Body = append(switchStmt.Body, caseClause)
+ }
+
+ panicStmt := r.panicStmt(d.Pos(), "unsupported vector size in simd-rewritten code")
+ return r.blockOf(d.Pos(), switchStmt, panicStmt)
+}
+
+func (r *Rewriter) blockOf(p syntax.Pos, stmts ...syntax.Stmt) *syntax.BlockStmt {
+ for _, s := range stmts {
+ s.SetPos(p)
+ }
+ blockStmt := &syntax.BlockStmt{List: stmts}
+ blockStmt.SetPos(p)
+ return blockStmt
+}
+
+func (r *Rewriter) panicStmt(p syntax.Pos, unquotedMessage string) *syntax.ExprStmt {
+ pe := func(e syntax.Expr) syntax.Expr {
+ e.SetPos(p)
+ return e
+ }
+ fnName := "panic"
+ fnIdent := pe(syntax.NewName(p, fnName))
+ callExpr := pe(&syntax.CallExpr{
+ Fun: fnIdent,
+ ArgList: []syntax.Expr{pe(&syntax.BasicLit{Value: `"` + unquotedMessage + `"`, Kind: syntax.StringLit})},
+ })
+ panicStmt := &syntax.ExprStmt{X: callExpr}
+ panicStmt.SetPos(p)
+ return panicStmt
+}
+
+func (r *Rewriter) generateForSize(fileAST *syntax.File, k int, newDecls []syntax.Decl) []syntax.Decl {
+ copier := NewDeepCopier(r.pkg, r.info, k, r.analyzer, fmt.Sprintf("@simd%d", k))
+ for _, decl := range fileAST.DeclList {
+ if r.shouldIncludeDecl(decl) {
+ newDecl := copier.CopyDecl(decl)
+ newDecls = append(newDecls, newDecl)
+ }
+ }
+ return newDecls
+}
+
+func nameToElemBitWidth(name string) int {
+ var width int
+ switch name {
+ case "Int8s", "Uint8s", "Mask8s":
+ width = 8
+ case "Int16s", "Uint16s", "Mask16s":
+ width = 16
+ case "Int32s", "Uint32s", "Float32s", "Mask32s":
+ width = 32
+ case "Int64s", "Uint64s", "Float64s", "Mask64s":
+ width = 64
+ }
+ return width
+}
+
+func (r *Rewriter) shouldIncludeDecl(decl syntax.Decl) bool {
+ // Files (and declarations) in the simd package are excluded
+ // from processing, except for those that whose name begins
+ // with "tofrom_".
+ if r.analyzer.inSimd {
+ theFile := decl.Pos().Base().Filename()
+
+ lastSlash := strings.LastIndex(theFile, simdPkg+"/")
+ lastBackslash := strings.LastIndex(theFile, simdPkg+"\\")
+
+ // Windows paths can be chaos, all we care, is whether the very last part
+ // of the path is any-path-separator + "tofrom_" + anything-else, given that
+ // we already know that we are in the simd package.
+ maxSlash := max(lastSlash, lastBackslash)
+ if maxSlash == -1 {
+ return false
+ }
+ if !strings.HasPrefix(theFile[maxSlash:], simdPkg+"/tofrom_") &&
+ !strings.HasPrefix(theFile[maxSlash:], simdPkg+"\\tofrom_") {
+ return false
+ }
+ }
+
+ switch d := decl.(type) {
+ case *syntax.FuncDecl:
+ if d.Name != nil {
+ return r.analyzer.isDependentObj[r.info.Defs[d.Name]]
+ }
+ case *syntax.TypeDecl:
+ return r.analyzer.isDependentObj[r.info.Defs[d.Name]]
+ case *syntax.VarDecl:
+ for _, name := range d.NameList {
+ if r.analyzer.isDependentObj[r.info.Defs[name]] {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// Generate an API matching the standalone compilation call
+func RewriteWrapper(pkg *types2.Package, info *types2.Info, files []*syntax.File) bool {
+ if !buildcfg.Experiment.SIMD {
+ return false
+ }
+
+ switch buildcfg.GOARCH {
+ case "wasm", "amd64", "arm64":
+ default:
+ return false
+ }
+
+ sizes := rewriteSizes()
+ if len(sizes) == 0 {
+ return false
+ }
+ analyzer := NewAnalyzer(pkg, info)
+ if !analyzer.Analyze(files) {
+ return false
+ }
+
+ CheckPositions(files, "before midway")
+
+ rewriter := NewRewriter(pkg, info, analyzer, sizes)
+ rewriter.Rewrite(files)
+
+ CheckPositions(files, "after midway")
+
+ return true
+}
diff --git a/src/cmd/compile/internal/mips/ssa.go b/src/cmd/compile/internal/mips/ssa.go
index c338fdc6b3e46b..d89d2afcaf3fad 100644
--- a/src/cmd/compile/internal/mips/ssa.go
+++ b/src/cmd/compile/internal/mips/ssa.go
@@ -197,7 +197,6 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpMIPSANDconst,
ssa.OpMIPSORconst,
ssa.OpMIPSXORconst,
- ssa.OpMIPSNORconst,
ssa.OpMIPSSLLconst,
ssa.OpMIPSSRLconst,
ssa.OpMIPSSRAconst,
@@ -213,11 +212,25 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpMIPSMULTU,
ssa.OpMIPSDIV,
ssa.OpMIPSDIVU:
- // result in hi,lo
+ // HI, LO results exist in low quality registers that can't
+ // be stored without using REGTMP.
+ // This used to cause corruptions since store might also need
+ // REGTMP to materialize the address.
+ // Instead we move the result into high quality registers.
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[1].Reg()
p.Reg = v.Args[0].Reg()
+ p1 := s.Prog(mips.AMOVW)
+ p1.From.Type = obj.TYPE_REG
+ p1.From.Reg = mips.REG_HI
+ p1.To.Type = obj.TYPE_REG
+ p1.To.Reg = v.Reg0()
+ p2 := s.Prog(mips.AMOVW)
+ p2.From.Type = obj.TYPE_REG
+ p2.From.Reg = mips.REG_LO
+ p2.To.Type = obj.TYPE_REG
+ p2.To.Reg = v.Reg1()
case ssa.OpMIPSMOVWconst:
r := v.Reg()
p := s.Prog(v.Op.Asm())
@@ -479,7 +492,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p6.To.SetTarget(p2)
case ssa.OpMIPSCALLstatic, ssa.OpMIPSCALLclosure, ssa.OpMIPSCALLinter:
s.Call(v)
- case ssa.OpMIPSCALLtail:
+ case ssa.OpMIPSCALLtail, ssa.OpMIPSCALLtailinter:
s.TailCall(v)
case ssa.OpMIPSLoweredWB:
p := s.Prog(obj.ACALL)
diff --git a/src/cmd/compile/internal/mips64/ssa.go b/src/cmd/compile/internal/mips64/ssa.go
index 6eae8fe0dd50eb..8b096f0014ac7c 100644
--- a/src/cmd/compile/internal/mips64/ssa.go
+++ b/src/cmd/compile/internal/mips64/ssa.go
@@ -191,7 +191,6 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpMIPS64ANDconst,
ssa.OpMIPS64ORconst,
ssa.OpMIPS64XORconst,
- ssa.OpMIPS64NORconst,
ssa.OpMIPS64SLLVconst,
ssa.OpMIPS64SRLVconst,
ssa.OpMIPS64SRAVconst,
@@ -207,11 +206,25 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpMIPS64MULVU,
ssa.OpMIPS64DIVV,
ssa.OpMIPS64DIVVU:
- // result in hi,lo
+ // HI, LO results exist in low quality registers that can't
+ // be stored without using REGTMP.
+ // This used to cause corruptions since store might also need
+ // REGTMP to materialize the address.
+ // Instead we move the result into high quality registers.
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[1].Reg()
p.Reg = v.Args[0].Reg()
+ p1 := s.Prog(mips.AMOVV)
+ p1.From.Type = obj.TYPE_REG
+ p1.From.Reg = mips.REG_HI
+ p1.To.Type = obj.TYPE_REG
+ p1.To.Reg = v.Reg0()
+ p2 := s.Prog(mips.AMOVV)
+ p2.From.Type = obj.TYPE_REG
+ p2.From.Reg = mips.REG_LO
+ p2.To.Type = obj.TYPE_REG
+ p2.To.Reg = v.Reg1()
case ssa.OpMIPS64MOVVconst:
r := v.Reg()
p := s.Prog(v.Op.Asm())
@@ -353,6 +366,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpMIPS64MOVVgpfp,
ssa.OpMIPS64NEGF,
ssa.OpMIPS64NEGD,
+ ssa.OpMIPS64ABSF,
ssa.OpMIPS64ABSD,
ssa.OpMIPS64SQRTF,
ssa.OpMIPS64SQRTD:
@@ -490,7 +504,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p6.To.SetTarget(p2)
case ssa.OpMIPS64CALLstatic, ssa.OpMIPS64CALLclosure, ssa.OpMIPS64CALLinter:
s.Call(v)
- case ssa.OpMIPS64CALLtail:
+ case ssa.OpMIPS64CALLtail, ssa.OpMIPS64CALLtailinter:
s.TailCall(v)
case ssa.OpMIPS64LoweredWB:
p := s.Prog(obj.ACALL)
diff --git a/src/cmd/compile/internal/noder/README.md b/src/cmd/compile/internal/noder/README.md
new file mode 100644
index 00000000000000..eb7a3697ef92da
--- /dev/null
+++ b/src/cmd/compile/internal/noder/README.md
@@ -0,0 +1,83 @@
+# Making Unified Intermediate Representation (UIR) Changes
+
+For general information on export data, see [here](../../README.md).
+
+UIR is the serial form of the compiler's intermediate representation, used to
+propagate bodies of generic and/or inlined functions from one compilation unit
+to another.
+
+The Go compiler has a single, canonical UIR writer implementation in
+`src/cmd/compile/internal/noder/writer.go`. When we update the byte stream that
+the UIR writer writes, *all* of the UIR readers need to be reviewed and
+potentially updated; a change might not be backward compatible for them. These
+instructions outline the steps required to keep all UIR readers up-to-date.
+
+## The Writer
+
+The UIR version written by the compiler is controlled by
+`src/cmd/compile/internal/noder/unified.go`. Do not change this yet. Instead:
+
+1. Add a version flag N+1 for `MyChange` in `internal/pkgbits/version.go`.
+2. Update the UIR writer in `src/cmd/compile/internal/noder/writer.go` to guard
+ the writing of any fields added in N+1. Note: readers still on version N
+ *must* be oblivious to this change to avoid breaking the readers on
+ submission.
+
+## The Readers
+
+Besides the compiler itself, there are other readers in go, x/tools, and
+externally. Those in x/tools and the general public exist because
+`go list -export` produces export data files in this format and we support the
+ability of applications to decode it.
+
+> Note that there is an upcoming plan to decouple the compiler's IR from x/tools
+> by changing the format encoded by `go list -export`; this would make UIR a
+> private detail of the compiler, free to break at any time. For now, these
+> instructions must still be followed.
+
+We assume that external readers will update on their own. The necessary reader
+updates in go and x/tools are detailed below.
+
+### go
+
+3. Update the compiler's own UIR reader in
+ `src/cmd/compile/internal/noder/reader.go` to guard the reading of any fields
+ added in N+1.
+4. Repeat this change for the readers in `src/go/internal/gcimporter/ureader.go`
+ and `src/cmd/compile/internal/importer/ureader.go`. Note that these readers
+ only read data needed for type checking (in `src/go/types` and
+ `src/cmd/compile/internal/types2` respectively). For instance, they do not
+ read exported function bodies. Thus, it's possible that a change to UIR (such
+ as the encoding of function bodies) would require no change to these readers.
+
+### x/tools
+
+5. Add a version flag for `MyChange` in `internal/pkgbits/version.go`. Note:
+ x/tools has its own pkgbits implementation, which is intended to be an exact
+ copy of the [one in go](#the-writer). Any change made to one must be
+ reflected in the other.
+6. Update the x/tools UIR reader in `internal/gcimporter/ureader.go` to guard
+ the reading of any fields added in N+1. Call this commit C.
+7. In go, take the commit hash for C and update `src/cmd/go.mod` to use x/tools@C
+ per the [vendoring instructions](https://go.dev/wiki/MinorReleases#cherry-pick-cls-for-vendored-golangorgx-packages).
+
+## Finalizing
+
+> If this UIR change will be tested, check the [following section](#testing) and
+> consider when it makes to finalize.
+
+Only after reviewing *all* of the readers, bump the UIR version written by the
+writer to N+1 in `src/cmd/compile/internal/noder/unified.go`. Because the
+readers have already been updated to handle version N+1, this change is
+compatible.
+
+## Testing
+
+If making changes related to some new feature requiring extensive testing, it's
+best to postpone bumping the UIR version until *all* of the tests are in. To
+commit tests incrementally, develop them with a locally-incremented UIR version
+and commit *skipped* tests; don't yet bump the remote UIR version.
+
+Once all of the required tests are in, bump the remote UIR version while turning
+on all of the previously skipped tests. This minimizes churn on the UIR version
+as testing uncovers any discrepancies.
diff --git a/src/cmd/compile/internal/noder/dump.go b/src/cmd/compile/internal/noder/dump.go
new file mode 100644
index 00000000000000..629848fff576c5
--- /dev/null
+++ b/src/cmd/compile/internal/noder/dump.go
@@ -0,0 +1,70 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noder
+
+import (
+ "sync"
+
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/ir"
+ "cmd/compile/internal/syntax"
+ "cmd/compile/internal/types2"
+)
+
+// MatchASTDump returns true if the fn matches the value
+// of the astdump debug flag.
+func MatchASTDump(fn *syntax.FuncDecl) bool {
+ if len(base.Debug.AstDump) == 0 {
+ return false
+ }
+ if fn.Name == nil {
+ return false
+ }
+ return matchForDump(fn, base.Ctxt.Pkgpath)
+}
+
+// matchForDump is marked noinline to ensure that the exported
+// function MatchAstDump IS inlineable and is also small, because
+// common case is AstDump is not set.
+//
+//go:noinline
+func matchForDump(fn *syntax.FuncDecl, pkgPath string) bool {
+ return ir.MatchPkgFn(pkgPath, fn.Name.Value, base.Debug.AstDump)
+}
+
+func escapedFileName(fn *syntax.FuncDecl, suffix string) string {
+ return ir.EscapedFileName(base.Ctxt.Pkgpath+"."+fn.Name.Value, suffix)
+}
+
+var mu sync.Mutex
+var htmlWriters = make(map[*syntax.FuncDecl]*HTMLWriter)
+var orderedFuncs = []*syntax.FuncDecl{}
+
+// DumpNodeHTML dumps the node n to the HTML writer for fn.
+func DumpNodeHTML(pkg *types2.Package, file *syntax.File, info *types2.Info, fn *syntax.FuncDecl, why string, n syntax.Node) {
+ mu.Lock()
+ defer mu.Unlock()
+ w, ok := htmlWriters[fn]
+ if !ok {
+ name := escapedFileName(fn, ".syntax.html")
+ w = NewHTMLWriter(pkg, file, info, name, fn, "")
+ htmlWriters[fn] = w
+ orderedFuncs = append(orderedFuncs, fn)
+ }
+ w.WritePhase(why, why)
+}
+
+// CloseHTMLWriters closes the HTML writer for fn, if one exists.
+func CloseHTMLWriters() {
+ mu.Lock()
+ defer mu.Unlock()
+ for _, fn := range orderedFuncs {
+ if w, ok := htmlWriters[fn]; ok {
+ w.Close("Writing html syntax output for %s to %s\n", w.pkgFuncName(), w.Path())
+ delete(htmlWriters, fn)
+ }
+ }
+ orderedFuncs = nil
+}
diff --git a/src/cmd/compile/internal/noder/html.go b/src/cmd/compile/internal/noder/html.go
new file mode 100644
index 00000000000000..d68dd56b52441c
--- /dev/null
+++ b/src/cmd/compile/internal/noder/html.go
@@ -0,0 +1,420 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package noder
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/ir"
+ "cmd/compile/internal/syntax"
+ "cmd/compile/internal/types2"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "html"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+)
+
+// An HTMLWriter dumps syntax nodes to multicolumn HTML, similar to what the
+// ssa backend does for GOSSAFUNC.
+type HTMLWriter struct {
+ ir.HTMLWriterBase
+
+ Decl *syntax.FuncDecl
+ pkg *types2.Package
+ file *syntax.File
+ info *types2.Info
+}
+
+func NewHTMLWriter(pkg *types2.Package, file *syntax.File, info *types2.Info, path string, decl *syntax.FuncDecl, cfgMask string) *HTMLWriter {
+ path = strings.ReplaceAll(path, "/", string(filepath.Separator))
+ out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+ if err != nil {
+ panic(err)
+ }
+ reportPath := path
+ if !filepath.IsAbs(reportPath) {
+ pwd, err := os.Getwd()
+ if err != nil {
+ panic(err)
+ }
+ reportPath = filepath.Join(pwd, path)
+ }
+ h := HTMLWriter{
+ pkg: pkg,
+ file: file,
+ info: info,
+ Decl: decl,
+ }
+ h.Init(out, reportPath, h.DeclHTML)
+ h.start()
+ return &h
+}
+
+func (w *HTMLWriter) pkgFuncName() string {
+ p := w.pkg.Path()
+ if p == "" {
+ p = base.Ctxt.Pkgpath
+ }
+ return p + "." + w.Decl.Name.Value
+}
+
+func (w *HTMLWriter) start() {
+ if w == nil {
+ return
+ }
+ escName := html.EscapeString(w.pkgFuncName())
+ w.Print("")
+ w.Print("")
+ w.Printf(`
+AST display for %s
+`, escName, ir.CSS, ir.JS("checked", "rangefunc"), escName)
+ w.Print("")
+ w.Print("")
+ w.Print(escName)
+ w.Print(" ")
+ w.Print(`
+help
+
+
+
+Click anywhere on a node (with "cell" cursor) to outline a node and all of its subtrees.
+
+
+Click on a name (with "crosshair" cursor) to highlight every occurrence of a name.
+(Note that all the name nodes are the same node, so those also all outline together).
+
+
+Click on a file, line, or column (with "crosshair" cursor) to highlight positions
+in that file, at that file:line, or at that file:line:column, respectively.
+
+Click on a ` + ir.DownArrow + ` to collapse a subtree, or on a ` + ir.RightArrow + ` to expand a subtree.
+
+
+Non-tree attributes, like scope and type lookups, are displayed in italics. Those may
+also be clicked to highlight identity relationships within and between phases.
+
+
+
darkmode
+")
+ w.Print("")
+}
+
+func (w *HTMLWriter) DeclHTML(phase string) func() {
+ return func() {
+ w.Print("") // use pre for formatting to preserve indentation
+ w.dumpScopeHTML(w.pkg.Scope(), 1, false)
+ w.dumpScopeHTML(w.info.Scopes[w.file], 1, false)
+ w.dumpNodeHTML(w.Decl, 1, "")
+ w.Print(" ")
+ }
+}
+
+func (h *HTMLWriter) dumpNodesHTML(list []syntax.Node, depth int) {
+ if len(list) == 0 {
+ h.Print(" ")
+ return
+ }
+
+ for _, n := range list {
+ h.dumpNodeHTML(n, depth, "")
+ }
+}
+
+func isValid(t types2.Type) bool {
+ return t != nil && types2.Unalias(t) != types2.Typ[types2.Invalid]
+}
+
+const indentString = ". "
+
+func (w *HTMLWriter) indent(n int) {
+ w.Print("\n")
+ for range n {
+ w.Print(indentString)
+ }
+}
+
+// indentForToggle prints indentation to w.
+func (h *HTMLWriter) indentForToggle(depth int, hasChildren bool) {
+ h.Print("\n")
+ if depth == 0 {
+ return
+ }
+ for range depth - 1 {
+ h.Print(indentString)
+ }
+ if hasChildren {
+ // Remove 2 spaces, which have similar rendered width to
+ // leading ir.DownArrow and trailing space.
+ h.Print(indentString[:len(indentString)-2])
+ } else {
+ h.Print(indentString)
+ }
+}
+
+// dumpScopeHTML writes a string representation of the scope to w,
+// with the scope elements sorted by name.
+// The level of indentation is controlled by n >= 0, with
+// n == 0 for no indentation.
+// If recurse is set, it also writes nested (children) scopes.
+func (h *HTMLWriter) dumpScopeHTML(s *types2.Scope, depth int, recur bool) {
+ hasChildren := true // TODO detect empty scopes
+ h.indentForToggle(depth, hasChildren)
+ if hasChildren {
+ h.Printf("", h.CanonId(s))
+ defer h.Printf(" ")
+ // NOTE TRAILING SPACE after ! See indentForToggle above.
+ h.Print(`` + ir.DownArrow + ` `)
+ }
+ h.Printf("scope %s %p", html.EscapeString(s.Comment()), s)
+
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+
+ for _, name := range s.Names() {
+ obj := s.Lookup(name)
+ h.dumpOutlineNodeHTML(depth+1, "", obj)
+ }
+
+ if recur {
+ for i := range s.NumChildren() {
+ c := s.Child(i)
+ h.dumpScopeHTML(c, depth+1, recur)
+ }
+ }
+ }
+}
+
+func (h *HTMLWriter) dumpOutlineNodeHTML(depth int, pfx string, obj fmt.Stringer) {
+ h.indentForToggle(depth, false)
+ h.Printf("%s%s ",
+ h.CanonId(obj), pfx, html.EscapeString(obj.String()))
+}
+
+func (h *HTMLWriter) dumpNodeHTML(n syntax.Node, depth int, prefix string) {
+ hasChildren := h.nodeHasChildren(n)
+ h.indentForToggle(depth, hasChildren)
+
+ if depth > 40 {
+ h.Print("...")
+ return
+ }
+
+ if n == nil {
+ h.Print("NilSyntaxNode")
+ return
+ }
+
+ h.Printf("", h.CanonId(n))
+ defer h.Printf(" ")
+
+ if hasChildren {
+ // NOTE TRAILING SPACE after ! See indentForToggle above.
+ h.Print(`` + ir.DownArrow + ` `) // NOTE TRAILING SPACE after !
+ }
+
+ opName := strings.TrimPrefix(fmt.Sprintf("%T", n), "*syntax.")
+
+ if prefix != "" {
+ h.Printf("%s", html.EscapeString(prefix))
+ }
+
+ switch n := n.(type) {
+ case *syntax.BasicLit:
+ h.Printf("%s-%v", opName, html.EscapeString(n.Value))
+ h.dumpNodeHeaderHTML(n)
+ return
+
+ case *syntax.Name:
+ name := n.Value
+ hash := sha256.Sum256([]byte(name))
+ symID := "sym-" + hex.EncodeToString(hash[:6])
+ h.Printf("%s-%s ", opName, symID, html.EscapeString(name))
+ h.dumpNodeHeaderHTML(n)
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+
+ if obj := h.info.ObjectOf(n); obj != nil {
+ h.dumpOutlineNodeHTML(depth+1, "objectOf=", obj)
+ }
+ if typ := h.info.TypeOf(n); isValid(typ) {
+ h.dumpOutlineNodeHTML(depth+1, "typeOf=", typ)
+ }
+ }
+ return
+
+ case syntax.Expr:
+ h.Printf("%s", opName)
+ h.dumpNodeHeaderHTML(n)
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+
+ if typ := h.info.TypeOf(n); isValid(typ) {
+ h.dumpOutlineNodeHTML(depth+1, "typeOf=", typ)
+ }
+ }
+
+ default:
+ h.Printf("%s", opName)
+ h.dumpNodeHeaderHTML(n)
+ if hasChildren {
+ h.Print(``)
+ defer h.Print(` `)
+ }
+ }
+
+ if s := h.info.Scopes[n]; s != nil && s.Len() > 0 {
+ h.dumpScopeHTML(s, depth+1, false)
+ }
+
+ v := reflect.ValueOf(n).Elem()
+ t := v.Type()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ vf := v.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ switch tf.Type.Kind() {
+ case reflect.Interface, reflect.Ptr, reflect.Slice:
+ if vf.IsNil() {
+ continue
+ }
+ }
+ name := strings.TrimSuffix(tf.Name, "_")
+
+ switch val := vf.Interface().(type) {
+ case syntax.Node:
+ if name != "" {
+ h.dumpNodeHTML(val, depth+1, name+": ")
+ } else {
+ h.dumpNodeHTML(val, depth+1, "")
+ }
+ default:
+ if vf.Kind() == reflect.Slice && vf.Type().Elem().Implements(nodeType) {
+ if vf.Len() == 0 {
+ continue
+ }
+ if name != "" {
+ for i := range vf.Len() {
+ h.dumpNodeHTML(vf.Index(i).Interface().(syntax.Node), depth+1,
+ fmt.Sprintf("%s[%d]: ", name, i))
+ }
+ } else {
+ for i := range vf.Len() {
+ h.dumpNodeHTML(vf.Index(i).Interface().(syntax.Node), depth+1, "")
+ }
+ }
+ }
+ }
+ }
+}
+
+var nodeType = reflect.TypeFor[syntax.Node]()
+
+func (h *HTMLWriter) nodeHasChildren(n syntax.Node) bool {
+ if n == nil {
+ return false
+ }
+ switch x := n.(type) {
+ case *syntax.BasicLit:
+ return false
+ case *syntax.Name:
+ return h.info.ObjectOf(x) != nil || isValid(h.info.TypeOf(x))
+ case syntax.Expr:
+ if isValid(h.info.TypeOf(x)) {
+ return true
+ }
+ }
+
+ v := reflect.ValueOf(n).Elem()
+ t := reflect.TypeOf(n).Elem()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ vf := v.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ switch tf.Type.Kind() {
+ case reflect.Interface, reflect.Ptr, reflect.Slice:
+ if vf.IsNil() {
+ continue
+ }
+ }
+ switch vf.Interface().(type) {
+ case syntax.Node:
+ return true
+ default:
+ if vf.Kind() == reflect.Slice && vf.Type().Elem().Implements(nodeType) && vf.Len() > 0 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (h *HTMLWriter) dumpNodeHeaderHTML(n syntax.Node) {
+ v := reflect.ValueOf(n).Elem()
+ t := v.Type()
+ nf := t.NumField()
+ for i := 0; i < nf; i++ {
+ tf := t.Field(i)
+ if tf.PkgPath != "" {
+ continue
+ }
+ k := tf.Type.Kind()
+ if reflect.Bool <= k && k <= reflect.Complex128 || k == reflect.String {
+ name := strings.TrimSuffix(tf.Name, "_")
+ if name == "Value" {
+ continue
+ }
+ vf := v.Field(i)
+ vfi := vf.Interface()
+ if vf.IsZero() {
+ continue
+ }
+ if vfi == true {
+ h.Printf(" %s", name)
+ } else {
+ h.Printf(" %s:%+v", name, html.EscapeString(fmt.Sprint(vf.Interface())))
+ }
+ }
+ }
+
+ if n.Pos().IsKnown() {
+ h.Print(" ")
+ file := n.Pos().Base().Filename()
+ if file != "" {
+ hash := sha256.Sum256([]byte(file))
+ fileID := "loc-" + hex.EncodeToString(hash[:6])
+ lineID := fmt.Sprintf("%s-L%d", fileID, n.Pos().Line())
+ colID := fmt.Sprintf("%s-C%d", lineID, n.Pos().Col())
+
+ h.Printf("%s :", fileID, html.EscapeString(filepath.Base(file)))
+ h.Printf("%d :", lineID, fileID, n.Pos().Line())
+ h.Printf("%d ", colID, lineID, fileID, n.Pos().Col())
+ } else {
+ h.Printf("%v", html.EscapeString(n.Pos().String()))
+ }
+ h.Print(" ")
+ }
+}
diff --git a/src/cmd/compile/internal/noder/import.go b/src/cmd/compile/internal/noder/import.go
index 910988f0612ca9..1069cbdfe8f3f5 100644
--- a/src/cmd/compile/internal/noder/import.go
+++ b/src/cmd/compile/internal/noder/import.go
@@ -192,7 +192,10 @@ func readImportFile(path string, target *ir.Package, env *types2.Context, packag
pkg1 = types.NewPkg(path, "")
if packages != nil {
pkg2 = packages[path]
- assert(pkg1.Direct == (pkg2 != nil && pkg2.Complete()))
+ if !(pkg1.Direct == (pkg2 != nil && pkg2.Complete())) {
+
+ base.Fatalf("pkg1.Direct == (pkg2 != nil && pkg2.Complete()), path=%s\npackages=%v", path, packages)
+ }
}
if pkg1.Direct {
diff --git a/src/cmd/compile/internal/noder/irgen.go b/src/cmd/compile/internal/noder/irgen.go
index 05f0affe8a8a43..e8b177f18fa12e 100644
--- a/src/cmd/compile/internal/noder/irgen.go
+++ b/src/cmd/compile/internal/noder/irgen.go
@@ -12,8 +12,11 @@ import (
"sort"
"cmd/compile/internal/base"
+ "cmd/compile/internal/midway"
"cmd/compile/internal/rangefunc"
"cmd/compile/internal/syntax"
+ "cmd/compile/internal/typecheck"
+ "cmd/compile/internal/types"
"cmd/compile/internal/types2"
"cmd/internal/src"
)
@@ -44,6 +47,9 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info, map[*
fileBaseMap[p.file.Pos().FileBase()] = p.file
}
+ didMidway := false
+
+recheck:
// typechecking
ctxt := types2.NewContext()
importer := gcimports{
@@ -56,7 +62,6 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info, map[*
IgnoreBranchErrors: true, // parser already checked via syntax.CheckBranches mode
Importer: &importer,
Sizes: types2.SizesFor("gc", buildcfg.GOARCH),
- EnableAlias: true,
}
if base.Flag.ErrorURL {
conf.ErrorURL = " [go.dev/e/%s]"
@@ -172,6 +177,33 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info, map[*
}
base.ExitIfErrors()
+ if len(base.Debug.AstDump) > 0 {
+ dumpSyntax(pkg, info, files, "checked")
+ }
+
+ if buildcfg.Experiment.SIMD && !didMidway {
+ didMidway = true
+ // Perform midway transformation on AST directly
+ if midway.RewriteWrapper(pkg, info, files) {
+ // midway made changes; type checking must be repeated.
+ if len(base.Debug.AstDump) > 0 {
+ // TODO how should this interact with -W and textual dumps
+ dumpSyntax(pkg, info, files, "midway before recheck")
+ }
+ // necessary to reset type checking
+ for _, p := range types.PkgMap() {
+ p.Direct = false
+ }
+ // necessary to reset type checking
+ typecheck.Target.Imports = nil
+ goto recheck
+ }
+ }
+
+ if len(base.Debug.AstDump) > 0 {
+ dumpSyntax(pkg, info, files, "midway after recheck")
+ }
+
// Rewrite range over function to explicit function calls
// with the loop bodies converted into new implicit closures.
// We do this now, before serialization to unified IR, so that if the
@@ -181,9 +213,25 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info, map[*
// and bodyReaderFor will fail.
rangeInfo := rangefunc.Rewrite(pkg, info, files)
+ if len(base.Debug.AstDump) > 0 {
+ dumpSyntax(pkg, info, files, "rangefunc")
+ }
+
return pkg, info, rangeInfo
}
+func dumpSyntax(pkg *types2.Package, info *types2.Info, files []*syntax.File, phase string) {
+ for _, file := range files {
+ for _, decl := range file.DeclList {
+ if fn, ok := decl.(*syntax.FuncDecl); ok {
+ if MatchASTDump(fn) {
+ DumpNodeHTML(pkg, file, info, fn, phase, fn)
+ }
+ }
+ }
+ }
+}
+
// A cycleFinder detects anonymous interface cycles (go.dev/issue/56103).
type cycleFinder struct {
cyclic map[*types2.Interface]bool
diff --git a/src/cmd/compile/internal/noder/linker.go b/src/cmd/compile/internal/noder/linker.go
index 51b03a1897e830..df1e248398b8c8 100644
--- a/src/cmd/compile/internal/noder/linker.go
+++ b/src/cmd/compile/internal/noder/linker.go
@@ -338,6 +338,7 @@ func (l *linker) linkname(w *pkgbits.Encoder, name *ir.Name) {
linkname := name.Sym().Linkname
if !l.lsymIdx(w, linkname, name.Linksym()) {
w.String(linkname)
+ w.Bool(name.Linksym().IsLinknameStd())
}
}
diff --git a/src/cmd/compile/internal/noder/noder.go b/src/cmd/compile/internal/noder/noder.go
index 79a90783336030..5d33072690f075 100644
--- a/src/cmd/compile/internal/noder/noder.go
+++ b/src/cmd/compile/internal/noder/noder.go
@@ -41,7 +41,6 @@ func LoadPackage(filenames []string) {
// Move the entire syntax processing logic into a separate goroutine to avoid blocking on the "sem".
go func() {
for i, filename := range filenames {
- filename := filename
p := noders[i]
sem <- struct{}{}
go func() {
@@ -104,9 +103,10 @@ type noder struct {
err chan syntax.Error
}
-// linkname records a //go:linkname directive.
+// linkname records a //go:linkname or //go:linknamestd directive.
type linkname struct {
pos syntax.Pos
+ std bool
local string
remote string
}
@@ -175,6 +175,10 @@ type pragmas struct {
WasmExport *WasmExport
}
+func (p *pragmas) Nointerface() bool {
+ return p.Flag&ir.Nointerface != 0
+}
+
// WasmImport stores metadata associated with the //go:wasmimport pragma
type WasmImport struct {
Pos syntax.Pos
@@ -274,10 +278,10 @@ func (p *noder) pragma(pos syntax.Pos, blankLine bool, text string, old syntax.P
}
}
- case strings.HasPrefix(text, "go:linkname "):
+ case strings.HasPrefix(text, "go:linkname "), strings.HasPrefix(text, "go:linknamestd "):
f := strings.Fields(text)
if !(2 <= len(f) && len(f) <= 3) {
- p.error(syntax.Error{Pos: pos, Msg: "usage: //go:linkname localname [linkname]"})
+ p.error(syntax.Error{Pos: pos, Msg: fmt.Sprintf("usage: //%s localname [linkname]", f[0])})
break
}
// The second argument is optional. If omitted, we use
@@ -295,7 +299,7 @@ func (p *noder) pragma(pos syntax.Pos, blankLine bool, text string, old syntax.P
} else {
panic("missing pkgpath")
}
- p.linknames = append(p.linknames, linkname{pos, f[1], target})
+ p.linknames = append(p.linknames, linkname{pos, f[0] == "go:linknamestd", f[1], target})
case text == "go:embed", strings.HasPrefix(text, "go:embed "):
args, err := parseGoEmbed(text[len("go:embed"):])
diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go
index d7dd58d8caafe8..405f9c44ad372d 100644
--- a/src/cmd/compile/internal/noder/reader.go
+++ b/src/cmd/compile/internal/noder/reader.go
@@ -19,6 +19,7 @@ import (
"cmd/compile/internal/inline/interleaved"
"cmd/compile/internal/ir"
"cmd/compile/internal/objw"
+ "cmd/compile/internal/pgoir"
"cmd/compile/internal/reflectdata"
"cmd/compile/internal/staticinit"
"cmd/compile/internal/typecheck"
@@ -100,6 +101,11 @@ type reader struct {
dict *readerDict
+ // funcLitGen is a counter for closure names.
+ funcLitGen int
+ // rangeLitGen is a counter for range func closure names.
+ rangeLitGen int
+
// TODO(mdempsky): The state below is all specific to reading
// function bodies. It probably makes sense to split it out
// separately so that it doesn't take up space in every reader
@@ -198,6 +204,9 @@ type readerDict struct {
// implicits counts how many of types within targs are implicit type
// arguments; the rest are explicit.
implicits int
+ // receivers counts how many of types within targs are receiver type
+ // arguments; they are explicit.
+ receivers int
derived []derivedInfo // reloc index of the derived type's descriptor
derivedTypes []*types.Type // slice of previously computed derived types
@@ -781,17 +790,32 @@ func (pr *pkgReader) objIdxMayFail(idx index, implicits, explicits []*types.Type
return name, nil
case pkgbits.ObjFunc:
- if sym.Name == "init" {
- sym = Renameinit()
- }
-
npos := r.pos()
setBasePos(npos)
+
+ var sel *types.Sym
+ var recv *types.Field
+ if r.Version().Has(pkgbits.GenericMethods) && r.Bool() {
+ sel = r.selector()
+ r.recvTypeParamNames()
+ recv = r.param()
+ if r.Version().Has(pkgbits.PreserveMethodOrder) {
+ _ = r.Len() // method index not needed in compiler
+ }
+ } else {
+ if sym.Name == "init" {
+ sym = Renameinit()
+ }
+ }
r.typeParamNames()
- typ := r.signature(nil)
+ typ := r.signature(recv)
fpos := r.pos()
fn := ir.NewFunc(fpos, npos, sym, typ)
+ if r.hasTypeParams() && r.dict.shaped {
+ typ.SetHasShape(true)
+ }
+
name := fn.Nname
if !sym.IsBlank() {
if sym.Def != nil {
@@ -812,7 +836,7 @@ func (pr *pkgReader) objIdxMayFail(idx index, implicits, explicits []*types.Type
}
}
- rext.funcExt(name, nil)
+ rext.funcExt(name, sel)
return name, nil
case pkgbits.ObjType:
@@ -860,39 +884,83 @@ func (pr *pkgReader) objIdxMayFail(idx index, implicits, explicits []*types.Type
}
}
+// mangle shapes the non-shaped symbol sym under the current dictionary.
func (dict *readerDict) mangle(sym *types.Sym) *types.Sym {
if !dict.hasTypeParams() {
return sym
}
+ var buf strings.Builder
// If sym is a locally defined generic type, we need the suffix to
// stay at the end after mangling so that types/fmt.go can strip it
// out again when writing the type's runtime descriptor (#54456).
- base, suffix := types.SplitVargenSuffix(sym.Name)
+ n0, vsuff := types.SplitVargenSuffix(sym.Name)
+ n1, msuff := types.SplitMethSuffix(sym.Name)
+
+ // Methods are never locally defined.
+ var n string
+ assert(vsuff == "" || msuff == "")
+ if vsuff != "" {
+ n = n0
+ } else {
+ n = n1
+ }
- var buf strings.Builder
- buf.WriteString(base)
- buf.WriteByte('[')
- for i, targ := range dict.targs {
- if i > 0 {
- if i == dict.implicits {
- buf.WriteByte(';')
- } else {
+ var j int
+ assert(dict.implicits == 0 || dict.receivers == 0)
+ if msuff != "" {
+ j = dict.receivers // consume receiver type arguments
+ } else {
+ j = len(dict.targs) // consume all type arguments
+ }
+
+ // put type arguments inside parenthesis; (*T)[int] -> (*T[int])
+ n, ok := strings.CutSuffix(n, ")")
+
+ // type arguments, if any
+ buf.WriteString(n)
+ if j > 0 {
+ buf.WriteByte('[')
+ for i := 0; i < j; i++ {
+ if i > 0 {
+ if i == dict.implicits {
+ buf.WriteByte(';')
+ } else {
+ buf.WriteByte(',')
+ }
+ }
+ buf.WriteString(dict.targs[i].LinkString())
+ }
+ buf.WriteByte(']')
+ }
+
+ if ok {
+ buf.WriteString(")")
+ }
+
+ buf.WriteString(vsuff)
+ buf.WriteString(msuff)
+
+ // method arguments, if any
+ if msuff != "" {
+ buf.WriteByte('[')
+ for i := j; i < len(dict.targs); i++ {
+ if i > j {
buf.WriteByte(',')
}
+ buf.WriteString(dict.targs[i].LinkString())
}
- buf.WriteString(targ.LinkString())
+ buf.WriteByte(']')
}
- buf.WriteByte(']')
- buf.WriteString(suffix)
+
return sym.Pkg.Lookup(buf.String())
}
-// shapify returns the shape type for targ.
+// Shapify returns the shape type for targ.
//
// If basic is true, then the type argument is used to instantiate a
// type parameter whose constraint is a basic interface.
-func shapify(targ *types.Type, basic bool) *types.Type {
+func Shapify(targ *types.Type, basic bool) *types.Type {
if targ.Kind() == types.TFORW {
if targ.IsFullyInstantiated() {
// For recursive instantiated type argument, it may still be a TFORW
@@ -930,19 +998,8 @@ func shapify(targ *types.Type, basic bool) *types.Type {
// types, and discarding struct field names and tags. However, we'll
// need to start tracking how type parameters are actually used to
// implement some of these optimizations.
- pointerShaping := basic && targ.IsPtr() && !targ.Elem().NotInHeap()
- // The exception is when the type parameter is a pointer to a type
- // which `Type.HasShape()` returns true, but `Type.IsShape()` returns
- // false, like `*[]go.shape.T`. This is because the type parameter is
- // used to instantiate a generic function inside another generic function.
- // In this case, we want to keep the targ as-is, otherwise, we may lose the
- // original type after `*[]go.shape.T` is shapified to `*go.shape.uint8`.
- // See issue #54535, #71184.
- if pointerShaping && !targ.Elem().IsShape() && targ.Elem().HasShape() {
- return targ
- }
under := targ.Underlying()
- if pointerShaping {
+ if basic && targ.IsPtr() && !targ.Elem().NotInHeap() {
under = types.NewPtr(types.Types[types.TUINT8])
}
@@ -977,7 +1034,11 @@ func (pr *pkgReader) objDictIdx(sym *types.Sym, idx index, implicits, explicits
}
nimplicits := r.Len()
- nexplicits := r.Len()
+ nreceivers := 0
+ if r.Version().Has(pkgbits.GenericMethods) {
+ nreceivers = r.Len()
+ }
+ nexplicits := r.Len() + nreceivers
if nimplicits > len(implicits) || nexplicits != len(explicits) {
return nil, fmt.Errorf("%v has %v+%v params, but instantiated with %v+%v args", sym, nimplicits, nexplicits, len(implicits), len(explicits))
@@ -985,6 +1046,7 @@ func (pr *pkgReader) objDictIdx(sym *types.Sym, idx index, implicits, explicits
dict.targs = append(implicits[:nimplicits:nimplicits], explicits...)
dict.implicits = nimplicits
+ dict.receivers = nreceivers
// Within the compiler, we can just skip over the type parameters.
for range dict.targs[dict.implicits:] {
@@ -1019,7 +1081,7 @@ func (pr *pkgReader) objDictIdx(sym *types.Sym, idx index, implicits, explicits
for i, targ := range dict.targs {
basic := r.Bool()
if dict.shaped {
- dict.targs[i] = shapify(targ, basic)
+ dict.targs[i] = Shapify(targ, basic)
}
}
@@ -1051,10 +1113,19 @@ func (pr *pkgReader) objDictIdx(sym *types.Sym, idx index, implicits, explicits
return &dict, nil
}
+func (r *reader) recvTypeParamNames() {
+ r.Sync(pkgbits.SyncTypeParamNames)
+
+ for range r.dict.targs[r.dict.implicits : r.dict.implicits+r.dict.receivers] {
+ r.pos()
+ r.localIdent()
+ }
+}
+
func (r *reader) typeParamNames() {
r.Sync(pkgbits.SyncTypeParamNames)
- for range r.dict.targs[r.dict.implicits:] {
+ for range r.dict.targs[r.dict.implicits+r.dict.receivers:] {
r.pos()
r.localIdent()
}
@@ -1062,6 +1133,9 @@ func (r *reader) typeParamNames() {
func (r *reader) method(rext *reader) *types.Field {
r.Sync(pkgbits.SyncMethod)
+ if r.Version().Has(pkgbits.PreserveMethodOrder) {
+ _ = r.Len() // method index not needed in compiler
+ }
npos := r.pos()
sym := r.selector()
r.typeParamNames()
@@ -1233,6 +1307,7 @@ func (r *reader) linkname(name *ir.Name) {
lsym.Set(obj.AttrIndexed, true)
} else {
linkname := r.String()
+ std := r.Bool()
sym := name.Sym()
sym.Linkname = linkname
if sym.Pkg == types.LocalPkg && linkname != "" {
@@ -1242,7 +1317,11 @@ func (r *reader) linkname(name *ir.Name) {
// corresponding packages). So we can tell in which package
// the linkname is used (pulled), and the linker can
// make a decision for allowing or disallowing it.
- sym.Linksym().Set(obj.AttrLinkname, true)
+ if std {
+ sym.Linksym().Set(obj.AttrLinknameStd, true)
+ } else {
+ sym.Linksym().Set(obj.AttrLinkname, true)
+ }
}
}
}
@@ -1304,7 +1383,19 @@ func (r *reader) addBody(fn *ir.Func, method *types.Sym) {
func (pri pkgReaderIndex) funcBody(fn *ir.Func) {
r := pri.asReader(pkgbits.SectionBody, pkgbits.SyncFuncBody)
+ panicking := true
+ defer func() {
+ if panicking {
+ // TODO not sure what the best way to print in this context is.
+ // If code panics in unified IR reading, you want *something* like this.
+ // Whoever ends up debugging the next unified IR failure, please
+ // improve this (base.Warnf?) if you can figure out how.
+ fmt.Printf("****** panic traversed funcBody of %v\n", fn)
+ }
+ }()
r.funcBody(fn)
+ panicking = false
+
}
// funcBody reads a function body definition from the element
@@ -1364,13 +1455,12 @@ func (r *reader) callShaped(pos src.XPos) {
var shapedFn ir.Node
if r.methodSym == nil {
- // Instantiating a generic function; shapedObj is the shaped
- // function itself.
+ // Instantiating a generic function; shapedObj is the shaped function itself.
assert(shapedObj.Op() == ir.ONAME && shapedObj.Class == ir.PFUNC)
shapedFn = shapedObj
} else {
- // Instantiating a generic type's method; shapedObj is the shaped
- // type, so we need to select it's corresponding method.
+ // Instantiating a generic type's method; shapedObj is the shaped method itself
+ // if the method is generic — else, it is the shaped type declaring the method.
shapedFn = shapedMethodExpr(pos, shapedObj, r.methodSym)
}
@@ -2496,6 +2586,11 @@ func (r *reader) expr() (res ir.Node) {
}
x.SetType(typ)
+
+ if call, ok := x.(*ir.CallExpr); ok {
+ call.Reshape = true
+ }
+
return x
case exprConvert:
@@ -2507,6 +2602,22 @@ func (r *reader) expr() (res ir.Node) {
identical := r.Bool()
x := r.expr()
+ // spec: "If the type is a type parameter, the constant is converted
+ // into a non-constant value of the type parameter."
+ if dstTypeParam && ir.IsConstNode(x) {
+ // ConvertVal only handles conversions to constant types.
+ if v := typecheck.ConvertVal(x.Val(), typ, false); v.Kind() != constant.Unknown {
+ x = ir.NewBasicLit(x.Pos(), typ, v)
+ // Wrap in an OCONVNOP node to ensure result is non-constant.
+ n := Implicit(ir.NewConvExpr(pos, ir.OCONVNOP, typ, x))
+ n.SetTypecheck(1)
+ return n
+ }
+ // A Go language constant could be converted to a non-constant value,
+ // like converting string to []byte/[]rune. In this case, just construct
+ // the conversion expression as usual, see #79960.
+ }
+
// TODO(mdempsky): Stop constructing expressions of untyped type.
x = typecheck.DefaultLit(x, typ)
@@ -2539,13 +2650,6 @@ func (r *reader) expr() (res ir.Node) {
}
}
- // spec: "If the type is a type parameter, the constant is converted
- // into a non-constant value of the type parameter."
- if dstTypeParam && ir.IsConstNode(n) {
- // Wrap in an OCONVNOP node to ensure result is non-constant.
- n = Implicit(ir.NewConvExpr(pos, ir.OCONVNOP, n.Type(), n))
- n.SetTypecheck(1)
- }
return n
case exprRuntimeBuiltin:
@@ -2846,14 +2950,18 @@ func (r *reader) optExpr() ir.Node {
// otherwise, they need to create their own wrapper.
func (r *reader) methodExpr() (wrapperFn, baseFn, dictPtr ir.Node) {
recv := r.typ()
- sig0 := r.typ()
+
+ var sig *types.Type
+ generic := r.Version().Has(pkgbits.GenericMethods) && r.Bool()
+ if !generic {
+ // Signature type to return (i.e., recv prepended to the method's
+ // normal parameters list).
+ sig = typecheck.NewMethodType(r.typ(), recv)
+ }
+
pos := r.pos()
sym := r.selector()
- // Signature type to return (i.e., recv prepended to the method's
- // normal parameters list).
- sig := typecheck.NewMethodType(sig0, recv)
-
if r.Bool() { // type parameter method expression
idx := r.Len()
word := r.dictWord(pos, r.dict.typeParamMethodExprsOffset()+idx)
@@ -2871,20 +2979,12 @@ func (r *reader) methodExpr() (wrapperFn, baseFn, dictPtr ir.Node) {
return fn, fn, nil
}
- // TODO(mdempsky): I'm pretty sure this isn't needed: implicits is
- // only relevant to locally defined types, but they can't have
- // (non-promoted) methods.
- var implicits []*types.Type
- if r.dict != nil {
- implicits = r.dict.targs
- }
-
if r.Bool() { // dynamic subdictionary
idx := r.Len()
info := r.dict.subdicts[idx]
explicits := r.p.typListIdx(info.explicits, r.dict)
- shapedObj := r.p.objIdx(info.idx, implicits, explicits, true).(*ir.Name)
+ shapedObj := r.p.objIdx(info.idx, nil, explicits, true).(*ir.Name)
shapedFn := shapedMethodExpr(pos, shapedObj, sym)
// TODO(mdempsky): Is there a more robust way to get the
@@ -2899,10 +2999,10 @@ func (r *reader) methodExpr() (wrapperFn, baseFn, dictPtr ir.Node) {
info := r.objInfo()
explicits := r.p.typListIdx(info.explicits, r.dict)
- shapedObj := r.p.objIdx(info.idx, implicits, explicits, true).(*ir.Name)
+ shapedObj := r.p.objIdx(info.idx, nil, explicits, true).(*ir.Name)
shapedFn := shapedMethodExpr(pos, shapedObj, sym)
- dict := r.p.objDictName(info.idx, implicits, explicits)
+ dict := r.p.objDictName(info.idx, nil, explicits)
dictPtr := typecheck.Expr(ir.NewAddrExpr(pos, dict))
// Check that dictPtr matches shapedFn's dictionary parameter.
@@ -2910,13 +3010,20 @@ func (r *reader) methodExpr() (wrapperFn, baseFn, dictPtr ir.Node) {
base.FatalfAt(pos, "dict %L, but shaped method %L", dict, shapedFn)
}
- // For statically known instantiations, we can take advantage of
- // the stenciled wrapper.
- base.AssertfAt(!recv.HasShape(), pos, "shaped receiver %v", recv)
- wrapperFn := typecheck.NewMethodExpr(pos, recv, sym)
- base.AssertfAt(types.Identical(sig, wrapperFn.Type()), pos, "wrapper %L does not have type %v", wrapperFn, sig)
-
- return wrapperFn, shapedFn, dictPtr
+ if !generic {
+ // For statically known instantiations, we can take advantage of
+ // the stenciled wrapper.
+ base.AssertfAt(!recv.HasShape(), pos, "shaped receiver %v", recv)
+ wrapperFn := typecheck.NewMethodExpr(pos, recv, sym)
+ base.AssertfAt(types.Identical(sig, wrapperFn.Type()), pos, "wrapper %L does not have type %v", wrapperFn, sig)
+ return wrapperFn, shapedFn, dictPtr
+ } else {
+ // Also statically known, but there is a good amount of existing
+ // machinery downstream which makes assumptions about method
+ // wrapper functions. It's safest not to emit them for now.
+ // TODO(mark): Emit wrapper functions for generic methods.
+ return nil, shapedFn, dictPtr
+ }
}
// Simple method expression; no dictionary needed.
@@ -2925,28 +3032,90 @@ func (r *reader) methodExpr() (wrapperFn, baseFn, dictPtr ir.Node) {
return fn, fn, nil
}
-// shapedMethodExpr returns the specified method on the given shaped
-// type.
-func shapedMethodExpr(pos src.XPos, obj *ir.Name, sym *types.Sym) *ir.SelectorExpr {
- assert(obj.Op() == ir.OTYPE)
+// shapedMethodExpr creates an OMETHEXPR for obj using sym.
+//
+// If obj is an OTYPE, it must refer to a generic type. If obj is an ONAME,
+// it must refer to a generic method. In either case, sym.Name must be the
+// unqualified name of the method.
+//
+// For example, given:
+//
+// package p
+//
+// type T[P any] struct {}
+//
+// func (T[P]) m() {}
+// func (T[P]) n[Q any]() {}
+//
+// then, using S as go.shape.int:
+// - in T[int].m, obj is T[S] and sym.Name is "m".
+// - in T[int].n[int], obj is T[S].n[S] and sym.Name is "n".
+//
+// Note that we could have pushed dictionaries down to methods in every case,
+// but since non-generic methods will always share the same "type environment"
+// as their defining type, we can optimize by reusing the type's dictionary.
+func shapedMethodExpr(pos src.XPos, obj *ir.Name, sym *types.Sym) ir.Node {
+ if obj.Op() == ir.OTYPE {
+ // non-generic method on generic type
+ typ := obj.Type()
+ assert(typ.HasShape())
+
+ method := func() *types.Field {
+ for _, m := range typ.Methods() {
+ if m.Sym == sym {
+ return m
+ }
+ }
- typ := obj.Type()
- assert(typ.HasShape())
+ base.FatalfAt(pos, "failed to find method %v in shaped type %v", sym, typ)
+ panic("unreachable")
+ }()
- method := func() *types.Field {
- for _, method := range typ.Methods() {
- if method.Sym == sym {
- return method
+ return typecheck.NewMethodExpr(pos, method.Type.Recv().Type, sym)
+ } else {
+ // generic method on possibly generic type
+ assert(obj.Op() == ir.ONAME && obj.Class == ir.PFUNC)
+ typ := obj.Type()
+ assert(typ.HasShape())
+
+ // OMETHEXPR assumes that the linker symbol to call looks like ".".
+ // This works because non-generic method symbols are relative to their type. But generic
+ // methods use fully-qualified names, so this won't work.
+ //
+ // To use OMETHEXPR for generic methods, we craft a dummy field on the type by removing
+ // the qualifier; OMETHEXPR will put it back later.
+ lsym := obj.Linksym().Name
+ // Since the method is generic, we know the method name must be followed by a bracket.
+ // TODO(mark): It's not ideal to rely on string naming here. Find a more robust solution.
+ idx := func() int { // Find the index of the bracket following the method name.
+ depth, i := 0, len(lsym)-1
+ for {
+ switch lsym[i] {
+ case ']':
+ depth++
+ case '[':
+ depth--
+ }
+ if depth == 0 {
+ return i
+ }
+ i--
}
- }
+ }()
+ msym := sym.Pkg.Lookup(lsym[idx-len(sym.Name):])
- base.FatalfAt(pos, "failed to find method %v in shaped type %v", sym, typ)
- panic("unreachable")
- }()
+ // Note that the field name here includes the type arguments; while also not ideal, the
+ // types package does not seem to complain.
+ m := types.NewField(obj.Pos(), msym, typ)
+ m.Nname = obj
- // Construct an OMETHEXPR node.
- recv := method.Type.Recv().Type
- return typecheck.NewMethodExpr(pos, recv, sym)
+ n := ir.NewSelectorExpr(pos, ir.OMETHEXPR, ir.TypeNode(typ.Recv().Type), msym)
+ n.Selection = m
+ n.SetType(typecheck.NewMethodType(typ, typ.Recv().Type))
+ n.SetTypecheck(1)
+
+ return n
+ }
}
func (r *reader) multiExpr() []ir.Node {
@@ -3005,7 +3174,7 @@ func (r *reader) tempCopy(pos src.XPos, expr ir.Node, init *ir.Nodes) *ir.Name {
assign := ir.NewAssignStmt(pos, tmp, expr)
assign.Def = true
- init.Append(typecheck.Stmt(ir.NewAssignStmt(pos, tmp, expr)))
+ init.Append(typecheck.Stmt(assign))
tmp.Defn = assign
@@ -3028,21 +3197,37 @@ func (r *reader) compLit() ir.Node {
if typ.IsMap() {
rtype = r.rtype(pos)
}
- isStruct := typ.Kind() == types.TSTRUCT
- elems := make([]ir.Node, r.Len())
- for i := range elems {
- elemp := &elems[i]
-
- if isStruct {
- sk := ir.NewStructKeyExpr(r.pos(), typ.Field(r.Len()), nil)
- *elemp, elemp = sk, &sk.Value
- } else if r.Bool() {
- kv := ir.NewKeyExpr(r.pos(), r.expr(), nil)
- *elemp, elemp = kv, &kv.Value
+ var elems []ir.Node
+ if r.Version().Has(pkgbits.CompactCompLiterals) {
+ n := r.Int()
+ elems = make([]ir.Node, max(n, -n) /* abs(n) */)
+ switch typ.Kind() {
+ default:
+ base.FatalfAt(pos, "unexpected composite literal type: %v", typ)
+ case types.TARRAY:
+ r.arrayElems(n >= 0, elems)
+ case types.TMAP:
+ r.mapElems(elems)
+ case types.TSLICE:
+ r.arrayElems(n >= 0, elems)
+ case types.TSTRUCT:
+ r.structElems(typ, n >= 0, elems)
+ }
+ } else {
+ elems = make([]ir.Node, r.Len())
+ isStruct := typ.Kind() == types.TSTRUCT
+ for i := range elems {
+ elemp := &elems[i]
+ if isStruct {
+ sk := ir.NewStructKeyExpr(r.pos(), typ.Field(r.Len()), nil)
+ *elemp, elemp = sk, &sk.Value
+ } else if r.Bool() {
+ kv := ir.NewKeyExpr(r.pos(), r.expr(), nil)
+ *elemp, elemp = kv, &kv.Value
+ }
+ *elemp = r.expr()
}
-
- *elemp = r.expr()
}
lit := typecheck.Expr(ir.NewCompLitExpr(pos, ir.OCOMPLIT, typ, elems))
@@ -3057,6 +3242,64 @@ func (r *reader) compLit() ir.Node {
return lit
}
+func (r *reader) arrayElems(valuesOnly bool, elems []ir.Node) {
+ if valuesOnly {
+ for i := range elems {
+ elems[i] = r.expr()
+ }
+ return
+ }
+ // some elements may have a key
+ for i := range elems {
+ if r.Bool() {
+ kv := ir.NewKeyExpr(r.pos(), r.expr(), nil)
+ kv.Value = r.expr()
+ elems[i] = kv
+ } else {
+ elems[i] = r.expr()
+ }
+ }
+}
+
+func (r *reader) mapElems(elems []ir.Node) {
+ // all elements have a key
+ for i := range elems {
+ kv := ir.NewKeyExpr(r.pos(), r.expr(), nil)
+ kv.Value = r.expr()
+ elems[i] = kv
+ }
+}
+
+func (r *reader) structElems(typ *types.Type, valuesOnly bool, elems []ir.Node) {
+ if valuesOnly {
+ for i := range elems {
+ sk := ir.NewStructKeyExpr(r.pos(), typ.Field(i), nil)
+ sk.Value = r.expr()
+ elems[i] = sk
+ }
+ return
+ }
+
+ // all elements have a key
+ for i := range elems {
+ pos := r.pos()
+ var fld *types.Field
+ if n := r.Int(); n < 0 {
+ // embedded field
+ typ := typ // don't modify the original typ
+ for range -n {
+ fld = typ.Field(r.Int())
+ typ = fld.Type
+ }
+ } else { // n >= 0
+ fld = typ.Field(n)
+ }
+ sk := ir.NewStructKeyExpr(pos, fld, nil)
+ sk.Value = r.expr()
+ elems[i] = sk
+ }
+}
+
func (r *reader) funcLit() ir.Node {
r.Sync(pkgbits.SyncFuncLit)
@@ -3112,8 +3355,17 @@ func (r *reader) inlClosureFunc(origPos src.XPos, sig *types.Type, why ir.Op) *i
curfn = r.curfn
}
+ var gen int
+ if why == ir.ORANGE {
+ r.rangeLitGen++
+ gen = r.rangeLitGen
+ } else {
+ r.funcLitGen++
+ gen = r.funcLitGen
+ }
+
// TODO(mdempsky): Remove hard-coding of typecheck.Target.
- return ir.NewClosureFunc(origPos, r.inlPos(origPos), why, sig, curfn, typecheck.Target)
+ return ir.NewClosureFunc(origPos, r.inlPos(origPos), why, sig, curfn, typecheck.Target, gen)
}
func (r *reader) exprList() []ir.Node {
@@ -3449,7 +3701,7 @@ var inlgen = 0
// unifiedInlineCall implements inline.NewInline by re-reading the function
// body from its Unified IR export data.
-func unifiedInlineCall(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
+func unifiedInlineCall(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int, profile *pgoir.Profile) *ir.InlinedCallExpr {
pri, ok := bodyReaderFor(fn)
if !ok {
base.FatalfAt(call.Pos(), "cannot inline call to %v: missing inline body", fn)
@@ -3563,7 +3815,7 @@ func unifiedInlineCall(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInd
// potentially be recursively inlined themselves; but we shouldn't
// need to read in the non-inlined bodies for the declarations
// themselves. But currently it's an easy fix to #50552.
- readBodies(typecheck.Target, true)
+ readBodies(typecheck.Target, true, profile)
// Replace any "return" statements within the function body.
var edit func(ir.Node) ir.Node
@@ -3599,6 +3851,7 @@ func unifiedInlineCall(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInd
res.SetInit(init)
res.SetType(call.Type())
res.SetTypecheck(1)
+ res.Reshape = call.Reshape
// Inlining shouldn't add any functions to todoBodies.
assert(len(todoBodies) == 0)
@@ -3750,7 +4003,7 @@ type methodValueWrapper struct {
// needWrapper records that wrapper methods may be needed at link
// time.
func (r *reader) needWrapper(typ *types.Type) {
- if typ.IsPtr() {
+ if typ.IsPtr() || typ.IsKind(types.TFORW) {
return
}
@@ -3993,9 +4246,12 @@ func addTailCall(pos src.XPos, fn *ir.Func, recv ir.Node, method *types.Field) {
call := typecheck.Call(pos, dot, args, method.Type.IsVariadic()).(*ir.CallExpr)
if recv.Type() != nil && recv.Type().IsPtr() && method.Type.Recv().Type.IsPtr() &&
- method.Embedded != 0 && !types.IsInterfaceMethod(method.Type) &&
- !unifiedHaveInlineBody(ir.MethodExprName(dot).Func) &&
- !(base.Ctxt.Arch.Name == "ppc64le" && base.Ctxt.Flag_dynlink) {
+ method.Embedded != 0 &&
+ (types.IsInterfaceMethod(method.Type) && base.Ctxt.Arch.Name != "wasm" ||
+ !types.IsInterfaceMethod(method.Type) && !unifiedHaveInlineBody(ir.MethodExprName(dot).Func)) &&
+ // TODO: implement wasm indirect tail calls
+ // TODO: do we need the ppc64le/dynlink restriction for interface tail calls?
+ !((base.Ctxt.Arch.Name == "ppc64le" || base.Ctxt.Arch.Name == "ppc64") && base.Ctxt.Flag_dynlink) {
if base.Debug.TailCall != 0 {
base.WarnfAt(fn.Nname.Type().Recv().Type.Elem().Pos(), "tail call emitted for the method %v wrapper", method.Nname)
}
@@ -4034,6 +4290,8 @@ const dictParamName = typecheck.LocalDictName
//
// The parameter types.Fields are all copied too, so their Nname
// fields can be initialized for use by the shape function.
+//
+// All signatures returned by shapeSig are marked as shaped.
func shapeSig(fn *ir.Func, dict *readerDict) *types.Type {
sig := fn.Nname.Type()
oldRecv := sig.Recv()
@@ -4056,5 +4314,7 @@ func shapeSig(fn *ir.Func, dict *readerDict) *types.Type {
results[i] = types.NewField(result.Pos, result.Sym, result.Type)
}
- return types.NewSignature(recv, params, results)
+ typ := types.NewSignature(recv, params, results)
+ typ.SetHasShape(true)
+ return typ
}
diff --git a/src/cmd/compile/internal/noder/unified.go b/src/cmd/compile/internal/noder/unified.go
index 05f4483d0d9f28..b8b0a0a144a6ec 100644
--- a/src/cmd/compile/internal/noder/unified.go
+++ b/src/cmd/compile/internal/noder/unified.go
@@ -24,6 +24,10 @@ import (
"cmd/internal/src"
)
+// uirVersion is the unified IR version to use for encoding/decoding.
+// Use V5 for generic method index ordering.
+const uirVersion = pkgbits.V5
+
// localPkgReader holds the package reader used for reading the local
// package. It exists so the unified IR linker can refer back to it
// later.
@@ -76,7 +80,7 @@ func LookupFunc(fullName string) (*ir.Func, error) {
// readBodies to post-process any funcs on the "todoBodies" list
// that were added as a result of the lookup operations.
func PostLookupCleanup() {
- readBodies(typecheck.Target, false)
+ readBodies(typecheck.Target, false, nil)
}
func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
@@ -201,7 +205,7 @@ func unified(m posMap, noders []*noder) {
r := localPkgReader.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
r.pkgInit(types.LocalPkg, target)
- readBodies(target, false)
+ readBodies(target, false, nil)
// Check that nothing snuck past typechecking.
for _, fn := range target.Funcs {
@@ -235,7 +239,7 @@ func unified(m posMap, noders []*noder) {
// If duringInlining is true, then the inline.InlineDecls is called as
// necessary on instantiations of imported generic functions, so their
// inlining costs can be computed.
-func readBodies(target *ir.Package, duringInlining bool) {
+func readBodies(target *ir.Package, duringInlining bool, profile *pgoir.Profile) {
var inlDecls []*ir.Func
// Don't use range--bodyIdx can add closures to todoBodies.
@@ -302,7 +306,7 @@ func readBodies(target *ir.Package, duringInlining bool) {
oldLowerM := base.Flag.LowerM
base.Flag.LowerM = 0
- inline.CanInlineFuncs(inlDecls, nil)
+ inline.CanInlineFuncs(inlDecls, profile)
base.Flag.LowerM = oldLowerM
for _, fn := range inlDecls {
@@ -463,10 +467,8 @@ func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
// writeUnifiedExport writes to `out` the finalized, self-contained
// Unified IR export data file for the current compilation unit.
func writeUnifiedExport(out io.Writer) {
- // Use V2 as the encoded version for aliastypeparams.
- version := pkgbits.V2
l := linker{
- pw: pkgbits.NewPkgEncoder(version, base.Debug.SyncFrames),
+ pw: pkgbits.NewPkgEncoder(uirVersion, base.Debug.SyncFrames),
pkgs: make(map[string]index),
decls: make(map[*types.Sym]index),
@@ -487,6 +489,12 @@ func writeUnifiedExport(out io.Writer) {
r.Sync(pkgbits.SyncPkg)
selfPkgIdx = l.relocIdx(pr, pkgbits.SectionPkg, r.Reloc(pkgbits.SectionPkg))
+ // Versions must match.
+ // TODO: It seems that we should be able to use r.Version() for NewPkgEncoder
+ // instead of passing uirVersion, but NewPkgEncoder is created before r.
+ // If that is correct, we should make that happen.
+ assert(r.Version() == uirVersion)
+
if r.Version().Has(pkgbits.HasInit) {
r.Bool()
}
diff --git a/src/cmd/compile/internal/noder/writer.go b/src/cmd/compile/internal/noder/writer.go
index 9c90d221c28a7e..8ca91973e3d632 100644
--- a/src/cmd/compile/internal/noder/writer.go
+++ b/src/cmd/compile/internal/noder/writer.go
@@ -11,7 +11,9 @@ import (
"go/version"
"internal/buildcfg"
"internal/pkgbits"
+ "log"
"os"
+ "slices"
"strings"
"cmd/compile/internal/base"
@@ -81,12 +83,16 @@ type pkgWriter struct {
// Maps from types2.Objects back to their syntax.Decl.
- funDecls map[*types2.Func]*syntax.FuncDecl
- typDecls map[*types2.TypeName]typeDeclGen
+ funDecls map[*types2.Func]*syntax.FuncDecl
+ typDecls map[*types2.TypeName]typeDeclGen
+ methodIdx map[*types2.Func]int // method declaration order, for x/tools decoder (#81188)
// linknames maps package-scope objects to their linker symbol name,
- // if specified by a //go:linkname directive.
- linknames map[types2.Object]string
+ // if specified by a //go:linkname or //go:linknamestd directive.
+ linknames map[types2.Object]struct {
+ remote string
+ std bool
+ }
// cgoPragmas accumulates any //go:cgo_* pragmas that need to be
// passed through to cmd/link.
@@ -96,10 +102,8 @@ type pkgWriter struct {
// newPkgWriter returns an initialized pkgWriter for the specified
// package.
func newPkgWriter(m posMap, pkg *types2.Package, info *types2.Info, otherInfo map[*syntax.FuncLit]bool) *pkgWriter {
- // Use V2 as the encoded version for aliastypeparams.
- version := pkgbits.V2
return &pkgWriter{
- PkgEncoder: pkgbits.NewPkgEncoder(version, base.Debug.SyncFrames),
+ PkgEncoder: pkgbits.NewPkgEncoder(uirVersion, base.Debug.SyncFrames),
m: m,
curpkg: pkg,
@@ -112,20 +116,24 @@ func newPkgWriter(m posMap, pkg *types2.Package, info *types2.Info, otherInfo ma
posBasesIdx: make(map[*syntax.PosBase]index),
- funDecls: make(map[*types2.Func]*syntax.FuncDecl),
- typDecls: make(map[*types2.TypeName]typeDeclGen),
+ funDecls: make(map[*types2.Func]*syntax.FuncDecl),
+ typDecls: make(map[*types2.TypeName]typeDeclGen),
+ methodIdx: make(map[*types2.Func]int),
- linknames: make(map[types2.Object]string),
+ linknames: make(map[types2.Object]struct {
+ remote string
+ std bool
+ }),
}
}
// errorf reports a user error about thing p.
-func (pw *pkgWriter) errorf(p poser, msg string, args ...interface{}) {
+func (pw *pkgWriter) errorf(p poser, msg string, args ...any) {
base.ErrorfAt(pw.m.pos(p), 0, msg, args...)
}
// fatalf reports an internal compiler error about thing p.
-func (pw *pkgWriter) fatalf(p poser, msg string, args ...interface{}) {
+func (pw *pkgWriter) fatalf(p poser, msg string, args ...any) {
base.FatalfAt(pw.m.pos(p), msg, args...)
}
@@ -198,9 +206,10 @@ type writer struct {
// A writerDict tracks types and objects that are used by a declaration.
type writerDict struct {
- // implicits is a slice of type parameters from the enclosing
- // declarations.
+ // implicits contains type parameters from enclosing declarations.
implicits []*types2.TypeParam
+ // receivers contains receiver type parameters of the declaration.
+ receivers []*types2.TypeParam
// derived is a slice of type indices for computing derived types
// (i.e., types that depend on the declaration's type parameters).
@@ -224,8 +233,7 @@ type itabInfo struct {
// typeParamIndex returns the index of the given type parameter within
// the dictionary. This may differ from typ.Index() when there are
-// implicit type parameters due to defined types declared within a
-// generic function or method.
+// implicit or receiver type parameters.
func (dict *writerDict) typeParamIndex(typ *types2.TypeParam) int {
for idx, implicit := range dict.implicits {
if implicit == typ {
@@ -233,7 +241,13 @@ func (dict *writerDict) typeParamIndex(typ *types2.TypeParam) int {
}
}
- return len(dict.implicits) + typ.Index()
+ for idx, receiver := range dict.receivers {
+ if receiver == typ {
+ return len(dict.implicits) + idx
+ }
+ }
+
+ return len(dict.implicits) + len(dict.receivers) + typ.Index()
}
// A derivedInfo represents a reference to an encoded generic Go type.
@@ -618,7 +632,7 @@ func (pw *pkgWriter) typIdx(typ types2.Type, dict *writerDict) typeInfo {
}
// namedType writes a use of the given named type into the bitstream.
-func (w *writer) namedType(obj *types2.TypeName, targs *types2.TypeList) {
+func (w *writer) namedType(obj *types2.TypeName, targs []types2.Type) {
// Named types that are declared within a generic function (and
// thus have implicit type parameters) are always derived types.
if w.p.hasImplicitTypeParams(obj) {
@@ -723,7 +737,7 @@ func (w *writer) param(param *types2.Var) {
// If obj is a generic object, then explicits are the explicit type
// arguments used to instantiate it (i.e., used to substitute the
// object's own declared type parameters).
-func (w *writer) obj(obj types2.Object, explicits *types2.TypeList) {
+func (w *writer) obj(obj types2.Object, explicits []types2.Type) {
w.objInfo(w.p.objInstIdx(obj, explicits, w.dict))
}
@@ -745,10 +759,10 @@ func (w *writer) objInfo(info objInfo) {
// objInstIdx returns the indices for an object and a corresponding
// list of type arguments used to instantiate it, adding them to the
// export data as needed.
-func (pw *pkgWriter) objInstIdx(obj types2.Object, explicits *types2.TypeList, dict *writerDict) objInfo {
- explicitInfos := make([]typeInfo, explicits.Len())
+func (pw *pkgWriter) objInstIdx(obj types2.Object, explicits []types2.Type, dict *writerDict) objInfo {
+ explicitInfos := make([]typeInfo, len(explicits))
for i := range explicitInfos {
- explicitInfos[i] = pw.typIdx(explicits.At(i), dict)
+ explicitInfos[i] = pw.typIdx(explicits[i], dict)
}
return objInfo{idx: pw.objIdx(obj), explicits: explicitInfos}
}
@@ -769,10 +783,16 @@ func (pw *pkgWriter) objIdx(obj types2.Object) index {
if isDefinedType(obj) && obj.Pkg() == pw.curpkg {
decl, ok := pw.typDecls[obj.(*types2.TypeName)]
- assert(ok)
+ if !ok {
+ base.Fatalf("%v not in pw.typDecls", obj.(*types2.TypeName))
+ }
dict.implicits = decl.implicits
}
+ if isGenericMethod(obj.Type()) {
+ dict.receivers = asTypeParamSlice(obj.Type().(*types2.Signature).RecvTypeParams())
+ }
+
// We encode objects into 4 elements across different sections, all
// sharing the same index:
//
@@ -842,11 +862,34 @@ func (w *writer) doObj(wext *writer, obj types2.Object) pkgbits.CodeObj {
return pkgbits.ObjConst
case *types2.Func:
+ if base.Flag.LowerH > 0 {
+ // Unified IR panics are the worst; this is a huge help in debugging them.
+ defer func() {
+ if p := recover(); p != nil {
+ log.Printf("Intercepted unified IR writer panic for function %s, repanicking", obj.FullName())
+ panic(p)
+ }
+ }()
+ }
decl, ok := w.p.funDecls[obj]
assert(ok)
sig := obj.Type().(*types2.Signature)
w.pos(obj)
+ if isGenericMethod(sig) {
+ w.Bool(true) // generic method
+
+ w.selector(obj)
+ w.typeParamNames(sig.RecvTypeParams())
+ w.param(sig.Recv())
+ if w.Version().Has(pkgbits.PreserveMethodOrder) {
+ w.Len(w.p.methodIdx[obj.Origin()])
+ }
+ } else {
+ if w.Version().Has(pkgbits.GenericMethods) {
+ w.Bool(false) // function
+ }
+ }
w.typeParamNames(sig.TypeParams())
w.signature(sig)
w.pos(decl)
@@ -879,9 +922,31 @@ func (w *writer) doObj(wext *writer, obj types2.Object) pkgbits.CodeObj {
wext.typeExt(obj)
w.typ(named.Underlying())
- w.Len(named.NumMethods())
- for i := 0; i < named.NumMethods(); i++ {
- w.method(wext, named.Method(i))
+ // separate generic and non-generic methods
+ var methods, gmethods []*types2.Func
+ for i := range named.NumMethods() {
+ m := named.Method(i)
+ w.p.methodIdx[m] = i
+ if isGenericMethod(m.Type()) {
+ gmethods = append(gmethods, m)
+ } else {
+ methods = append(methods, m)
+ }
+ }
+ // encode non-generic methods inline
+ w.Len(len(methods))
+ for _, m := range methods {
+ w.method(wext, m)
+ }
+ if len(gmethods) > 0 {
+ assert(w.Version().Has(pkgbits.GenericMethods))
+ }
+ // encode a pointer to each generic method
+ if w.Version().Has(pkgbits.GenericMethods) {
+ w.Len(len(gmethods))
+ for _, m := range gmethods {
+ w.Reloc(pkgbits.SectionObj, w.p.objIdx(m))
+ }
}
return pkgbits.ObjType
@@ -901,14 +966,23 @@ func (w *writer) objDict(obj types2.Object, dict *writerDict) {
// doesn't care about referenced functions.
w.dict = dict // TODO(mdempsky): This is a bit sketchy.
-
w.Len(len(dict.implicits))
+ rtparams := objRecvTypeParams(obj)
tparams := objTypeParams(obj)
- ntparams := tparams.Len()
- w.Len(ntparams)
- for i := 0; i < ntparams; i++ {
- w.typ(tparams.At(i).Constraint())
+
+ if w.Version().Has(pkgbits.GenericMethods) {
+ w.Len(len(rtparams))
+ } else {
+ assert(len(rtparams) == 0)
+ }
+ w.Len(len(tparams))
+
+ for _, rtparam := range rtparams {
+ w.typ(rtparam.Constraint())
+ }
+ for _, tparam := range tparams {
+ w.typ(tparam.Constraint())
}
nderived := len(dict.derived)
@@ -938,8 +1012,10 @@ func (w *writer) objDict(obj types2.Object, dict *writerDict) {
for _, implicit := range dict.implicits {
w.Bool(implicit.Underlying().(*types2.Interface).IsMethodSet())
}
- for i := 0; i < ntparams; i++ {
- tparam := tparams.At(i)
+ for _, rtparam := range rtparams {
+ w.Bool(rtparam.Underlying().(*types2.Interface).IsMethodSet())
+ }
+ for _, tparam := range tparams {
w.Bool(tparam.Underlying().(*types2.Interface).IsMethodSet())
}
@@ -985,6 +1061,9 @@ func (w *writer) method(wext *writer, meth *types2.Func) {
sig := meth.Type().(*types2.Signature)
w.Sync(pkgbits.SyncMethod)
+ if w.Version().Has(pkgbits.PreserveMethodOrder) {
+ w.Len(w.p.methodIdx[meth.Origin()])
+ }
w.pos(meth)
w.selector(meth)
w.typeParamNames(sig.RecvTypeParams())
@@ -995,8 +1074,8 @@ func (w *writer) method(wext *writer, meth *types2.Func) {
wext.funcExt(meth)
}
-// qualifiedIdent writes out the name of an object declared at package
-// scope. (For now, it's also used to refer to local defined types.)
+// qualifiedIdent writes out the name of an object typically declared at package
+// scope. It's also used to refer to generic methods and locally defined types.
func (w *writer) qualifiedIdent(obj types2.Object) {
w.Sync(pkgbits.SyncSym)
@@ -1014,6 +1093,17 @@ func (w *writer) qualifiedIdent(obj types2.Object) {
}
}
+ // Generic methods are promoted to objects and thus need qualified identifiers.
+ // They must be contextualized by their defining type.
+ if isGenericMethod(obj.Type()) {
+ recv := obj.Type().(*types2.Signature).Recv().Type()
+ fstr := "%s.%s"
+ if _, ok := types2.Unalias(recv).(*types2.Pointer); ok {
+ fstr = "(*%s).%s"
+ }
+ name = fmt.Sprintf(fstr, types2.Unalias(deref2(recv)).(*types2.Named).Obj().Name(), name)
+ }
+
w.pkg(obj.Pkg())
w.String(name)
}
@@ -1154,7 +1244,9 @@ func (w *writer) varExt(obj *types2.Var) {
func (w *writer) linkname(obj types2.Object) {
w.Sync(pkgbits.SyncLinkname)
w.Int64(-1)
- w.String(w.p.linknames[obj])
+ info := w.p.linknames[obj]
+ w.String(info.remote)
+ w.Bool(info.std)
}
func (w *writer) pragmaFlag(p ir.PragmaFlag) {
@@ -1557,7 +1649,7 @@ func (w *writer) forStmt(stmt *syntax.ForStmt) {
func (w *writer) distinctVars(stmt *syntax.ForStmt) bool {
lv := base.Debug.LoopVar
- fileVersion := w.p.info.FileVersions[stmt.Pos().Base()]
+ fileVersion := w.p.info.FileVersions[stmt.Pos().FileBase()]
is122 := fileVersion == "" || version.Compare(fileVersion, "go1.22") >= 0
// Turning off loopvar for 1.22 is only possible with loopvarhash=qn
@@ -1801,7 +1893,7 @@ func (w *writer) expr(expr syntax.Expr) {
expr = syntax.Unparen(expr) // skip parens; unneeded after typecheck
obj, inst := lookupObj(w.p, expr)
- targs := inst.TypeArgs
+ targs := asTypeSlice(inst.TypeArgs)
if tv, ok := w.p.maybeTypeAndValue(expr); ok {
if tv.IsRuntimeHelper() {
@@ -1846,7 +1938,7 @@ func (w *writer) expr(expr syntax.Expr) {
}
if obj != nil {
- if targs.Len() != 0 {
+ if len(targs) != 0 {
obj := obj.(*types2.Func)
w.Code(exprFuncInst)
@@ -1896,41 +1988,30 @@ func (w *writer) expr(expr syntax.Expr) {
w.selector(sel.Obj())
case types2.MethodVal:
- w.Code(exprMethodVal)
- typ := w.recvExpr(expr, sel)
- w.pos(expr)
- w.methodExpr(expr, typ, sel)
+ w.methVal(expr, sel)
case types2.MethodExpr:
- w.Code(exprMethodExpr)
-
- tv := w.p.typeAndValue(expr.X)
- assert(tv.IsType())
-
- index := sel.Index()
- implicits := index[:len(index)-1]
-
- typ := tv.Type
- w.typ(typ)
-
- w.Len(len(implicits))
- for _, ix := range implicits {
- w.Len(ix)
- typ = deref2(typ).Underlying().(*types2.Struct).Field(ix).Type()
- }
-
- recv := sel.Obj().(*types2.Func).Type().(*types2.Signature).Recv().Type()
- if w.Bool(isPtrTo(typ, recv)) { // need deref
- typ = recv
- } else if w.Bool(isPtrTo(recv, typ)) { // need addr
- typ = recv
- }
-
- w.pos(expr)
- w.methodExpr(expr, typ, sel)
+ w.methExpr(expr, sel)
}
case *syntax.IndexExpr:
+ // might be explicit instantiation of a generic method
+ if selector, ok := expr.X.(*syntax.SelectorExpr); ok {
+ if sel, ok := w.p.info.Selections[selector]; ok {
+ switch sel.Kind() {
+ default:
+ w.p.fatalf(selector, "unexpected selection kind: %v", sel.Kind())
+ case types2.FieldVal:
+ // not a method
+ case types2.MethodVal:
+ w.methVal(selector, sel)
+ return
+ case types2.MethodExpr:
+ w.methExpr(selector, sel)
+ return
+ }
+ }
+ }
_ = w.p.typeOf(expr.Index) // ensure this is an index expression, not an instantiation
xtyp := w.p.typeOf(expr.X)
@@ -2102,7 +2183,11 @@ func (w *writer) expr(expr syntax.Expr) {
writeFunExpr := func() {
fun := syntax.Unparen(expr.Fun)
- if selector, ok := fun.(*syntax.SelectorExpr); ok {
+ expr := fun
+ if idx, ok := expr.(*syntax.IndexExpr); ok {
+ expr = idx.X
+ }
+ if selector, ok := expr.(*syntax.SelectorExpr); ok {
if sel, ok := w.p.info.Selections[selector]; ok && sel.Kind() == types2.MethodVal {
w.Bool(true) // method call
typ := w.recvExpr(selector, sel)
@@ -2117,7 +2202,7 @@ func (w *writer) expr(expr syntax.Expr) {
obj := obj.(*types2.Func)
w.pos(fun)
- w.funcInst(obj, inst.TypeArgs)
+ w.funcInst(obj, asTypeSlice(inst.TypeArgs))
return
}
@@ -2156,6 +2241,42 @@ func (w *writer) optExpr(expr syntax.Expr) {
}
}
+func (w *writer) methVal(expr *syntax.SelectorExpr, sel *types2.Selection) {
+ w.Code(exprMethodVal)
+ typ := w.recvExpr(expr, sel)
+ w.pos(expr)
+ w.methodExpr(expr, typ, sel)
+}
+
+func (w *writer) methExpr(expr *syntax.SelectorExpr, sel *types2.Selection) {
+ w.Code(exprMethodExpr)
+
+ tv := w.p.typeAndValue(expr.X)
+ assert(tv.IsType())
+
+ index := sel.Index()
+ implicits := index[:len(index)-1]
+
+ typ := tv.Type
+ w.typ(typ)
+
+ w.Len(len(implicits))
+ for _, ix := range implicits {
+ w.Len(ix)
+ typ = deref2(typ).Underlying().(*types2.Struct).Field(ix).Type()
+ }
+
+ recv := sel.Obj().(*types2.Func).Type().(*types2.Signature).Recv().Type()
+ if w.Bool(isPtrTo(typ, recv)) { // need deref
+ typ = recv
+ } else if w.Bool(isPtrTo(recv, typ)) { // need addr
+ typ = recv
+ }
+
+ w.pos(expr)
+ w.methodExpr(expr, typ, sel)
+}
+
// recvExpr writes out expr.X, but handles any implicit addressing,
// dereferencing, and field selections appropriate for the method
// selection.
@@ -2185,7 +2306,7 @@ func (w *writer) recvExpr(expr *syntax.SelectorExpr, sel *types2.Selection) type
}
// funcInst writes a reference to an instantiated function.
-func (w *writer) funcInst(obj *types2.Func, targs *types2.TypeList) {
+func (w *writer) funcInst(obj *types2.Func, targs []types2.Type) {
info := w.p.objInstIdx(obj, targs, w.dict)
// Type arguments list contains derived types; we can emit a static
@@ -2213,7 +2334,18 @@ func (w *writer) methodExpr(expr *syntax.SelectorExpr, recv types2.Type, sel *ty
sig := fun.Type().(*types2.Signature)
w.typ(recv)
- w.typ(sig)
+
+ // only pass the signature if it's not a generic method
+ if isGenericMethod(sig) {
+ assert(w.Version().Has(pkgbits.GenericMethods))
+ w.Bool(true)
+ } else {
+ if w.Version().Has(pkgbits.GenericMethods) {
+ w.Bool(false)
+ }
+ w.typ(sig)
+ }
+
w.pos(expr)
w.selector(fun)
@@ -2231,31 +2363,37 @@ func (w *writer) methodExpr(expr *syntax.SelectorExpr, recv types2.Type, sel *ty
w.p.fatalf(expr, "isInterface inconsistency: %v and %v", recv, sig.Recv().Type())
}
- if !isInterface(recv) {
- if named, ok := types2.Unalias(deref2(recv)).(*types2.Named); ok {
- obj, targs := splitNamed(named)
- info := w.p.objInstIdx(obj, targs, w.dict)
-
- // Method on a derived receiver type. These can be handled by a
- // static call to the shaped method, but require dynamically
- // looking up the appropriate dictionary argument in the current
- // function's runtime dictionary.
- if w.p.hasImplicitTypeParams(obj) || info.anyDerived() {
- w.Bool(true) // dynamic subdictionary
- w.Len(w.dict.subdictIdx(info))
- return
- }
-
- // Method on a fully known receiver type. These can be handled
- // by a static call to the shaped method, and with a static
- // reference to the receiver type's dictionary.
- if targs.Len() != 0 {
- w.Bool(false) // no dynamic subdictionary
- w.Bool(true) // static dictionary
- w.objInfo(info)
- return
- }
+ if isConcreteMethod(sig) {
+ tname, tExplicits := splitNamed(types2.Unalias(deref2(recv)).(*types2.Named))
+ var info objInfo
+ if isGenericMethod(sig) {
+ // For generic methods, the shaped object is the method itself.
+ mExplicits := asTypeSlice(w.p.info.Instances[expr.Sel].TypeArgs)
+ info = w.p.objInstIdx(fun.Origin(), slices.Concat(tExplicits, mExplicits), w.dict)
+ } else {
+ // For non-generic concrete methods on generic types, the shaped object
+ // is the type. The method must be looked up on the type by name.
+ info = w.p.objInstIdx(tname, tExplicits, w.dict)
+ }
+ // We don't know all of the type arguments statically. These can be
+ // handled by a static call to the shaped method, but require
+ // dynamically looking up the appropriate dictionary argument
+ // in the current function's runtime dictionary.
+ if info.anyDerived() {
+ w.Bool(true) // dynamic subdictionary
+ w.Len(w.dict.subdictIdx(info))
+ return
}
+ // We know all of the type arguments statically. These can be handled
+ // by a static call to the shaped method, and with a static reference
+ // to either the receiver type's or method's dictionary (see above).
+ if len(info.explicits) > 0 {
+ w.Bool(false) // no dynamic subdictionary
+ w.Bool(true) // static dictionary
+ w.objInfo(info)
+ return
+ }
+ // no type arguments
}
w.Bool(false) // no dynamic subdictionary
@@ -2346,6 +2484,25 @@ func (w *writer) compLit(lit *syntax.CompositeLit) {
if ptr, ok := types2.CoreType(typ).(*types2.Pointer); ok {
typ = ptr.Elem()
}
+
+ if w.Version().Has(pkgbits.CompactCompLiterals) {
+ switch typ0 := typ; typ := types2.CoreType(typ).(type) {
+ default:
+ w.p.fatalf(lit, "unexpected composite literal type: %v", typ)
+ case *types2.Array:
+ w.arrayElems(typ.Elem(), lit.ElemList)
+ case *types2.Map:
+ w.rtype(typ0)
+ w.mapElems(typ.Key(), typ.Elem(), lit.ElemList)
+ case *types2.Slice:
+ w.arrayElems(typ.Elem(), lit.ElemList)
+ case *types2.Struct:
+ w.structElems(typ, lit.NKeys == 0, lit.ElemList)
+ }
+ return
+ }
+
+ // old format
var keyType, elemType types2.Type
var structType *types2.Struct
switch typ0 := typ; typ := types2.CoreType(typ).(type) {
@@ -2388,6 +2545,76 @@ func (w *writer) compLit(lit *syntax.CompositeLit) {
}
}
+func (w *writer) arrayElems(elemType types2.Type, elems []syntax.Expr) {
+ valuesOnly := true
+ for _, elem := range elems {
+ if _, ok := elem.(*syntax.KeyValueExpr); ok {
+ valuesOnly = false
+ break
+ }
+ }
+
+ if valuesOnly {
+ w.Int(len(elems))
+ for _, elem := range elems {
+ w.implicitConvExpr(elemType, elem)
+ }
+ return
+ }
+ // some elements may have a key
+ w.Int(-len(elems))
+ for _, elem := range elems {
+ if kv, ok := elem.(*syntax.KeyValueExpr); w.Bool(ok) {
+ w.pos(kv.Key) // use position of Key rather than of elem (which has position of ':')
+ w.implicitConvExpr(nil, kv.Key)
+ elem = kv.Value
+ }
+ w.implicitConvExpr(elemType, elem)
+ }
+}
+
+func (w *writer) mapElems(keyType, valueType types2.Type, elems []syntax.Expr) {
+ // all elements have a key
+ w.Int(-len(elems))
+ for _, elem := range elems {
+ kv := elem.(*syntax.KeyValueExpr)
+ w.pos(kv.Key) // use position of Key rather than of elem (which has position of ':')
+ w.implicitConvExpr(keyType, kv.Key)
+ w.implicitConvExpr(valueType, kv.Value)
+ }
+}
+
+func (w *writer) structElems(typ *types2.Struct, valuesOnly bool, elems []syntax.Expr) {
+ n := len(elems)
+ if valuesOnly {
+ // no element has a key
+ w.Int(n)
+ for i, elem := range elems {
+ w.pos(elem)
+ w.implicitConvExpr(typ.Field(i).Type(), elem)
+ }
+ return
+ }
+ // all elements have a key
+ w.Int(-n)
+ for _, elem := range elems {
+ kv := elem.(*syntax.KeyValueExpr)
+ w.pos(kv.Key) // use position of Key rather than of elem (which has position of ':')
+ // TODO(gri): rather than doing this lookup again, perhaps the index should be recorded by types2
+ fld, index, _ := types2.LookupFieldOrMethod(typ, false, w.p.curpkg, kv.Key.(*syntax.Name).Value)
+ if n := len(index); n > 1 {
+ // embedded field
+ w.Int(-n)
+ for _, i := range index {
+ w.Int(i)
+ }
+ } else { // n == 1
+ w.Int(index[0])
+ }
+ w.implicitConvExpr(fld.Type(), kv.Value)
+ }
+}
+
func (w *writer) funcLit(expr *syntax.FuncLit) {
sig := w.p.typeOf(expr).(*types2.Signature)
@@ -2521,6 +2748,26 @@ func isInterface(typ types2.Type) bool {
return ok
}
+// isConcreteMethod reports whether typ is a concrete method. That is,
+// it's a method with a receiver that isn't an interface type.
+func isConcreteMethod(typ types2.Type) bool {
+ sig, ok := typ.(*types2.Signature)
+ return ok && sig.Recv() != nil && !isInterface(sig.Recv().Type())
+}
+
+// TODO(mark): Use isGenericMethod. It is included now to help justify
+// the existence of isConcreteMethod.
+
+// isGenericMethod reports whether typ is a generic method. That is,
+// it's a method with type parameters apart from those which may or
+// may not appear on the receiver type.
+//
+// Note that generic methods are always concrete methods.
+func isGenericMethod(typ types2.Type) bool {
+ sig, ok := typ.(*types2.Signature)
+ return ok && sig.Recv() != nil && sig.TypeParams().Len() > 0
+}
+
// op writes an Op into the bitstream.
func (w *writer) op(op ir.Op) {
// TODO(mdempsky): Remove in favor of explicit codes? Would make
@@ -2563,16 +2810,15 @@ type declCollector struct {
}
func (c *declCollector) withTParams(obj types2.Object) *declCollector {
- tparams := objTypeParams(obj)
- n := tparams.Len()
- if n == 0 {
+ tparams := slices.Concat(objRecvTypeParams(obj), objTypeParams(obj))
+ if len(tparams) == 0 {
return c
}
copy := *c
copy.implicits = copy.implicits[:len(copy.implicits):len(copy.implicits)]
- for i := 0; i < n; i++ {
- copy.implicits = append(copy.implicits, tparams.At(i))
+ for _, tparam := range tparams {
+ copy.implicits = append(copy.implicits, tparam)
}
return ©
}
@@ -2663,26 +2909,33 @@ func (pw *pkgWriter) collectDecls(noders []*noder) {
pw.cgoPragmas = append(pw.cgoPragmas, p.pragcgobuf...)
for _, l := range p.linknames {
+ directive := "go:linkname"
+ if l.std {
+ directive = "go:linknamestd"
+ }
if !file.importedUnsafe {
- pw.errorf(l.pos, "//go:linkname only allowed in Go files that import \"unsafe\"")
+ pw.errorf(l.pos, "//%s only allowed in Go files that import \"unsafe\"", directive)
continue
}
if strings.Contains(l.remote, "[") && strings.Contains(l.remote, "]") {
- pw.errorf(l.pos, "//go:linkname reference of an instantiation is not allowed")
+ pw.errorf(l.pos, "//%s reference of an instantiation is not allowed", directive)
continue
}
switch obj := pw.curpkg.Scope().Lookup(l.local).(type) {
case *types2.Func, *types2.Var:
if _, ok := pw.linknames[obj]; !ok {
- pw.linknames[obj] = l.remote
+ pw.linknames[obj] = struct {
+ remote string
+ std bool
+ }{l.remote, l.std}
} else {
- pw.errorf(l.pos, "duplicate //go:linkname for %s", l.local)
+ pw.errorf(l.pos, "duplicate //%s for %s", directive, l.local)
}
default:
if types.AllowsGoVersion(1, 18) {
- pw.errorf(l.pos, "//go:linkname must refer to declared function or variable")
+ pw.errorf(l.pos, "//%s must refer to declared function or variable", directive)
}
}
}
@@ -2998,44 +3251,72 @@ func fieldIndex(info *types2.Info, str *types2.Struct, key *syntax.Name) int {
panic(fmt.Sprintf("%s: %v is not a field of %v", key.Pos(), field, str))
}
+// objRecvTypeParams returns the receiver type parameters on the given object.
+func objRecvTypeParams(obj types2.Object) []*types2.TypeParam {
+ if f, ok := obj.(*types2.Func); ok {
+ return asTypeParamSlice(f.Signature().RecvTypeParams())
+ }
+ return nil
+}
+
// objTypeParams returns the type parameters on the given object.
-func objTypeParams(obj types2.Object) *types2.TypeParamList {
- switch obj := obj.(type) {
+func objTypeParams(obj types2.Object) []*types2.TypeParam {
+ switch t := obj.(type) {
case *types2.Func:
- sig := obj.Type().(*types2.Signature)
- if sig.Recv() != nil {
- return sig.RecvTypeParams()
- }
- return sig.TypeParams()
+ return asTypeParamSlice(t.Signature().TypeParams())
case *types2.TypeName:
switch t := obj.Type().(type) {
case *types2.Named:
- return t.TypeParams()
+ return asTypeParamSlice(t.TypeParams())
case *types2.Alias:
- return t.TypeParams()
+ return asTypeParamSlice(t.TypeParams())
}
}
return nil
}
+// asTypeParamSlice unpacks a types2.TypeParamList to a []types2.TypeParam
+func asTypeParamSlice(l *types2.TypeParamList) []*types2.TypeParam {
+ if l.Len() == 0 {
+ return nil
+ }
+ s := make([]*types2.TypeParam, l.Len())
+ for i := range l.Len() {
+ s[i] = l.At(i)
+ }
+ return s
+}
+
// splitNamed decomposes a use of a defined type into its original
// type definition and the type arguments used to instantiate it.
-func splitNamed(typ *types2.Named) (*types2.TypeName, *types2.TypeList) {
+func splitNamed(typ *types2.Named) (*types2.TypeName, []types2.Type) {
base.Assertf(typ.TypeParams().Len() == typ.TypeArgs().Len(), "use of uninstantiated type: %v", typ)
orig := typ.Origin()
base.Assertf(orig.TypeArgs() == nil, "origin %v of %v has type arguments", orig, typ)
base.Assertf(typ.Obj() == orig.Obj(), "%v has object %v, but %v has object %v", typ, typ.Obj(), orig, orig.Obj())
- return typ.Obj(), typ.TypeArgs()
+ return typ.Obj(), asTypeSlice(typ.TypeArgs())
}
// splitAlias is like splitNamed, but for an alias type.
-func splitAlias(typ *types2.Alias) (*types2.TypeName, *types2.TypeList) {
+func splitAlias(typ *types2.Alias) (*types2.TypeName, []types2.Type) {
orig := typ.Origin()
base.Assertf(typ.Obj() == orig.Obj(), "alias type %v has object %v, but %v has object %v", typ, typ.Obj(), orig, orig.Obj())
- return typ.Obj(), typ.TypeArgs()
+ return typ.Obj(), asTypeSlice(typ.TypeArgs())
+}
+
+// asTypeSlice unpacks a types2.TypeList to a []types2.Type
+func asTypeSlice(l *types2.TypeList) []types2.Type {
+ if l.Len() == 0 {
+ return nil
+ }
+ s := make([]types2.Type, l.Len())
+ for i := range l.Len() {
+ s[i] = l.At(i)
+ }
+ return s
}
func asPragmaFlag(p syntax.Pragma) ir.PragmaFlag {
diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go
index f0d228559f3a87..a0d81d31d6de7d 100644
--- a/src/cmd/compile/internal/ppc64/ssa.go
+++ b/src/cmd/compile/internal/ppc64/ssa.go
@@ -135,6 +135,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
// AND/OR Rarg1, Rtmp
// STBCCC/STWCCC Rtmp, (Rarg0)
// BNE -3(PC)
+ // LWSYNC
ld := ppc64.ALBAR
st := ppc64.ASTBCCC
if v.Op == ssa.OpPPC64LoweredAtomicAnd32 || v.Op == ssa.OpPPC64LoweredAtomicOr32 {
@@ -170,6 +171,10 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p3 := s.Prog(ppc64.ABNE)
p3.To.Type = obj.TYPE_BRANCH
p3.To.SetTarget(p)
+ // LWSYNC - Provide acquire ordering to pair with the
+ // release (pre-LWSYNC) above, making the operation
+ // sequentially consistent.
+ s.Prog(ppc64.ALWSYNC)
case ssa.OpPPC64LoweredAtomicAdd32,
ssa.OpPPC64LoweredAtomicAdd64:
@@ -179,6 +184,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
// STDCCC/STWCCC Rout, (Rarg0)
// BNE -3(PC)
// MOVW Rout,Rout (if Add32)
+ // LWSYNC
ld := ppc64.ALDAR
st := ppc64.ASTDCCC
if v.Op == ssa.OpPPC64LoweredAtomicAdd32 {
@@ -188,10 +194,10 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
r0 := v.Args[0].Reg()
r1 := v.Args[1].Reg()
out := v.Reg0()
- // LWSYNC - Assuming shared data not write-through-required nor
- // caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
- plwsync := s.Prog(ppc64.ALWSYNC)
- plwsync.To.Type = obj.TYPE_NONE
+ // LWSYNC - Provide acquire ordering to pair with the
+ // release (pre-LWSYNC) above, making the operation
+ // sequentially consistent.
+ s.Prog(ppc64.ALWSYNC)
// LDAR or LWAR
p := s.Prog(ld)
p.From.Type = obj.TYPE_MEM
@@ -223,6 +229,11 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p5.From.Type = obj.TYPE_REG
p5.From.Reg = out
}
+ // LWSYNC - Provide acquire ordering to pair with the
+ // release (pre-LWSYNC) above, making the operation
+ // sequentially consistent.
+ plwsync2 := s.Prog(ppc64.ALWSYNC)
+ plwsync2.To.Type = obj.TYPE_NONE
case ssa.OpPPC64LoweredAtomicExchange8,
ssa.OpPPC64LoweredAtomicExchange32,
@@ -1869,7 +1880,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpPPC64CALLstatic:
s.Call(v)
- case ssa.OpPPC64CALLtail:
+ case ssa.OpPPC64CALLtail, ssa.OpPPC64CALLtailinter:
s.TailCall(v)
case ssa.OpPPC64CALLclosure, ssa.OpPPC64CALLinter:
diff --git a/src/cmd/compile/internal/rangefunc/rewrite.go b/src/cmd/compile/internal/rangefunc/rewrite.go
index 74c7f55801c6d7..2421efbad83f68 100644
--- a/src/cmd/compile/internal/rangefunc/rewrite.go
+++ b/src/cmd/compile/internal/rangefunc/rewrite.go
@@ -994,7 +994,14 @@ func (r *rewriter) computeBranchNext() {
l := n.Label.Value
labels = append(labels, l)
f := forStack[len(forStack)-1]
- r.labelLoop[l] = f
+ if _, existed := r.labelLoop[l]; existed {
+ // The typechecker has already validated that labels are unique within this scope.
+ // If a duplicate label 'X' is encountered, it must reside in a nested scope
+ // (e.g., inside a function literal). Because nested labels do not affect
+ // the current control flow analysis, they can be safely ignored.
+ } else {
+ r.labelLoop[l] = f
+ }
}
} else {
n := stack[len(stack)-1]
@@ -1175,7 +1182,7 @@ func (r *rewriter) bodyFunc(body []syntax.Stmt, lhs []syntax.Expr, def bool, fty
Type: types2.NewSignatureType(nil, nil, nil,
types2.NewTuple(params...),
types2.NewTuple(results...),
- false),
+ ftyp.Variadic()),
}
tv.SetIsValue()
bodyFunc.SetTypeInfo(tv)
diff --git a/src/cmd/compile/internal/reflectdata/alg.go b/src/cmd/compile/internal/reflectdata/alg.go
index 7cc50d866f2740..f74a42eac11a6a 100644
--- a/src/cmd/compile/internal/reflectdata/alg.go
+++ b/src/cmd/compile/internal/reflectdata/alg.go
@@ -6,136 +6,93 @@ package reflectdata
import (
"fmt"
+ "go/constant"
+ "strconv"
+ "strings"
"cmd/compile/internal/base"
- "cmd/compile/internal/compare"
"cmd/compile/internal/ir"
"cmd/compile/internal/objw"
"cmd/compile/internal/typecheck"
"cmd/compile/internal/types"
"cmd/internal/obj"
- "cmd/internal/src"
)
-// AlgType returns the fixed-width AMEMxx variants instead of the general
-// AMEM kind when possible.
-func AlgType(t *types.Type) types.AlgKind {
- a := types.AlgType(t)
- if a == types.AMEM {
- if t.Alignment() < int64(base.Ctxt.Arch.Alignment) && t.Alignment() < t.Size() {
- // For example, we can't treat [2]int16 as an int32 if int32s require
- // 4-byte alignment. See issue 46283.
- return a
- }
- switch t.Size() {
- case 0:
- return types.AMEM0
- case 1:
- return types.AMEM8
- case 2:
- return types.AMEM16
- case 4:
- return types.AMEM32
- case 8:
- return types.AMEM64
- case 16:
- return types.AMEM128
- }
- }
-
- return a
-}
-
// genhash returns a symbol which is the closure used to compute
// the hash of a value of type t.
-// Note: the generated function must match runtime.typehash exactly.
func genhash(t *types.Type) *obj.LSym {
- switch AlgType(t) {
- default:
- // genhash is only called for types that have equality
- base.Fatalf("genhash %v", t)
- case types.AMEM0:
+ return genhashSig(eqSignature(t))
+}
+
+func genhashSig(sig string) *obj.LSym {
+ if len(sig) > 0 && sig[0] == sigAlign {
+ _, sig = parseNum(sig[1:])
+ }
+ switch sig {
+ case "":
return sysClosure("memhash0")
- case types.AMEM8:
+ case string(sigMemory) + "1":
return sysClosure("memhash8")
- case types.AMEM16:
+ case string(sigMemory) + "2":
return sysClosure("memhash16")
- case types.AMEM32:
+ case string(sigMemory) + "4":
return sysClosure("memhash32")
- case types.AMEM64:
+ case string(sigMemory) + "8":
return sysClosure("memhash64")
- case types.AMEM128:
+ case string(sigMemory) + "16":
return sysClosure("memhash128")
- case types.ASTRING:
+ case string(sigString):
return sysClosure("strhash")
- case types.AINTER:
+ case string(sigIface):
return sysClosure("interhash")
- case types.ANILINTER:
+ case string(sigEface):
return sysClosure("nilinterhash")
- case types.AFLOAT32:
+ case string(sigFloat32):
return sysClosure("f32hash")
- case types.AFLOAT64:
+ case string(sigFloat64):
return sysClosure("f64hash")
- case types.ACPLX64:
+ case string(sigFloat32) + string(sigFloat32):
return sysClosure("c64hash")
- case types.ACPLX128:
+ case string(sigFloat64) + string(sigFloat64):
return sysClosure("c128hash")
- case types.AMEM:
- // For other sizes of plain memory, we build a closure
- // that calls memhash_varlen. The size of the memory is
- // encoded in the first slot of the closure.
- closure := TypeLinksymLookup(fmt.Sprintf(".hashfunc%d", t.Size()))
- if len(closure.P) > 0 { // already generated
- return closure
- }
- if memhashvarlen == nil {
- memhashvarlen = typecheck.LookupRuntimeFunc("memhash_varlen")
- }
- ot := 0
- ot = objw.SymPtr(closure, ot, memhashvarlen, 0)
- ot = objw.Uintptr(closure, ot, uint64(t.Size())) // size encoded in closure
- objw.Global(closure, int32(ot), obj.DUPOK|obj.RODATA)
- return closure
- case types.ASPECIAL:
- break
}
- closure := TypeLinksymPrefix(".hashfunc", t)
+ closure := TypeLinksymLookup(".hashfunc." + sig)
if len(closure.P) > 0 { // already generated
return closure
}
- // Generate hash functions for subtypes.
- // There are cases where we might not use these hashes,
- // but in that case they will get dead-code eliminated.
- // (And the closure generated by genhash will also get
- // dead-code eliminated, as we call the subtype hashers
- // directly.)
- switch t.Kind() {
- case types.TARRAY:
- genhash(t.Elem())
- case types.TSTRUCT:
- for _, f := range t.Fields() {
- genhash(f.Type)
+ if sig[0] == sigMemory {
+ n, rest := parseNum(sig[1:])
+ if rest == "" {
+ // Just M%d. We can make a memhash_varlen closure.
+ // The size of the memory region to hash is encoded in the closure.
+ if memhashvarlen == nil {
+ memhashvarlen = typecheck.LookupRuntimeFunc("memhash_varlen")
+ }
+ ot := 0
+ ot = objw.SymPtr(closure, ot, memhashvarlen, 0)
+ ot = objw.Uintptr(closure, ot, uint64(n)) // size encoded in closue
+ objw.Global(closure, int32(ot), obj.DUPOK|obj.RODATA)
+ return closure
}
}
if base.Flag.LowerR != 0 {
- fmt.Printf("genhash %v %v\n", closure, t)
+ fmt.Printf("genhash %s\n", sig)
}
- fn := hashFunc(t)
+ fn := hashFunc(sig)
// Build closure. It doesn't close over any variables, so
// it contains just the function pointer.
objw.SymPtr(closure, 0, fn.Linksym(), 0)
objw.Global(closure, int32(types.PtrSize), obj.DUPOK|obj.RODATA)
-
return closure
}
-func hashFunc(t *types.Type) *ir.Func {
- sym := TypeSymPrefix(".hash", t)
+func hashFunc(sig string) *ir.Func {
+ sym := types.TypeSymLookup(".hash." + sig)
if sym.Def != nil {
return sym.Def.(*ir.Name).Func
}
@@ -143,10 +100,10 @@ func hashFunc(t *types.Type) *ir.Func {
pos := base.AutogeneratedPos // less confusing than end of input
base.Pos = pos
- // func sym(p *T, h uintptr) uintptr
+ // func sym(p unsafe.Pointer, h uintptr) uintptr
fn := ir.NewFunc(pos, pos, sym, types.NewSignature(nil,
[]*types.Field{
- types.NewField(pos, typecheck.Lookup("p"), types.NewPtr(t)),
+ types.NewField(pos, typecheck.Lookup("p"), types.Types[types.TUNSAFEPTR]),
types.NewField(pos, typecheck.Lookup("h"), types.Types[types.TUINTPTR]),
},
[]*types.Field{
@@ -155,81 +112,99 @@ func hashFunc(t *types.Type) *ir.Func {
))
sym.Def = fn.Nname
fn.Pragma |= ir.Noinline // TODO(mdempsky): We need to emit this during the unified frontend instead, to allow inlining.
-
typecheck.DeclFunc(fn)
np := fn.Dcl[0]
nh := fn.Dcl[1]
- switch t.Kind() {
- case types.TARRAY:
- // An array of pure memory would be handled by the
- // standard algorithm, so the element type must not be
- // pure memory.
- hashel := hashfor(t.Elem())
-
- // for i := 0; i < nelem; i++
- ni := typecheck.TempAt(base.Pos, ir.CurFunc, types.Types[types.TINT])
- init := ir.NewAssignStmt(base.Pos, ni, ir.NewInt(base.Pos, 0))
- cond := ir.NewBinaryExpr(base.Pos, ir.OLT, ni, ir.NewInt(base.Pos, t.NumElem()))
- post := ir.NewAssignStmt(base.Pos, ni, ir.NewBinaryExpr(base.Pos, ir.OADD, ni, ir.NewInt(base.Pos, 1)))
- loop := ir.NewForStmt(base.Pos, nil, cond, post, nil, false)
- loop.PtrInit().Append(init)
-
- // h = hashel(&p[i], h)
- call := ir.NewCallExpr(base.Pos, ir.OCALL, hashel, nil)
-
- nx := ir.NewIndexExpr(base.Pos, np, ni)
- nx.SetBounded(true)
- na := typecheck.NodAddr(nx)
- call.Args.Append(na)
- call.Args.Append(nh)
- loop.Body.Append(ir.NewAssignStmt(base.Pos, nh, call))
-
- fn.Body.Append(loop)
+ // Skip alignment, hash functions can handle unaligned data.
+ if len(sig) > 0 && sig[0] == sigAlign {
+ _, sig = parseNum(sig[1:])
+ }
- case types.TSTRUCT:
- // Walk the struct using memhash for runs of AMEM
- // and calling specific hash functions for the others.
- for i, fields := 0, t.Fields(); i < len(fields); {
- f := fields[i]
+ // offset from np that we're currently working on
+ var off int64
- // Skip blank fields.
- if f.Sym.IsBlank() {
- i++
- continue
- }
+ // Return np+off (as an unsafe.Pointer).
+ ptr := func() ir.Node {
+ c := ir.NewBasicLit(pos, types.Types[types.TUINTPTR], constant.MakeInt64(off))
+ return ir.NewBinaryExpr(pos, ir.OUNSAFEADD, np, c)
+ }
+ // hash data using function name at np+off.
+ // Increment off by size.
+ hash := func(name string, size int64) {
+ p := ptr()
+ hashFn := typecheck.LookupRuntime(name)
+ call := ir.NewCallExpr(pos, ir.OCALL, hashFn, []ir.Node{p, nh})
+ fn.Body.Append(ir.NewAssignStmt(pos, nh, call))
+ off += size
+ }
- // Hash non-memory fields with appropriate hash function.
- if !compare.IsRegularMemory(f.Type) {
- hashel := hashfor(f.Type)
- call := ir.NewCallExpr(base.Pos, ir.OCALL, hashel, nil)
- na := typecheck.NodAddr(typecheck.DotField(base.Pos, np, i))
- call.Args.Append(na)
- call.Args.Append(nh)
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nh, call))
- i++
- continue
+ for len(sig) > 0 {
+ kind := sig[0]
+ sig = sig[1:]
+ switch kind {
+ case sigMemory:
+ var n int64
+ n, sig = parseNum(sig)
+ switch {
+ case n == 4:
+ p := ptr()
+ memhash := typecheck.LookupRuntime("memhash32")
+ call := ir.NewCallExpr(pos, ir.OCALL, memhash, []ir.Node{p, nh})
+ fn.Body.Append(ir.NewAssignStmt(pos, nh, call))
+ case n == 8:
+ p := ptr()
+ memhash := typecheck.LookupRuntime("memhash64")
+ call := ir.NewCallExpr(pos, ir.OCALL, memhash, []ir.Node{p, nh})
+ fn.Body.Append(ir.NewAssignStmt(pos, nh, call))
+ default:
+ p := ptr()
+ memhash := typecheck.LookupRuntime("memhash")
+ size := ir.NewBasicLit(pos, types.Types[types.TUINTPTR], constant.MakeInt64(n))
+ call := ir.NewCallExpr(pos, ir.OCALL, memhash, []ir.Node{p, nh, size})
+ fn.Body.Append(ir.NewAssignStmt(pos, nh, call))
}
-
- // Otherwise, hash a maximal length run of raw memory.
- size, next := compare.Memrun(t, i)
-
- // h = hashel(&p.first, size, h)
- hashel := hashmem(f.Type)
- call := ir.NewCallExpr(base.Pos, ir.OCALL, hashel, nil)
- na := typecheck.NodAddr(typecheck.DotField(base.Pos, np, i))
- call.Args.Append(na)
- call.Args.Append(nh)
- call.Args.Append(ir.NewInt(base.Pos, size))
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nh, call))
-
- i = next
+ off += n
+ case sigFloat32:
+ hash("f32hash", 4)
+ case sigFloat64:
+ hash("f64hash", 8)
+ case sigString:
+ hash("strhash", 2*int64(types.PtrSize))
+ case sigEface:
+ hash("nilinterhash", 2*int64(types.PtrSize))
+ case sigIface:
+ hash("interhash", 2*int64(types.PtrSize))
+ case sigSkip:
+ var n int64
+ n, sig = parseNum(sig)
+ off += n
+ case sigArrayStart:
+ var n int64
+ var elemSig string
+ n, elemSig, sig = parseArray(sig)
+ elemSize := sigSize(elemSig)
+
+ // Loop N times, calling hash function for the element.
+ // for i := off; i < off + N*elemSize; i += elemSize {
+ // h = elemfn(p+i, h)
+ // }
+ elemFn := hashFunc(elemSig).Nname
+ idx := typecheck.TempAt(pos, ir.CurFunc, types.Types[types.TUINTPTR])
+ init := ir.NewAssignStmt(pos, idx, ir.NewInt(pos, off))
+ cond := ir.NewBinaryExpr(pos, ir.OLT, idx, ir.NewInt(pos, off+n*elemSize))
+ post := ir.NewAssignStmt(pos, idx, ir.NewBinaryExpr(pos, ir.OADD, idx, ir.NewInt(pos, elemSize)))
+
+ p := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, np, idx)
+ call := typecheck.Call(pos, elemFn, []ir.Node{p, nh}, false)
+ as := ir.NewAssignStmt(pos, nh, call)
+ loop := ir.NewForStmt(pos, init, cond, post, []ir.Node{as}, false)
+ fn.Body.Append(loop)
+ off += n * elemSize
}
}
- r := ir.NewReturnStmt(base.Pos, nil)
- r.Results.Append(nh)
- fn.Body.Append(r)
+ fn.Body.Append(ir.NewReturnStmt(pos, []ir.Node{nh}))
if base.Flag.LowerR != 0 {
ir.DumpList("genhash body", fn.Body)
@@ -244,39 +219,9 @@ func hashFunc(t *types.Type) *ir.Func {
})
fn.SetNilCheckDisabled(true)
-
return fn
}
-func runtimeHashFor(name string, t *types.Type) *ir.Name {
- return typecheck.LookupRuntime(name, t)
-}
-
-// hashfor returns the function to compute the hash of a value of type t.
-func hashfor(t *types.Type) *ir.Name {
- switch types.AlgType(t) {
- case types.AMEM:
- base.Fatalf("hashfor with AMEM type")
- case types.AINTER:
- return runtimeHashFor("interhash", t)
- case types.ANILINTER:
- return runtimeHashFor("nilinterhash", t)
- case types.ASTRING:
- return runtimeHashFor("strhash", t)
- case types.AFLOAT32:
- return runtimeHashFor("f32hash", t)
- case types.AFLOAT64:
- return runtimeHashFor("f64hash", t)
- case types.ACPLX64:
- return runtimeHashFor("c64hash", t)
- case types.ACPLX128:
- return runtimeHashFor("c128hash", t)
- }
-
- fn := hashFunc(t)
- return fn.Nname
-}
-
// sysClosure returns a closure which will call the
// given runtime function (with no closed-over variables).
func sysClosure(name string) *obj.LSym {
@@ -292,66 +237,88 @@ func sysClosure(name string) *obj.LSym {
// geneq returns a symbol which is the closure used to compute
// equality for two objects of type t.
func geneq(t *types.Type) *obj.LSym {
- switch AlgType(t) {
+ switch types.AlgType(t) {
case types.ANOEQ, types.ANOALG:
// The runtime will panic if it tries to compare
// a type with a nil equality function.
return nil
- case types.AMEM0:
+ }
+ return geneqSig(eqSignature(t))
+}
+
+// geneqSig returns a symbol which is the closure used to compute
+// equality for two objects with equality signature sig.
+func geneqSig(sig string) *obj.LSym {
+ align := int64(types.PtrSize)
+ if len(sig) > 0 && sig[0] == sigAlign {
+ align, sig = parseNum(sig[1:])
+ }
+ if base.Ctxt.Arch.CanMergeLoads {
+ align = 8
+ }
+ switch sig {
+ case "":
return sysClosure("memequal0")
- case types.AMEM8:
+ case string(sigMemory) + "1":
return sysClosure("memequal8")
- case types.AMEM16:
- return sysClosure("memequal16")
- case types.AMEM32:
- return sysClosure("memequal32")
- case types.AMEM64:
- return sysClosure("memequal64")
- case types.AMEM128:
- return sysClosure("memequal128")
- case types.ASTRING:
+ case string(sigMemory) + "2":
+ if align >= 2 {
+ return sysClosure("memequal16")
+ }
+ case string(sigMemory) + "4":
+ if align >= 4 {
+ return sysClosure("memequal32")
+ }
+ case string(sigMemory) + "8":
+ if align >= 8 {
+ return sysClosure("memequal64")
+ }
+ case string(sigMemory) + "16":
+ if align >= 8 {
+ return sysClosure("memequal128")
+ }
+ case string(sigString):
return sysClosure("strequal")
- case types.AINTER:
+ case string(sigIface):
return sysClosure("interequal")
- case types.ANILINTER:
+ case string(sigEface):
return sysClosure("nilinterequal")
- case types.AFLOAT32:
+ case string(sigFloat32):
return sysClosure("f32equal")
- case types.AFLOAT64:
+ case string(sigFloat64):
return sysClosure("f64equal")
- case types.ACPLX64:
+ case string(sigFloat32) + string(sigFloat32):
return sysClosure("c64equal")
- case types.ACPLX128:
+ case string(sigFloat64) + string(sigFloat64):
return sysClosure("c128equal")
- case types.AMEM:
- // make equality closure. The size of the type
- // is encoded in the closure.
- closure := TypeLinksymLookup(fmt.Sprintf(".eqfunc%d", t.Size()))
- if len(closure.P) != 0 {
- return closure
- }
- if memequalvarlen == nil {
- memequalvarlen = typecheck.LookupRuntimeFunc("memequal_varlen")
- }
- ot := 0
- ot = objw.SymPtr(closure, ot, memequalvarlen, 0)
- ot = objw.Uintptr(closure, ot, uint64(t.Size()))
- objw.Global(closure, int32(ot), obj.DUPOK|obj.RODATA)
- return closure
- case types.ASPECIAL:
- break
}
- closure := TypeLinksymPrefix(".eqfunc", t)
+ closure := TypeLinksymLookup(".eqfunc." + sig)
if len(closure.P) > 0 { // already generated
return closure
}
+ if sig[0] == sigMemory {
+ n, rest := parseNum(sig[1:])
+ if rest == "" {
+ // Just M%d. We can make a memequal_varlen closure.
+ // The size of the memory region to compare is encoded in the closure.
+ if memequalvarlen == nil {
+ memequalvarlen = typecheck.LookupRuntimeFunc("memequal_varlen")
+ }
+ ot := 0
+ ot = objw.SymPtr(closure, ot, memequalvarlen, 0)
+ ot = objw.Uintptr(closure, ot, uint64(n))
+ objw.Global(closure, int32(ot), obj.DUPOK|obj.RODATA)
+ return closure
+ }
+ }
+
if base.Flag.LowerR != 0 {
- fmt.Printf("geneq %v\n", t)
+ fmt.Printf("geneqSig %s\n", sig)
}
- fn := eqFunc(t)
+ fn := eqFunc(sig)
// Generate a closure which points at the function we just generated.
objw.SymPtr(closure, 0, fn.Linksym(), 0)
@@ -359,9 +326,8 @@ func geneq(t *types.Type) *obj.LSym {
return closure
}
-func eqFunc(t *types.Type) *ir.Func {
- // Autogenerate code for equality of structs and arrays.
- sym := TypeSymPrefix(".eq", t)
+func eqFunc(sig string) *ir.Func {
+ sym := types.TypeSymLookup(".eq." + sig)
if sym.Def != nil {
return sym.Def.(*ir.Name).Func
}
@@ -369,11 +335,11 @@ func eqFunc(t *types.Type) *ir.Func {
pos := base.AutogeneratedPos // less confusing than end of input
base.Pos = pos
- // func sym(p, q *T) bool
+ // func sym(p, q unsafe.Pointer) bool
fn := ir.NewFunc(pos, pos, sym, types.NewSignature(nil,
[]*types.Field{
- types.NewField(pos, typecheck.Lookup("p"), types.NewPtr(t)),
- types.NewField(pos, typecheck.Lookup("q"), types.NewPtr(t)),
+ types.NewField(pos, typecheck.Lookup("p"), types.Types[types.TUNSAFEPTR]),
+ types.NewField(pos, typecheck.Lookup("q"), types.Types[types.TUNSAFEPTR]),
},
[]*types.Field{
types.NewField(pos, typecheck.Lookup("r"), types.Types[types.TBOOL]),
@@ -381,7 +347,6 @@ func eqFunc(t *types.Type) *ir.Func {
))
sym.Def = fn.Nname
fn.Pragma |= ir.Noinline // TODO(mdempsky): We need to emit this during the unified frontend instead, to allow inlining.
-
typecheck.DeclFunc(fn)
np := fn.Dcl[0]
nq := fn.Dcl[1]
@@ -390,231 +355,208 @@ func eqFunc(t *types.Type) *ir.Func {
// Label to jump to if an equality test fails.
neq := typecheck.AutoLabel(".neq")
- // We reach here only for types that have equality but
- // cannot be handled by the standard algorithms,
- // so t must be either an array or a struct.
- switch t.Kind() {
- default:
- base.Fatalf("geneq %v", t)
-
- case types.TARRAY:
- nelem := t.NumElem()
-
- // checkAll generates code to check the equality of all array elements.
- // If unroll is greater than nelem, checkAll generates:
- //
- // if eq(p[0], q[0]) && eq(p[1], q[1]) && ... {
- // } else {
- // goto neq
- // }
- //
- // And so on.
- //
- // Otherwise it generates:
- //
- // iterateTo := nelem/unroll*unroll
- // for i := 0; i < iterateTo; i += unroll {
- // if eq(p[i+0], q[i+0]) && eq(p[i+1], q[i+1]) && ... && eq(p[i+unroll-1], q[i+unroll-1]) {
- // } else {
- // goto neq
- // }
- // }
- // if eq(p[iterateTo+0], q[iterateTo+0]) && eq(p[iterateTo+1], q[iterateTo+1]) && ... {
- // } else {
- // goto neq
- // }
- //
- checkAll := func(unroll int64, last bool, eq func(pi, qi ir.Node) ir.Node) {
- // checkIdx generates a node to check for equality at index i.
- checkIdx := func(i ir.Node) ir.Node {
- // pi := p[i]
- pi := ir.NewIndexExpr(base.Pos, np, i)
- pi.SetBounded(true)
- pi.SetType(t.Elem())
- // qi := q[i]
- qi := ir.NewIndexExpr(base.Pos, nq, i)
- qi.SetBounded(true)
- qi.SetType(t.Elem())
- return eq(pi, qi)
- }
-
- iterations := nelem / unroll
- iterateTo := iterations * unroll
- // If a loop is iterated only once, there shouldn't be any loop at all.
- if iterations == 1 {
- iterateTo = 0
- }
-
- if iterateTo > 0 {
- // Generate an unrolled for loop.
- // for i := 0; i < nelem/unroll*unroll; i += unroll
- i := typecheck.TempAt(base.Pos, ir.CurFunc, types.Types[types.TINT])
- init := ir.NewAssignStmt(base.Pos, i, ir.NewInt(base.Pos, 0))
- cond := ir.NewBinaryExpr(base.Pos, ir.OLT, i, ir.NewInt(base.Pos, iterateTo))
- loop := ir.NewForStmt(base.Pos, nil, cond, nil, nil, false)
- loop.PtrInit().Append(init)
-
- // if eq(p[i+0], q[i+0]) && eq(p[i+1], q[i+1]) && ... && eq(p[i+unroll-1], q[i+unroll-1]) {
- // } else {
- // goto neq
- // }
- for j := int64(0); j < unroll; j++ {
- // if check {} else { goto neq }
- nif := ir.NewIfStmt(base.Pos, checkIdx(i), nil, nil)
- nif.Else.Append(ir.NewBranchStmt(base.Pos, ir.OGOTO, neq))
- loop.Body.Append(nif)
- post := ir.NewAssignStmt(base.Pos, i, ir.NewBinaryExpr(base.Pos, ir.OADD, i, ir.NewInt(base.Pos, 1)))
- loop.Body.Append(post)
- }
-
- fn.Body.Append(loop)
-
- if nelem == iterateTo {
- if last {
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nr, ir.NewBool(base.Pos, true)))
- }
- return
- }
- }
+ // Grab known alignment of argument pointers. (ptrSize is the default.)
+ align := int64(types.PtrSize)
+ if len(sig) > 0 && sig[0] == sigAlign {
+ sig = sig[1:]
+ align, sig = parseNum(sig)
+ }
+ unalignedOk := base.Ctxt.Arch.CanMergeLoads
+
+ // offset from np/nq that we're currently working on
+ var off int64
+ var hasCall bool
+
+ // test takes a boolean. If it evaluates to false, short circuit
+ // and return false immediately. Otherwise, keep checking.
+ var lastTest ir.Node
+ test := func(eq ir.Node) {
+ // Buffer one test in lastTest so we can use the
+ // last one as the return value.
+ if lastTest != nil {
+ nif := ir.NewIfStmt(pos, lastTest, nil, []ir.Node{ir.NewBranchStmt(pos, ir.OGOTO, neq)})
+ fn.Body.Append(nif)
+ }
+ lastTest = eq
+ }
+ // load loads data of type t from np+off and nq+off.
+ // Increments off by the size of t.
+ load := func(t *types.Type) (ir.Node, ir.Node) {
+ c := ir.NewBasicLit(pos, types.Types[types.TUINTPTR], constant.MakeInt64(off))
+ p := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, np, c)
+ q := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, nq, c)
+ x := ir.NewStarExpr(pos, ir.NewConvExpr(pos, ir.OCONVNOP, t.PtrTo(), p))
+ y := ir.NewStarExpr(pos, ir.NewConvExpr(pos, ir.OCONVNOP, t.PtrTo(), q))
+ off += t.Size()
+ return x, y
+ }
+ // compare compares x and y and jumps to neq if they are not equal.
+ compare := func(x, y ir.Node) {
+ test(ir.NewBinaryExpr(pos, ir.OEQ, x, y))
+ }
- // Generate remaining checks, if nelem is not a multiple of unroll.
- if last {
- // Do last comparison in a different manner.
- nelem--
- }
- // if eq(p[iterateTo+0], q[iterateTo+0]) && eq(p[iterateTo+1], q[iterateTo+1]) && ... {
- // } else {
- // goto neq
- // }
- for j := iterateTo; j < nelem; j++ {
- // if check {} else { goto neq }
- nif := ir.NewIfStmt(base.Pos, checkIdx(ir.NewInt(base.Pos, j)), nil, nil)
- nif.Else.Append(ir.NewBranchStmt(base.Pos, ir.OGOTO, neq))
- fn.Body.Append(nif)
- }
- if last {
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nr, checkIdx(ir.NewInt(base.Pos, nelem))))
- }
+ // We keep track of string contents that we don't compare immediately.
+ // We delay comparing string contents because they might be large and
+ // we'd rather compare scalars farther along in the signature first.
+ var pendingStrings []int64
+ flushStrings := func() {
+ defer func(saveOff int64) {
+ off = saveOff
+ }(off)
+ for _, x := range pendingStrings {
+ off = x
+ ptrA, ptrB := load(types.Types[types.TUNSAFEPTR])
+ len, _ := load(types.Types[types.TUINTPTR])
+ // Note: we already checked that the lengths are equal.
+ memeq := typecheck.LookupRuntime("memequal")
+ test(typecheck.Call(pos, memeq, []ir.Node{ptrA, ptrB, len}, false))
+ hasCall = true
}
+ pendingStrings = pendingStrings[:0]
+ }
- switch t.Elem().Kind() {
- case types.TSTRING:
- // Do two loops. First, check that all the lengths match (cheap).
- // Second, check that all the contents match (expensive).
- checkAll(3, false, func(pi, qi ir.Node) ir.Node {
- // Compare lengths.
- eqlen, _ := compare.EqString(pi, qi)
- return eqlen
- })
- checkAll(1, true, func(pi, qi ir.Node) ir.Node {
- // Compare contents.
- _, eqmem := compare.EqString(pi, qi)
- return eqmem
- })
- case types.TFLOAT32, types.TFLOAT64:
- checkAll(2, true, func(pi, qi ir.Node) ir.Node {
- // p[i] == q[i]
- return ir.NewBinaryExpr(base.Pos, ir.OEQ, pi, qi)
- })
- case types.TSTRUCT:
- isCall := func(n ir.Node) bool {
- return n.Op() == ir.OCALL || n.Op() == ir.OCALLFUNC
+ for len(sig) > 0 {
+ kind := sig[0]
+ sig = sig[1:]
+ switch kind {
+ case sigMemory:
+ var n int64
+ n, sig = parseNum(sig)
+ if n > 64 { // TODO: why 64?
+ // For big regions, call memequal.
+ c := ir.NewBasicLit(pos, types.Types[types.TUINTPTR], constant.MakeInt64(off))
+ p := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, np, c)
+ q := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, nq, c)
+ len := ir.NewBasicLit(pos, types.Types[types.TUINTPTR], constant.MakeInt64(n))
+ memeq := typecheck.LookupRuntime("memequal")
+ test(typecheck.Call(pos, memeq, []ir.Node{p, q, len}, false))
+ hasCall = true
+ off += n
+ n = 0
}
- var expr ir.Node
- var hasCallExprs bool
- allCallExprs := true
- and := func(cond ir.Node) {
- if expr == nil {
- expr = cond
- } else {
- expr = ir.NewLogicalExpr(base.Pos, ir.OANDAND, expr, cond)
+ n0 := n
+ for n != 0 {
+ switch {
+ case n >= 8 && (unalignedOk || align >= 8 && off%8 == 0):
+ compare(load(types.Types[types.TUINT64]))
+ n -= 8
+ case (n == 5 || n == 6 || n == 7) && unalignedOk && n0 >= 8:
+ off -= 8 - n
+ compare(load(types.Types[types.TUINT64]))
+ n = 0
+ case n >= 4 && (unalignedOk || align >= 4 && off%4 == 0):
+ compare(load(types.Types[types.TUINT32]))
+ n -= 4
+ case n == 3 && unalignedOk && n0 >= 4:
+ off--
+ compare(load(types.Types[types.TUINT32]))
+ n = 0
+ case n >= 2 && (unalignedOk || align >= 2 && off%2 == 0):
+ compare(load(types.Types[types.TUINT16]))
+ n -= 2
+ default:
+ compare(load(types.Types[types.TUINT8]))
+ n--
}
}
-
- var tmpPos src.XPos
- pi := ir.NewIndexExpr(tmpPos, np, ir.NewInt(tmpPos, 0))
- pi.SetBounded(true)
- pi.SetType(t.Elem())
- qi := ir.NewIndexExpr(tmpPos, nq, ir.NewInt(tmpPos, 0))
- qi.SetBounded(true)
- qi.SetType(t.Elem())
- flatConds, canPanic := compare.EqStruct(t.Elem(), pi, qi)
- for _, c := range flatConds {
- if isCall(c) {
- hasCallExprs = true
- } else {
- allCallExprs = false
- }
- }
- if !hasCallExprs || allCallExprs || canPanic {
- checkAll(1, true, func(pi, qi ir.Node) ir.Node {
- // p[i] == q[i]
- return ir.NewBinaryExpr(base.Pos, ir.OEQ, pi, qi)
- })
+ case sigFloat32:
+ compare(load(types.Types[types.TFLOAT32]))
+ case sigFloat64:
+ compare(load(types.Types[types.TFLOAT64]))
+ case sigString:
+ // Compare just the lengths right now.
+ // Save the contents for later.
+ pendingStrings = append(pendingStrings, off)
+ off += int64(types.PtrSize)
+ compare(load(types.Types[types.TUINTPTR]))
+ case sigEface, sigIface:
+ // flushStrings here to ensure that we only get a panic from
+ // this interface test if all previous equality checks pass.
+ flushStrings()
+ typeX, typeY := load(types.Types[types.TUINTPTR].PtrTo())
+ compare(typeX, typeY)
+ dataX, dataY := load(types.Types[types.TUNSAFEPTR])
+ var eqFn *ir.Name
+ if kind == sigEface {
+ eqFn = typecheck.LookupRuntime("efaceeq")
} else {
- checkAll(4, false, func(pi, qi ir.Node) ir.Node {
- expr = nil
- flatConds, _ := compare.EqStruct(t.Elem(), pi, qi)
- if len(flatConds) == 0 {
- return ir.NewBool(base.Pos, true)
- }
- for _, c := range flatConds {
- if !isCall(c) {
- and(c)
- }
- }
- return expr
- })
- checkAll(2, true, func(pi, qi ir.Node) ir.Node {
- expr = nil
- flatConds, _ := compare.EqStruct(t.Elem(), pi, qi)
- for _, c := range flatConds {
- if isCall(c) {
- and(c)
- }
- }
- return expr
- })
+ eqFn = typecheck.LookupRuntime("ifaceeq")
}
- default:
- checkAll(1, true, func(pi, qi ir.Node) ir.Node {
- // p[i] == q[i]
- return ir.NewBinaryExpr(base.Pos, ir.OEQ, pi, qi)
- })
- }
-
- case types.TSTRUCT:
- flatConds, _ := compare.EqStruct(t, np, nq)
- if len(flatConds) == 0 {
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nr, ir.NewBool(base.Pos, true)))
- } else {
- for _, c := range flatConds[:len(flatConds)-1] {
- // if cond {} else { goto neq }
- n := ir.NewIfStmt(base.Pos, c, nil, nil)
- n.Else.Append(ir.NewBranchStmt(base.Pos, ir.OGOTO, neq))
- fn.Body.Append(n)
+ test(typecheck.Call(pos, eqFn, []ir.Node{typeX, dataX, dataY}, false))
+ hasCall = true
+ case sigSkip:
+ var n int64
+ n, sig = parseNum(sig)
+ off += n
+ case sigArrayStart:
+ // Flush any pending test.
+ flushStrings()
+ // TODO: if the element comparison can't panic (no E or I), then
+ // maybe we don't need to do this flushStrings?
+ // On the other hand, maybe the unflushed string is not equal, but
+ // a big following array is all equal.
+ if lastTest != nil {
+ nif := ir.NewIfStmt(pos, lastTest, nil, []ir.Node{ir.NewBranchStmt(pos, ir.OGOTO, neq)})
+ fn.Body.Append(nif)
+ lastTest = nil
}
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nr, flatConds[len(flatConds)-1]))
+
+ var n int64
+ var elemSig string
+ n, elemSig, sig = parseArray(sig)
+ elemSize := sigSize(elemSig)
+
+ // Loop N times, calling comparison function for the element.
+ // for i := off; i < off + N*elemSize; i += elemSize {
+ // if !eqfn(p+i, q+i) { goto neq }
+ // }
+ elemFn := eqFunc(sigTrimSkip(elemSig)).Nname
+ idx := typecheck.TempAt(pos, ir.CurFunc, types.Types[types.TUINTPTR])
+ init := ir.NewAssignStmt(pos, idx, ir.NewInt(pos, off))
+ cond := ir.NewBinaryExpr(pos, ir.OLT, idx, ir.NewInt(pos, off+n*elemSize))
+ post := ir.NewAssignStmt(pos, idx, ir.NewBinaryExpr(pos, ir.OADD, idx, ir.NewInt(pos, elemSize)))
+
+ p := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, np, idx)
+ q := ir.NewBinaryExpr(pos, ir.OUNSAFEADD, nq, idx)
+ call := typecheck.Call(pos, elemFn, []ir.Node{p, q}, false)
+ nif := ir.NewIfStmt(pos, call, nil, []ir.Node{ir.NewBranchStmt(pos, ir.OGOTO, neq)})
+ loop := ir.NewForStmt(pos, init, cond, post, []ir.Node{nif}, false)
+ fn.Body.Append(loop)
+ off += n * elemSize
+
+ // TODO: if the element comparison can't panic, but has strings
+ // in it, maybe we do a loop first without string contents and a
+ // second loop with string contents. There is no way to accomplish
+ // this now they way this code works (to call the equality
+ // function of the sub-signature).
}
}
+ // Flush any pending tests.
+ // The last test is used directly as a result (instead of branching using it).
+ flushStrings()
+ if lastTest == nil {
+ lastTest = ir.NewBool(pos, true)
+ }
+ as := ir.NewAssignStmt(pos, nr, lastTest)
+ fn.Body.Append(as)
// ret:
// return
ret := typecheck.AutoLabel(".ret")
- fn.Body.Append(ir.NewLabelStmt(base.Pos, ret))
- fn.Body.Append(ir.NewReturnStmt(base.Pos, nil))
+ fn.Body.Append(ir.NewLabelStmt(pos, ret))
+ fn.Body.Append(ir.NewReturnStmt(pos, nil))
// neq:
// r = false
// return (or goto ret)
- fn.Body.Append(ir.NewLabelStmt(base.Pos, neq))
- fn.Body.Append(ir.NewAssignStmt(base.Pos, nr, ir.NewBool(base.Pos, false)))
- if compare.EqCanPanic(t) || anyCall(fn) {
+ fn.Body.Append(ir.NewLabelStmt(pos, neq))
+ fn.Body.Append(ir.NewAssignStmt(pos, nr, ir.NewBool(pos, false)))
+ if hasCall {
// Epilogue is large, so share it with the equal case.
- fn.Body.Append(ir.NewBranchStmt(base.Pos, ir.OGOTO, ret))
+ fn.Body.Append(ir.NewBranchStmt(pos, ir.OGOTO, ret))
} else {
// Epilogue is small, so don't bother sharing.
- fn.Body.Append(ir.NewReturnStmt(base.Pos, nil))
+ fn.Body.Append(ir.NewReturnStmt(pos, nil))
}
// TODO(khr): the epilogue size detection condition above isn't perfect.
// We should really do a generic CL that shares epilogues across
@@ -645,23 +587,311 @@ func eqFunc(t *types.Type) *ir.Func {
func EqFor(t *types.Type) (ir.Node, bool) {
switch types.AlgType(t) {
case types.AMEM:
- return typecheck.LookupRuntime("memequal", t, t), true
+ return typecheck.LookupRuntime("memequal"), true
case types.ASPECIAL:
- fn := eqFunc(t)
+ fn := eqFunc(eqSignature(t))
return fn.Nname, false
}
base.Fatalf("EqFor %v", t)
return nil, false
}
-func anyCall(fn *ir.Func) bool {
- return ir.Any(fn, func(n ir.Node) bool {
- // TODO(rsc): No methods?
- op := n.Op()
- return op == ir.OCALL || op == ir.OCALLFUNC
- })
+// eqSignature returns a signature of the equality function for type t.
+// If two types have the same signature, they can use the same equality function.
+// The signature lists the comparisons that the equality function needs
+// to make, in order. So for instance, a type like:
+//
+// type S struct {
+// i int32
+// j uint32
+// s string
+// e error
+// }
+//
+// Will have the signature "M8SI".
+//
+// M8 = 8 bytes of regular memory
+// S = string
+// I = nonempty interface
+//
+// The content of the signature is not intended for users. It is an
+// internal condensation of the comparison operations that need to be
+// performed.
+// (Although, note that these names might be seen in tracebacks where
+// the equality test panics due to incomparable interfaces.)
+//
+// Full signature spec:
+//
+// M%d = %d bytes of memory that should be compared directly
+// K%d = %d bytes of memory that should not be compared (sKip)
+// F = float32
+// G = float64
+// S = string
+// I = non-empty interface
+// E = empty interface
+// [%d%s] = array: repeat signature %s %d times.
+// A%d = known alignment of type pointers (defaults to ptrSize)
+//
+// An alignment directive is only needed on platforms that can't do
+// unaligned loads.
+// If an alignment directive is present, it must be first.
+// Signatures can end early; a K%d is not required at the end.
+func eqSignature(t *types.Type) string {
+ var e eqSigBuilder
+ if !base.Ctxt.Arch.CanMergeLoads { // alignment only matters if we can't use unaligned loads
+ if a := t.Alignment(); a != int64(types.PtrSize) {
+ e.r.WriteString(fmt.Sprintf("%c%d", sigAlign, a))
+ }
+ }
+ e.build(t)
+ e.flush(true)
+ return e.r.String()
}
-func hashmem(t *types.Type) ir.Node {
- return typecheck.LookupRuntime("memhash", t)
+const (
+ sigMemory = 'M' // followed by an integer number of bytes
+ sigSkip = 'K' // followed by an integer number of bytes
+ sigFloat32 = 'F'
+ sigFloat64 = 'G'
+ sigString = 'S'
+ sigIface = 'I' // non-empty interface
+ sigEface = 'E' // empty interface
+ sigArrayStart = '[' // followed by an iteration count, element signature, and sigArrayEnd
+ sigArrayEnd = ']'
+ sigAlign = 'A' // followed by an integer byte alignment
+)
+
+type eqSigBuilder struct {
+ r strings.Builder
+ regMem int64 // queued up region of regular memory
+ skipMem int64 // queued up region of memory to skip
+}
+
+func (e *eqSigBuilder) flush(atEnd bool) {
+ if e.regMem > 0 {
+ e.r.WriteString(fmt.Sprintf("%c%d", sigMemory, e.regMem))
+ e.regMem = 0
+ }
+ if e.skipMem > 0 {
+ if !atEnd {
+ e.r.WriteString(fmt.Sprintf("%c%d", sigSkip, e.skipMem))
+ }
+ e.skipMem = 0
+ }
+}
+func (e *eqSigBuilder) regular(n int64) {
+ if e.regMem == 0 {
+ e.flush(false)
+ }
+ e.regMem += n
+}
+func (e *eqSigBuilder) skip(n int64) {
+ if e.skipMem == 0 {
+ e.flush(false)
+ }
+ e.skipMem += n
+}
+func (e *eqSigBuilder) float32() {
+ e.flush(false)
+ e.r.WriteByte(sigFloat32)
+}
+func (e *eqSigBuilder) float64() {
+ e.flush(false)
+ e.r.WriteByte(sigFloat64)
+}
+func (e *eqSigBuilder) string() {
+ e.flush(false)
+ e.r.WriteByte(sigString)
+}
+func (e *eqSigBuilder) eface() {
+ e.flush(false)
+ e.r.WriteByte(sigEface)
+}
+func (e *eqSigBuilder) iface() {
+ e.flush(false)
+ e.r.WriteByte(sigIface)
+}
+
+func (e *eqSigBuilder) build(t *types.Type) {
+ switch t.Kind() {
+ case types.TINT8, types.TUINT8, types.TBOOL:
+ e.regular(1)
+ case types.TINT16, types.TUINT16:
+ e.regular(2)
+ case types.TINT32, types.TUINT32:
+ e.regular(4)
+ case types.TINT64, types.TUINT64:
+ e.regular(8)
+ case types.TINT, types.TUINT, types.TUINTPTR, types.TPTR, types.TUNSAFEPTR, types.TCHAN:
+ e.regular(int64(types.PtrSize))
+ case types.TFLOAT32:
+ e.float32()
+ case types.TFLOAT64:
+ e.float64()
+ case types.TCOMPLEX64:
+ e.float32()
+ e.float32()
+ case types.TCOMPLEX128:
+ e.float64()
+ e.float64()
+ case types.TSTRING:
+ e.string()
+ case types.TINTER:
+ if t.IsEmptyInterface() {
+ e.eface()
+ } else {
+ e.iface()
+ }
+ case types.TSTRUCT:
+ var off int64
+ for _, f := range t.Fields() {
+ if f.Sym.IsBlank() {
+ continue
+ }
+ if off < f.Offset {
+ e.skip(f.Offset - off)
+ }
+ e.build(f.Type)
+ off = f.Offset + f.Type.Size()
+ }
+ if off < t.Size() {
+ e.skip(t.Size() - off)
+ }
+ case types.TARRAY:
+ if types.AlgType(t) == types.AMEM {
+ // TODO: some "regular equality" types don't hit here,
+ // like [8]sync/atomic.Pointer. Figure out how to
+ // handle the subtle difference between "AMEM" and
+ // "can be compared byte-by-byte for equality".
+ e.regular(t.Size())
+ break
+ }
+ et := t.Elem()
+ n := t.NumElem()
+ switch n {
+ case 0:
+ case 1:
+ e.build(et)
+ default:
+ // To keep signatures small, we can't just repeat
+ // the element signature N times. Instead, we issue
+ // an array into the signature. Note that this can
+ // lead to a situation where two types which could
+ // share an equality function do not, like
+ // struct { a, b, c, d string } sig: SSSS
+ // [4]string sig: [4S]
+ // That's ok, just a tad inefficient.
+ //
+ // The generated loops are kind of inefficient as well,
+ // so unroll the loop a bit.
+ const unrollSize = 32 // make loop body compare around this many bytes
+ if et.Size() == 0 {
+ break // zero-size elements need no comparison
+ }
+ unroll := max(1, unrollSize/et.Size())
+ // Do partial loops directly.
+ for n%unroll != 0 {
+ e.build(et)
+ n--
+ }
+ if n == 0 {
+ break
+ }
+ // If we only have one loop left, do it directly.
+ if n == unroll {
+ for range n {
+ e.build(et)
+ }
+ break
+ }
+ e.flush(false)
+ e.r.WriteString(fmt.Sprintf("%c%d", sigArrayStart, n/unroll))
+ for range unroll {
+ e.build(et)
+ }
+ e.flush(false)
+ e.r.WriteByte(sigArrayEnd)
+ }
+ default:
+ base.Fatalf("eqSigBuilder %v", t)
+ }
+}
+
+// Parse and remove the number at the start of s.
+func parseNum(s string) (int64, string) {
+ n := 0
+ for n < len(s) && s[n] >= '0' && s[n] <= '9' {
+ n++
+ }
+ x, err := strconv.ParseInt(s[:n], 10, 64)
+ if err != nil {
+ base.Fatalf("bad integer: %s", s[:n])
+ }
+ return x, s[n:]
+}
+
+// parseArray parses "%d%s]" from the front of a signature.
+// Returns the repeat count (the %d), the element signature
+// (the %s), and any remaining signature after the closing ']'.
+func parseArray(sig string) (int64, string, string) {
+ var n int64
+ n, sig = parseNum(sig)
+ // Find matching closing brace.
+ i := 0
+ depth := 1
+ for {
+ if i == len(sig) {
+ base.Fatalf("mismatched brackets in %s", sig)
+ }
+ switch sig[i] {
+ case sigArrayStart:
+ depth++
+ case sigArrayEnd:
+ depth--
+ if depth == 0 {
+ return n, sig[:i], sig[i+1:]
+ }
+ }
+ i++
+ }
+}
+
+// sigSize returns the size of the type described by the signature.
+func sigSize(sig string) int64 {
+ var size int64
+ for len(sig) > 0 {
+ kind := sig[0]
+ sig = sig[1:]
+ switch kind {
+ case sigMemory, sigSkip:
+ var n int64
+ n, sig = parseNum(sig)
+ size += n
+ case sigFloat32:
+ size += 4
+ case sigFloat64:
+ size += 8
+ case sigString, sigIface, sigEface:
+ size += 2 * int64(types.PtrSize)
+ case sigArrayStart:
+ var n int64
+ var elemSig string
+ n, elemSig, sig = parseArray(sig)
+ size += n * sigSize(elemSig)
+ }
+ }
+ return size
+}
+
+func sigTrimSkip(s string) string {
+ i := strings.LastIndexByte(s, sigSkip)
+ if i < 0 {
+ return s
+ }
+ for j := i + 1; j < len(s); j++ {
+ if s[j] < '0' || s[j] > '9' {
+ return s
+ }
+ }
+ return s[:i]
}
diff --git a/src/cmd/compile/internal/reflectdata/map.go b/src/cmd/compile/internal/reflectdata/map.go
index 2b43d4af27a250..38eada4522d153 100644
--- a/src/cmd/compile/internal/reflectdata/map.go
+++ b/src/cmd/compile/internal/reflectdata/map.go
@@ -13,6 +13,7 @@ import (
"cmd/internal/objabi"
"cmd/internal/src"
"internal/abi"
+ "internal/buildcfg"
)
// MapGroupType makes the map slot group type given the type of the map.
@@ -26,14 +27,6 @@ func MapGroupType(t *types.Type) *types.Type {
// a correct GC program for it.
//
// Make sure this stays in sync with internal/runtime/maps/group.go.
- //
- // type group struct {
- // ctrl uint64
- // slots [abi.MapGroupSlots]struct {
- // key keyType
- // elem elemType
- // }
- // }
keytype := t.Key()
elemtype := t.Elem()
@@ -46,19 +39,48 @@ func MapGroupType(t *types.Type) *types.Type {
elemtype = types.NewPtr(elemtype)
}
- slotFields := []*types.Field{
- makefield("key", keytype),
- makefield("elem", elemtype),
- }
- slot := types.NewStruct(slotFields)
- slot.SetNoalg(true)
-
- slotArr := types.NewArray(slot, abi.MapGroupSlots)
- slotArr.SetNoalg(true)
-
- fields := []*types.Field{
- makefield("ctrl", types.Types[types.TUINT64]),
- makefield("slots", slotArr),
+ var fields []*types.Field
+ if buildcfg.Experiment.MapSplitGroup {
+ // Split layout (KKKKVVVV):
+ // type group struct {
+ // ctrl uint64
+ // keys [abi.MapGroupSlots]keyType
+ // elems [abi.MapGroupSlots]elemType
+ // }
+ keyArr := types.NewArray(keytype, abi.MapGroupSlots)
+ keyArr.SetNoalg(true)
+
+ elemArr := types.NewArray(elemtype, abi.MapGroupSlots)
+ elemArr.SetNoalg(true)
+
+ fields = []*types.Field{
+ makefield("ctrl", types.Types[types.TUINT64]),
+ makefield("keys", keyArr),
+ makefield("elems", elemArr),
+ }
+ } else {
+ // Interleaved slot layout (KVKVKVKV):
+ // type group struct {
+ // ctrl uint64
+ // slots [abi.MapGroupSlots]struct {
+ // key keyType
+ // elem elemType
+ // }
+ // }
+ slotFields := []*types.Field{
+ makefield("key", keytype),
+ makefield("elem", elemtype),
+ }
+ slot := types.NewStruct(slotFields)
+ slot.SetNoalg(true)
+
+ slotArr := types.NewArray(slot, abi.MapGroupSlots)
+ slotArr.SetNoalg(true)
+
+ fields = []*types.Field{
+ makefield("ctrl", types.Types[types.TUINT64]),
+ makefield("slots", slotArr),
+ }
}
group := types.NewStruct(fields)
@@ -269,13 +291,30 @@ func writeMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) {
s3 := writeType(gtyp)
hasher := genhash(t.Key())
- slotTyp := gtyp.Field(1).Type.Elem()
- elemOff := slotTyp.Field(1).Offset
- if AlgType(t.Key()) == types.AMEM64 && elemOff != 8 {
- base.Fatalf("runtime assumes elemOff for 8-byte keys is 8, got %d", elemOff)
- }
- if AlgType(t.Key()) == types.ASTRING && elemOff != int64(2*types.PtrSize) {
- base.Fatalf("runtime assumes elemOff for string keys is %d, got %d", 2*types.PtrSize, elemOff)
+ var keysOff int64
+ var keyStride int64
+ var elemsOff int64
+ var elemStride int64
+ var elemOff int64
+ if buildcfg.Experiment.MapSplitGroup {
+ // Split layout: field 1 is keys array, field 2 is elems array.
+ keysOff = gtyp.Field(1).Offset
+ keyStride = gtyp.Field(1).Type.Elem().Size()
+ elemsOff = gtyp.Field(2).Offset
+ elemStride = gtyp.Field(2).Type.Elem().Size()
+ } else {
+ // Interleaved layout: field 1 is slots array.
+ // KeysOff = offset of slots array (first key).
+ // KeyStride = ElemStride = slot stride.
+ // ElemsOff = offset of slots + offset of elem within slot.
+ keysOff = gtyp.Field(1).Offset
+ slotTyp := gtyp.Field(1).Type.Elem()
+ slotSize := slotTyp.Size()
+ elemOffInSlot := slotTyp.Field(1).Offset
+ keyStride = slotSize
+ elemsOff = keysOff + elemOffInSlot
+ elemStride = slotSize
+ elemOff = slotTyp.Field(1).Offset
}
c.Field("Key").WritePtr(s1)
@@ -283,7 +322,10 @@ func writeMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) {
c.Field("Group").WritePtr(s3)
c.Field("Hasher").WritePtr(hasher)
c.Field("GroupSize").WriteUintptr(uint64(gtyp.Size()))
- c.Field("SlotSize").WriteUintptr(uint64(slotTyp.Size()))
+ c.Field("KeysOff").WriteUintptr(uint64(keysOff))
+ c.Field("KeyStride").WriteUintptr(uint64(keyStride))
+ c.Field("ElemsOff").WriteUintptr(uint64(elemsOff))
+ c.Field("ElemStride").WriteUintptr(uint64(elemStride))
c.Field("ElemOff").WriteUintptr(uint64(elemOff))
var flags uint32
if needkeyupdate(t.Key()) {
@@ -295,7 +337,7 @@ func writeMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) {
if t.Key().Size() > abi.MapMaxKeyBytes {
flags |= abi.MapIndirectKey
}
- if t.Elem().Size() > abi.MapMaxKeyBytes {
+ if t.Elem().Size() > abi.MapMaxElemBytes {
flags |= abi.MapIndirectElem
}
c.Field("Flags").WriteUint32(flags)
diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go
index 38b9391c5f1a57..1fca65469b17b4 100644
--- a/src/cmd/compile/internal/reflectdata/reflect.go
+++ b/src/cmd/compile/internal/reflectdata/reflect.go
@@ -184,6 +184,7 @@ func dimportpath(p *types.Pkg) {
ot := dnameData(s, 0, p.Path, "", nil, false, false)
objw.Global(s, int32(ot), obj.DUPOK|obj.RODATA)
s.Set(obj.AttrContentAddressable, true)
+ s.Align = 1
p.Pathsym = s
}
@@ -308,6 +309,7 @@ func dname(name, tag string, pkg *types.Pkg, exported, embedded bool) *obj.LSym
ot := dnameData(s, 0, name, tag, pkg, exported, embedded)
objw.Global(s, int32(ot), obj.DUPOK|obj.RODATA)
s.Set(obj.AttrContentAddressable, true)
+ s.Align = 1
return s
}
@@ -756,6 +758,9 @@ func writeType(t *types.Type) *obj.LSym {
// | method list, if any | dextratype
// +--------------------------------+ - E
+ // internal/abi.Type.DescriptorSize is aware of this type layout,
+ // and must be changed if the layout change.
+
// UncommonType section is included if we have a name or a method.
extra := t.Sym() != nil || len(methods(t)) != 0
@@ -960,6 +965,7 @@ func writeType(t *types.Type) *obj.LSym {
keep = false
}
lsym.Set(obj.AttrMakeTypelink, keep)
+ lsym.Align = int16(types.PtrSize)
return lsym
}
@@ -1079,6 +1085,7 @@ func writeITab(lsym *obj.LSym, typ, iface *types.Type, allowNonImplement bool) {
// Nothing writes static itabs, so they are read only.
objw.Global(lsym, int32(rttype.ITab.Size()+delta), int16(obj.DUPOK|obj.RODATA))
lsym.Set(obj.AttrContentAddressable, true)
+ lsym.Align = int16(types.PtrSize)
}
func WritePluginTable() {
@@ -1274,6 +1281,9 @@ func dgcptrmask(t *types.Type, write bool) *obj.LSym {
}
objw.Global(lsym, int32(len(ptrmask)), obj.DUPOK|obj.RODATA|obj.LOCAL)
lsym.Set(obj.AttrContentAddressable, true)
+ // The runtime expects ptrmasks to be aligned
+ // as a uintptr.
+ lsym.Align = int16(types.PtrSize)
}
return lsym
}
@@ -1282,7 +1292,6 @@ func dgcptrmask(t *types.Type, write bool) *obj.LSym {
// word offsets in t that hold pointers.
// ptrmask is assumed to fit at least types.PtrDataSize(t)/PtrSize bits.
func fillptrmask(t *types.Type, ptrmask []byte) {
- clear(ptrmask)
if !t.HasPointers() {
return
}
@@ -1305,8 +1314,8 @@ func dgcptrmaskOnDemand(t *types.Type, write bool) *obj.LSym {
if write && !lsym.OnList() {
// Note: contains a pointer, but a pointer to a
// persistentalloc allocation. Starts with nil.
- objw.Uintptr(lsym, 0, 0)
- objw.Global(lsym, int32(types.PtrSize), obj.DUPOK|obj.NOPTR|obj.LOCAL) // TODO:bss?
+ // Allocated in BSS.
+ objw.Global(lsym, int32(types.PtrSize), obj.DUPOK|obj.NOPTR|obj.LOCAL)
}
return lsym
}
diff --git a/src/cmd/compile/internal/riscv64/ssa.go b/src/cmd/compile/internal/riscv64/ssa.go
index 61de983bb02c00..f01f1bbbcf3cbf 100644
--- a/src/cmd/compile/internal/riscv64/ssa.go
+++ b/src/cmd/compile/internal/riscv64/ssa.go
@@ -294,7 +294,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
ssa.OpRISCV64FADDD, ssa.OpRISCV64FSUBD, ssa.OpRISCV64FMULD, ssa.OpRISCV64FDIVD,
ssa.OpRISCV64FEQD, ssa.OpRISCV64FNED, ssa.OpRISCV64FLTD, ssa.OpRISCV64FLED, ssa.OpRISCV64FSGNJD,
ssa.OpRISCV64MIN, ssa.OpRISCV64MAX, ssa.OpRISCV64MINU, ssa.OpRISCV64MAXU,
- ssa.OpRISCV64SH1ADD, ssa.OpRISCV64SH2ADD, ssa.OpRISCV64SH3ADD:
+ ssa.OpRISCV64SH1ADD, ssa.OpRISCV64SH2ADD, ssa.OpRISCV64SH3ADD,
+ ssa.OpRISCV64CZEROEQZ, ssa.OpRISCV64CZERONEZ:
r := v.Reg()
r1 := v.Args[0].Reg()
r2 := v.Args[1].Reg()
@@ -418,7 +419,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: r3})
p.To.Type = obj.TYPE_REG
p.To.Reg = r
- case ssa.OpRISCV64FSQRTS, ssa.OpRISCV64FNEGS, ssa.OpRISCV64FABSD, ssa.OpRISCV64FSQRTD, ssa.OpRISCV64FNEGD,
+ case ssa.OpRISCV64FSQRTS, ssa.OpRISCV64FSQRTD,
+ ssa.OpRISCV64FNEGS, ssa.OpRISCV64FNEGD,
+ ssa.OpRISCV64FABSS, ssa.OpRISCV64FABSD,
ssa.OpRISCV64FMVSX, ssa.OpRISCV64FMVXS, ssa.OpRISCV64FMVDX, ssa.OpRISCV64FMVXD,
ssa.OpRISCV64FCVTSW, ssa.OpRISCV64FCVTSL, ssa.OpRISCV64FCVTWS, ssa.OpRISCV64FCVTLS,
ssa.OpRISCV64FCVTDW, ssa.OpRISCV64FCVTDL, ssa.OpRISCV64FCVTWD, ssa.OpRISCV64FCVTLD, ssa.OpRISCV64FCVTDS, ssa.OpRISCV64FCVTSD,
@@ -446,6 +449,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.From.Offset = v.AuxInt
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
+ case ssa.OpRISCV64FMOVDconst, ssa.OpRISCV64FMOVFconst:
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_FCONST
+ p.From.Val = v.AuxFloat()
+ p.From.Name = obj.NAME_NONE
+ p.From.Reg = obj.REG_NONE
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = v.Reg()
case ssa.OpRISCV64MOVaddr:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_ADDR
@@ -504,7 +515,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg()
case ssa.OpRISCV64CALLstatic, ssa.OpRISCV64CALLclosure, ssa.OpRISCV64CALLinter:
s.Call(v)
- case ssa.OpRISCV64CALLtail:
+ case ssa.OpRISCV64CALLtail, ssa.OpRISCV64CALLtailinter:
s.TailCall(v)
case ssa.OpRISCV64LoweredWB:
p := s.Prog(obj.ACALL)
@@ -599,13 +610,23 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Sym = ir.Syms.PanicBounds
case ssa.OpRISCV64LoweredAtomicLoad8:
- s.Prog(riscv.AFENCE)
+ p1 := s.Prog(riscv.AFENCE)
+ p1.From.Type = obj.TYPE_SPECIAL
+ p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
+ p1.To.Type = obj.TYPE_SPECIAL
+ p1.To.Offset = int64(riscv.SPOP_FENCE_RW)
+
p := s.Prog(riscv.AMOVBU)
p.From.Type = obj.TYPE_MEM
p.From.Reg = v.Args[0].Reg()
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg0()
- s.Prog(riscv.AFENCE)
+
+ p2 := s.Prog(riscv.AFENCE)
+ p2.From.Type = obj.TYPE_SPECIAL
+ p2.From.Offset = int64(riscv.SPOP_FENCE_R)
+ p2.To.Type = obj.TYPE_SPECIAL
+ p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
case ssa.OpRISCV64LoweredAtomicLoad32, ssa.OpRISCV64LoweredAtomicLoad64:
as := riscv.ALRW
@@ -619,13 +640,23 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg0()
case ssa.OpRISCV64LoweredAtomicStore8:
- s.Prog(riscv.AFENCE)
+ p1 := s.Prog(riscv.AFENCE)
+ p1.From.Type = obj.TYPE_SPECIAL
+ p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
+ p1.To.Type = obj.TYPE_SPECIAL
+ p1.To.Offset = int64(riscv.SPOP_FENCE_W)
+
p := s.Prog(riscv.AMOVB)
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[1].Reg()
p.To.Type = obj.TYPE_MEM
p.To.Reg = v.Args[0].Reg()
- s.Prog(riscv.AFENCE)
+
+ p2 := s.Prog(riscv.AFENCE)
+ p2.From.Type = obj.TYPE_SPECIAL
+ p2.From.Offset = int64(riscv.SPOP_FENCE_RW)
+ p2.To.Type = obj.TYPE_SPECIAL
+ p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
case ssa.OpRISCV64LoweredAtomicStore32, ssa.OpRISCV64LoweredAtomicStore64:
as := riscv.AAMOSWAPW
@@ -741,10 +772,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpRISCV64LoweredZero:
ptr := v.Args[0].Reg()
- sc := v.AuxValAndOff()
- n := sc.Val64()
-
- mov, sz := largestMove(sc.Off64())
+ n, align := v.AuxSizeAndAlign()
+ mov, sz := largestMove(align)
// mov ZERO, (offset)(Rarg0)
var off int64
@@ -766,9 +795,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
case ssa.OpRISCV64LoweredZeroLoop:
ptr := v.Args[0].Reg()
- sc := v.AuxValAndOff()
- n := sc.Val64()
- mov, sz := largestMove(sc.Off64())
+ n, align := v.AuxSizeAndAlign()
+ mov, sz := largestMove(align)
chunk := 8 * sz
if n <= 3*chunk {
@@ -777,9 +805,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
tmp := v.RegTmp()
+ if n >= 1<<31 {
+ p := s.Prog(riscv.AMOV)
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = n - n%chunk
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = tmp
+ }
p := s.Prog(riscv.AADD)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = n - n%chunk
+ if n >= 1<<31 {
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = tmp
+ } else {
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = n - n%chunk
+ }
p.Reg = ptr
p.To.Type = obj.TYPE_REG
p.To.Reg = tmp
@@ -828,9 +868,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
break
}
- sa := v.AuxValAndOff()
- n := sa.Val64()
- mov, sz := largestMove(sa.Off64())
+ n, align := v.AuxSizeAndAlign()
+ mov, sz := largestMove(align)
var off int64
tmp := int16(riscv.REG_X5)
@@ -857,9 +896,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
break
}
- sc := v.AuxValAndOff()
- n := sc.Val64()
- mov, sz := largestMove(sc.Off64())
+ n, align := v.AuxSizeAndAlign()
+ mov, sz := largestMove(align)
chunk := 8 * sz
if n <= 3*chunk {
@@ -867,9 +905,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
}
tmp := int16(riscv.REG_X5)
+ if n >= 1<<31 {
+ p := s.Prog(riscv.AMOV)
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = n - n%chunk
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = riscv.REG_X6
+ }
p := s.Prog(riscv.AADD)
- p.From.Type = obj.TYPE_CONST
- p.From.Offset = n - n%chunk
+ if n >= 1<<31 {
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = riscv.REG_X6
+ } else {
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = n - n%chunk
+ }
p.Reg = src
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X6
@@ -950,8 +1000,12 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg()
case ssa.OpRISCV64LoweredPubBarrier:
- // FENCE
- s.Prog(v.Op.Asm())
+ // FENCE W, W
+ p := s.Prog(v.Op.Asm())
+ p.From.Type = obj.TYPE_SPECIAL
+ p.From.Offset = int64(riscv.SPOP_FENCE_W)
+ p.To.Type = obj.TYPE_SPECIAL
+ p.To.Offset = int64(riscv.SPOP_FENCE_W)
case ssa.OpRISCV64LoweredRound32F, ssa.OpRISCV64LoweredRound64F:
// input is already rounded
diff --git a/src/cmd/compile/internal/rttype/rttype.go b/src/cmd/compile/internal/rttype/rttype.go
index b8c95339915a52..b3fabedffa6348 100644
--- a/src/cmd/compile/internal/rttype/rttype.go
+++ b/src/cmd/compile/internal/rttype/rttype.go
@@ -88,6 +88,24 @@ func Init() {
if got, want := int64(abi.ITabTypeOff(ptrSize)), ITab.OffsetOf("Type"); got != want {
base.Fatalf("abi.ITabTypeOff() == %d, want %d", got, want)
}
+ for _, test := range []struct {
+ kind abi.Kind
+ typ *types.Type
+ name string
+ }{
+ {abi.Struct, StructType, "Struct"},
+ {abi.Pointer, PtrType, "Pointer"},
+ {abi.Func, FuncType, "Func"},
+ {abi.Slice, SliceType, "Slice"},
+ {abi.Array, ArrayType, "Array"},
+ {abi.Chan, ChanType, "Chan"},
+ {abi.Map, MapType, "Map"},
+ {abi.Interface, InterfaceType, "Interface"},
+ } {
+ if got, want := int64(abi.RTypeSize(test.kind, ptrSize)), test.typ.Size(); got != want {
+ base.Fatalf("abi.RTypeSize(%s) == %d, want %d", test.name, got, want)
+ }
+ }
}
// FromReflect translates from a host type to the equivalent target type.
diff --git a/src/cmd/compile/internal/s390x/galign.go b/src/cmd/compile/internal/s390x/galign.go
index d880834c220d91..1fb371a52c6ac2 100644
--- a/src/cmd/compile/internal/s390x/galign.go
+++ b/src/cmd/compile/internal/s390x/galign.go
@@ -20,4 +20,6 @@ func Init(arch *ssagen.ArchInfo) {
arch.SSAMarkMoves = ssaMarkMoves
arch.SSAGenValue = ssaGenValue
arch.SSAGenBlock = ssaGenBlock
+ arch.LoadRegResult = loadRegResult
+ arch.SpillArgReg = spillArgReg
}
diff --git a/src/cmd/compile/internal/s390x/ssa.go b/src/cmd/compile/internal/s390x/ssa.go
index de00f1ef8ce9ee..f34e02f72eb25c 100644
--- a/src/cmd/compile/internal/s390x/ssa.go
+++ b/src/cmd/compile/internal/s390x/ssa.go
@@ -10,6 +10,7 @@ import (
"cmd/compile/internal/base"
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
+ "cmd/compile/internal/objw"
"cmd/compile/internal/ssa"
"cmd/compile/internal/ssagen"
"cmd/compile/internal/types"
@@ -130,7 +131,7 @@ func moveByType(t *types.Type) obj.As {
return s390x.AMOVD
}
}
- panic("bad load type")
+ panic("bad move type")
}
// opregreg emits instructions for
@@ -540,6 +541,19 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[0].Reg()
ssagen.AddrAuto(&p.To, v)
+ case ssa.OpArgIntReg, ssa.OpArgFloatReg:
+ // The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
+ // The loop only runs once.
+ for _, a := range v.Block.Func.RegArgs {
+ // Pass the spill/unspill information along to the assembler, offset by size of
+ // the saved LR slot.
+ addr := ssagen.SpillSlotAddr(a, s390x.REGSP, base.Ctxt.Arch.FixedFrameSize)
+ s.FuncInfo().AddSpill(
+ obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
+ }
+ v.Block.Func.RegArgs = nil
+
+ ssagen.CheckArgReg(v)
case ssa.OpS390XLoweredGetClosurePtr:
// Closure pointer is R12 (already)
ssagen.CheckLoweredGetClosurePtr(v)
@@ -566,7 +580,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Reg = v.Reg()
case ssa.OpS390XCALLstatic, ssa.OpS390XCALLclosure, ssa.OpS390XCALLinter:
s.Call(v)
- case ssa.OpS390XCALLtail:
+ case ssa.OpS390XCALLtail, ssa.OpS390XCALLtailinter:
s.TailCall(v)
case ssa.OpS390XLoweredWB:
p := s.Prog(obj.ACALL)
@@ -1029,3 +1043,22 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
s.Br(s390x.ABR, succs[1])
}
}
+
+func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
+ p := s.Prog(loadByType(t))
+ p.From.Type = obj.TYPE_MEM
+ p.From.Name = obj.NAME_AUTO
+ p.From.Sym = n.Linksym()
+ p.From.Offset = n.FrameOffset() + off
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = reg
+ return p
+}
+
+func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
+ p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
+ p.To.Name = obj.NAME_PARAM
+ p.To.Sym = n.Linksym()
+ p.Pos = p.Pos.WithNotStmt()
+ return p
+}
diff --git a/src/cmd/compile/internal/slice/slice.go b/src/cmd/compile/internal/slice/slice.go
new file mode 100644
index 00000000000000..8669b0ed0cc7a4
--- /dev/null
+++ b/src/cmd/compile/internal/slice/slice.go
@@ -0,0 +1,514 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package slice
+
+// This file implements a stack-allocation optimization
+// for the backing store of slices.
+//
+// Consider the code:
+//
+// var s []int
+// for i := range ... {
+// s = append(s, i)
+// }
+// return s
+//
+// Some of the append operations will need to do an allocation
+// by calling growslice. This will happen on the 1st, 2nd, 4th,
+// 8th, etc. append calls. The allocations done by all but the
+// last growslice call will then immediately be garbage.
+//
+// We'd like to avoid doing some of those intermediate
+// allocations if possible.
+//
+// If we can determine that the "return s" statement is the
+// *only* way that the backing store for s escapes, then we
+// can rewrite the code to something like:
+//
+// var s []int
+// for i := range N {
+// s = append(s, i)
+// }
+// s = move2heap(s)
+// return s
+//
+// Using the move2heap runtime function, which does:
+//
+// move2heap(s):
+// If s is not backed by a stackframe-allocated
+// backing store, return s. Otherwise, copy s
+// to the heap and return the copy.
+//
+// Now we can treat the backing store of s allocated at the
+// append site as not escaping. Previous stack allocation
+// optimizations now apply, which can use a fixed-size
+// stack-allocated backing store for s when appending.
+// (See ../ssagen/ssa.go:(*state).append)
+//
+// It is tricky to do this optimization safely. To describe
+// our analysis, we first define what an "exclusive" slice
+// variable is.
+//
+// A slice variable (a variable of slice type) is called
+// "exclusive" if, when it has a reference to a
+// stackframe-allocated backing store, it is the only
+// variable with such a reference.
+//
+// In other words, a slice variable is exclusive if
+// any of the following holds:
+// 1) It points to a heap-allocated backing store
+// 2) It points to a stack-allocated backing store
+// for any parent frame.
+// 3) It is the only variable that references its
+// backing store.
+// 4) It is nil.
+//
+// The nice thing about exclusive slice variables is that
+// it is always safe to do
+// s = move2heap(s)
+// whenever s is an exclusive slice variable. Because no
+// one else has a reference to the backing store, no one
+// else can tell that we moved the backing store from one
+// location to another.
+//
+// Note that exclusiveness is a dynamic property. A slice
+// variable may be exclusive during some parts of execution
+// and not exclusive during others.
+//
+// The following operations set or preserve the exclusivity
+// of a slice variable s:
+// s = nil
+// s = append(s, ...)
+// s = s[i:j]
+// ... = s[i]
+// s[i] = ...
+// f(s) where f does not escape its argument
+// Other operations destroy exclusivity. A non-exhaustive list includes:
+// x = s
+// *p = s
+// f(s) where f escapes its argument
+// return s
+// To err on the safe side, we white list exclusivity-preserving
+// operations and we asssume that any other operations that mention s
+// destroy its exclusivity.
+//
+// Our strategy is to move the backing store of s to the heap before
+// any exclusive->nonexclusive transition. That way, s will only ever
+// have a reference to a stack backing store while it is exclusive.
+//
+// move2heap for a variable s is implemented with:
+// if s points to within the stack frame {
+// s2 := make([]T, s.len, s.cap)
+// copy(s2[:s.cap], s[:s.cap])
+// s = s2
+// }
+// Note that in general we need to copy all of s[:cap(s)] elements when
+// moving to the heap. As an optimization, we keep track of slice variables
+// whose capacity, and the elements in s[len(s):cap(s)], are never accessed.
+// For those slice variables, we can allocate to the next size class above
+// the length, which saves memory and copying cost.
+
+import (
+ "cmd/compile/internal/base"
+ "cmd/compile/internal/escape"
+ "cmd/compile/internal/ir"
+ "cmd/compile/internal/reflectdata"
+)
+
+func Funcs(all []*ir.Func) {
+ if base.Flag.N != 0 {
+ return
+ }
+ for _, fn := range all {
+ analyze(fn)
+ }
+ for _, fn := range all {
+ if ir.MatchAstDump(fn, "slice") {
+ ir.AstDump(fn, "slice, "+ir.FuncName(fn))
+ }
+ }
+}
+
+func analyze(fn *ir.Func) {
+ type sliceInfo struct {
+ // Slice variable.
+ s *ir.Name
+
+ // Count of uses that this pass understands.
+ okUses int32
+ // Count of all uses found.
+ allUses int32
+
+ // A place where the slice variable transitions from
+ // exclusive to nonexclusive.
+ // We could keep track of more than one, but one is enough for now.
+ // Currently, this can be either a return statement or
+ // an assignment.
+ // TODO: other possible transitions?
+ transition ir.Stmt
+
+ // Each s = append(s, ...) instance we found.
+ appends []*ir.CallExpr
+
+ // Weight of the number of s = append(s, ...) instances we found.
+ // The optimizations we do are only really useful if there are at
+ // least weight 2. (Note: appends in loops have weight >= 2.)
+ appendWeight int
+
+ // Loop depth at declaration point.
+ // Use for heuristics only, it is not guaranteed to be correct
+ // in the presence of gotos.
+ declDepth int
+
+ // Whether we ever do cap(s), or other operations that use cap(s)
+ // (possibly implicitly), like s[i:j].
+ capUsed bool
+ }
+
+ // Every variable (*ir.Name) that we are tracking will have
+ // a non-nil *sliceInfo in its Opt field.
+ haveLocalSlice := false
+ maxStackSize := int64(base.Debug.VariableMakeThreshold)
+ var namedRets []*ir.Name
+ for _, s := range fn.Dcl {
+ if !s.Type().IsSlice() {
+ continue
+ }
+ if s.Type().Elem().Size() > maxStackSize {
+ continue
+ }
+ if !base.VariableMakeHash.MatchPos(s.Pos(), nil) {
+ continue
+ }
+ s.Opt = &sliceInfo{s: s} // start tracking s
+ haveLocalSlice = true
+ if s.Class == ir.PPARAMOUT {
+ namedRets = append(namedRets, s)
+ }
+ }
+ if !haveLocalSlice {
+ return
+ }
+
+ // Keep track of loop depth while walking.
+ loopDepth := 0
+
+ // tracking returns the info for the slice variable if n is a slice
+ // variable that we're still considering, or nil otherwise.
+ tracking := func(n ir.Node) *sliceInfo {
+ if n == nil || n.Op() != ir.ONAME {
+ return nil
+ }
+ s := n.(*ir.Name)
+ if s.Opt == nil {
+ return nil
+ }
+ return s.Opt.(*sliceInfo)
+ }
+
+ // addTransition(n, loc) records that s experiences an exclusive->nonexclusive
+ // transition somewhere within loc.
+ addTransition := func(i *sliceInfo, loc ir.Stmt) {
+ if i.transition != nil {
+ // We only keep track of a single exclusive->nonexclusive transition
+ // for a slice variable. If we find more than one, give up.
+ // (More than one transition location would be fine, but we would
+ // start to get worried about introducing too much additional code.)
+ i.s.Opt = nil
+ return
+ }
+ if loopDepth > i.declDepth {
+ // Conservatively, we disable this optimization when the
+ // transition is inside a loop. This can result in adding
+ // overhead unnecessarily in cases like:
+ // func f(n int, p *[]byte) {
+ // var s []byte
+ // for i := range n {
+ // *p = s
+ // s = append(s, 0)
+ // }
+ // }
+ i.s.Opt = nil
+ return
+ }
+ i.transition = loc
+ }
+
+ // Examine an x = y assignment that occurs somewhere within statement stmt.
+ assign := func(x, y ir.Node, stmt ir.Stmt) {
+ if i := tracking(x); i != nil {
+ // s = y. Check for understood patterns for y.
+ if y == nil || y.Op() == ir.ONIL {
+ // s = nil is ok.
+ i.okUses++
+ } else if y.Op() == ir.OSLICELIT {
+ // s = []{...} is ok.
+ // Note: this reveals capacity. Should it?
+ i.okUses++
+ i.capUsed = true
+ } else if y.Op() == ir.OSLICE {
+ y := y.(*ir.SliceExpr)
+ if y.X == i.s {
+ // s = s[...:...] is ok
+ i.okUses += 2
+ i.capUsed = true
+ }
+ } else if y.Op() == ir.OAPPEND {
+ y := y.(*ir.CallExpr)
+ if y.Args[0] == i.s {
+ // s = append(s, ...) is ok
+ i.okUses += 2
+ i.appends = append(i.appends, y)
+ i.appendWeight += 1 + (loopDepth - i.declDepth)
+ }
+ // TODO: s = append(nil, ...)?
+ }
+ // Note that technically s = make([]T, ...) preserves exclusivity, but
+ // we don't track that because we assume users who wrote that know
+ // better than the compiler does.
+
+ // TODO: figure out how to handle s = fn(..., s, ...)
+ // It would be nice to maintain exclusivity of s in this situation.
+ // But unfortunately, fn can return one of its other arguments, which
+ // may be a slice with a stack-allocated backing store other than s.
+ // (which may have preexisting references to its backing store).
+ //
+ // Maybe we could do it if s is the only argument?
+ }
+
+ if i := tracking(y); i != nil {
+ // ... = s
+ // Treat this as an exclusive->nonexclusive transition.
+ i.okUses++
+ addTransition(i, stmt)
+ }
+ }
+
+ // do walks n and everything below it, recording what happens to the
+ // slice variables we are tracking. It counts every mention of such a
+ // variable in allUses, and the subset of those mentions that this pass
+ // understands to preserve exclusivity in okUses. A variable whose two
+ // counts end up equal is only ever used in ways we understand; any
+ // other variable is dropped at the end of the analysis. Uses that
+ // definitely destroy exclusivity, like &s[i], stop the tracking right
+ // away by clearing s.Opt, which makes tracking report the variable as
+ // no longer being considered.
+ //
+ // It is always used as an ir.DoChildren visitor and always returns
+ // false, so that the whole function body is walked.
+ var do func(ir.Node) bool
+ do = func(n ir.Node) bool {
+ if n == nil {
+ return false
+ }
+ switch n.Op() {
+ case ir.ONAME:
+ if i := tracking(n); i != nil {
+ // A use of a slice variable. Count it.
+ i.allUses++
+ }
+ case ir.ODCL:
+ n := n.(*ir.Decl)
+ if i := tracking(n.X); i != nil {
+ i.okUses++
+ i.declDepth = loopDepth
+ }
+ case ir.OINDEX:
+ n := n.(*ir.IndexExpr)
+ if i := tracking(n.X); i != nil {
+ // s[i] is ok.
+ i.okUses++
+ }
+ case ir.OLEN:
+ n := n.(*ir.UnaryExpr)
+ if i := tracking(n.X); i != nil {
+ // len(s) is ok
+ i.okUses++
+ }
+ case ir.OCAP:
+ n := n.(*ir.UnaryExpr)
+ if i := tracking(n.X); i != nil {
+ // cap(s) is ok
+ i.okUses++
+ i.capUsed = true
+ }
+ case ir.OADDR:
+ n := n.(*ir.AddrExpr)
+ // Walk down to the object whose interior we're taking the
+ // address of. Field selectors and array indexes don't leave
+ // that object, so &s[i], &s[i].f, and &s[i].f[j] all end up
+ // pointing into s's backing store.
+ // (We need this because s[i] is ok, but &s[i] is not.)
+ x := n.X
+ for x != nil {
+ switch x.Op() {
+ case ir.ODOT:
+ // &x.f points into x.
+ x = x.(*ir.SelectorExpr).X
+ continue
+ case ir.OINDEX:
+ idx := x.(*ir.IndexExpr)
+ if idx.X.Type().IsArray() {
+ // &a[i] points into a.
+ // Note: for a pointer to an array, or for a
+ // slice, the address points into a different
+ // object instead, so we stop here.
+ x = idx.X
+ continue
+ }
+ if i := tracking(idx.X); i != nil {
+ // &s[i] is definitely a nonexclusive transition.
+ i.s.Opt = nil
+ }
+ }
+ break
+ }
+ case ir.ORETURN:
+ n := n.(*ir.ReturnStmt)
+ for _, x := range n.Results {
+ if i := tracking(x); i != nil {
+ i.okUses++
+ // We go exclusive->nonexclusive here
+ addTransition(i, n)
+ }
+ }
+ if len(n.Results) == 0 {
+ // Uses of named result variables are implicit here.
+ for _, x := range namedRets {
+ if i := tracking(x); i != nil {
+ addTransition(i, n)
+ }
+ }
+ }
+ case ir.OCALLFUNC:
+ n := n.(*ir.CallExpr)
+ for idx, arg := range n.Args {
+ if i := tracking(arg); i != nil {
+ if !argLeak(n, idx) {
+ // Passing s to a nonescaping arg is ok.
+ i.okUses++
+ i.capUsed = true
+ }
+ }
+ }
+ case ir.ORANGE:
+ n := n.(*ir.RangeStmt)
+ if i := tracking(n.X); i != nil {
+ i.okUses++
+ // Range over slice keeps a pointer to the backing store, see #79909.
+ addTransition(i, n)
+ }
+ case ir.OAS:
+ n := n.(*ir.AssignStmt)
+ assign(n.X, n.Y, n)
+ case ir.OAS2:
+ n := n.(*ir.AssignListStmt)
+ for i := range len(n.Lhs) {
+ assign(n.Lhs[i], n.Rhs[i], n)
+ }
+ case ir.OCLOSURE:
+ n := n.(*ir.ClosureExpr)
+ for _, v := range n.Func.ClosureVars {
+ do(v.Outer)
+ }
+ }
+ if n.Op() == ir.OFOR || n.Op() == ir.ORANGE {
+ // Note: loopDepth isn't really right for init portion
+ // of the for statement, but that's ok. Correctness
+ // does not depend on depth info.
+ loopDepth++
+ defer func() { loopDepth-- }()
+ }
+ // Check all the children.
+ ir.DoChildren(n, do)
+ return false
+ }
+
+ // Run the analysis over the whole body.
+ for _, stmt := range fn.Body {
+ do(stmt)
+ }
+
+ // Process accumulated info to find slice variables
+ // that we can allocate on the stack.
+ for _, s := range fn.Dcl {
+ if s.Opt == nil {
+ continue
+ }
+ i := s.Opt.(*sliceInfo)
+ s.Opt = nil
+ if i.okUses != i.allUses {
+ // Some use of i.s that don't understand lurks. Give up.
+ continue
+ }
+
+ // At this point, we've decided that we *can* do
+ // the optimization.
+
+ if i.transition == nil {
+ // Exclusive for its whole lifetime. That means it
+ // didn't escape. We can already handle nonescaping
+ // slices without this pass.
+ continue
+ }
+ if i.appendWeight < 2 {
+ // This optimization only really helps if there is
+ // (dynamically) more than one append.
+ continue
+ }
+
+ // Commit point - at this point we've decided we *should*
+ // do the optimization.
+
+ // Insert a move2heap operation before the exclusive->nonexclusive
+ // transition.
+ move := ir.NewMoveToHeapExpr(i.transition.Pos(), i.s)
+ if i.capUsed {
+ move.PreserveCapacity = true
+ }
+ move.RType = reflectdata.AppendElemRType(i.transition.Pos(), i.appends[0])
+ move.SetType(i.s.Type())
+ move.SetTypecheck(1)
+ as := ir.NewAssignStmt(i.transition.Pos(), i.s, move)
+ as.SetTypecheck(1)
+ i.transition.PtrInit().Prepend(as)
+ // Note: we prepend because we need to put the move2heap
+ // operation first, before any other init work, as the transition
+ // might occur in the init work.
+
+ // Now that we've inserted a move2heap operation before every
+ // exclusive -> nonexclusive transition, appends can now use
+ // stack backing stores.
+ // (This is the whole point of this pass, to enable stack
+ // allocation of append backing stores.)
+ for _, a := range i.appends {
+ a.SetEsc(ir.EscNone)
+ if i.capUsed {
+ a.UseBuf = true
+ }
+ }
+ }
+}
+
+// argLeak reports if the idx'th argument to the call n escapes anywhere
+// (to the heap, another argument, return value, etc.)
+// If unknown returns true.
+func argLeak(n *ir.CallExpr, idx int) bool {
+ if n.Op() != ir.OCALLFUNC {
+ return true
+ }
+ fn := ir.StaticCalleeName(ir.StaticValue(n.Fun))
+ if fn == nil {
+ return true
+ }
+ fntype := fn.Type()
+ if recv := fntype.Recv(); recv != nil {
+ if idx == 0 {
+ return escape.ParseLeaks(recv.Note).Any()
+ }
+ idx--
+ }
+ return escape.ParseLeaks(fntype.Params()[idx].Note).Any()
+}
diff --git a/src/cmd/compile/internal/ssa/_gen/386.rules b/src/cmd/compile/internal/ssa/_gen/386.rules
index 5f1150241929eb..b7134e7cefd229 100644
--- a/src/cmd/compile/internal/ssa/_gen/386.rules
+++ b/src/cmd/compile/internal/ssa/_gen/386.rules
@@ -7,6 +7,7 @@
(Add(32|64)F ...) => (ADDS(S|D) ...)
(Add32carry ...) => (ADDLcarry ...)
(Add32withcarry ...) => (ADCL ...)
+(Add32carrywithcarry ...) => (ADCLcarry ...)
(Sub(Ptr|32|16|8) ...) => (SUBL ...)
(Sub(32|64)F ...) => (SUBS(S|D) ...)
@@ -325,6 +326,7 @@
(ClosureCall ...) => (CALLclosure ...)
(InterCall ...) => (CALLinter ...)
(TailCall ...) => (CALLtail ...)
+(TailCallInter ...) => (CALLtailinter ...)
// Miscellaneous
(IsNonNil p) => (SETNE (TESTL p p))
diff --git a/src/cmd/compile/internal/ssa/_gen/386Ops.go b/src/cmd/compile/internal/ssa/_gen/386Ops.go
index 60599a33abb587..791f93d21470e4 100644
--- a/src/cmd/compile/internal/ssa/_gen/386Ops.go
+++ b/src/cmd/compile/internal/ssa/_gen/386Ops.go
@@ -59,10 +59,10 @@ func init() {
num[name] = i
}
buildReg := func(s string) regMask {
- m := regMask(0)
+ m := regMask{}
for _, r := range strings.Split(s, " ") {
if n, ok := num[r]; ok {
- m |= regMask(1) << uint(n)
+ m = m.addReg(uint(n))
continue
}
panic("register " + r + " not found")
@@ -78,9 +78,9 @@ func init() {
bx = buildReg("BX")
gp = buildReg("AX CX DX BX BP SI DI")
fp = buildReg("X0 X1 X2 X3 X4 X5 X6 X7")
- gpsp = gp | buildReg("SP")
- gpspsb = gpsp | buildReg("SB")
- callerSave = gp | fp
+ gpsp = gp.union(buildReg("SP"))
+ gpspsb = gpsp.union(buildReg("SB"))
+ callerSave = gp.union(fp)
)
// Common slices of register masks
var (
@@ -90,121 +90,123 @@ func init() {
// Common regInfo
var (
- gp01 = regInfo{inputs: nil, outputs: gponly}
- gp11 = regInfo{inputs: []regMask{gp}, outputs: gponly}
- gp11sp = regInfo{inputs: []regMask{gpsp}, outputs: gponly}
- gp11sb = regInfo{inputs: []regMask{gpspsb}, outputs: gponly}
- gp21 = regInfo{inputs: []regMask{gp, gp}, outputs: gponly}
- gp11carry = regInfo{inputs: []regMask{gp}, outputs: []regMask{gp, 0}}
- gp21carry = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp, 0}}
- gp1carry1 = regInfo{inputs: []regMask{gp}, outputs: gponly}
- gp2carry1 = regInfo{inputs: []regMask{gp, gp}, outputs: gponly}
- gp21sp = regInfo{inputs: []regMask{gpsp, gp}, outputs: gponly}
- gp21sb = regInfo{inputs: []regMask{gpspsb, gpsp}, outputs: gponly}
- gp21shift = regInfo{inputs: []regMask{gp, cx}, outputs: []regMask{gp}}
- gp11div = regInfo{inputs: []regMask{ax, gpsp &^ dx}, outputs: []regMask{ax}, clobbers: dx}
- gp21hmul = regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{dx}, clobbers: ax}
- gp11mod = regInfo{inputs: []regMask{ax, gpsp &^ dx}, outputs: []regMask{dx}, clobbers: ax}
- gp21mul = regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{dx, ax}}
+ gp01 = regInfo{inputs: nil, outputs: gponly}
+ gp11 = regInfo{inputs: []regMask{gp}, outputs: gponly}
+ gp11sp = regInfo{inputs: []regMask{gpsp}, outputs: gponly}
+ gp11sb = regInfo{inputs: []regMask{gpspsb}, outputs: gponly}
+ gp21 = regInfo{inputs: []regMask{gp, gp}, outputs: gponly}
+ gp11carry = regInfo{inputs: []regMask{gp}, outputs: []regMask{gp, regMask{}}}
+ gp21carry = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp, regMask{}}}
+ gp1carry1 = regInfo{inputs: []regMask{gp}, outputs: gponly}
+ gp2carry1 = regInfo{inputs: []regMask{gp, gp}, outputs: gponly}
+ gp2carry1carry = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp, regMask{}}}
+ gp21sp = regInfo{inputs: []regMask{gpsp, gp}, outputs: gponly}
+ gp21sb = regInfo{inputs: []regMask{gpspsb, gpsp}, outputs: gponly}
+ gp21shift = regInfo{inputs: []regMask{gp, cx}, outputs: []regMask{gp}}
+ gp11div = regInfo{inputs: []regMask{ax, gpsp.minus(dx)}, outputs: []regMask{ax}, clobbers: dx}
+ gp21hmul = regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{dx}, clobbers: ax}
+ gp11mod = regInfo{inputs: []regMask{ax, gpsp.minus(dx)}, outputs: []regMask{dx}, clobbers: ax}
+ gp21mul = regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{dx, ax}}
gp2flags = regInfo{inputs: []regMask{gpsp, gpsp}}
gp1flags = regInfo{inputs: []regMask{gpsp}}
- gp0flagsLoad = regInfo{inputs: []regMask{gpspsb, 0}}
- gp1flagsLoad = regInfo{inputs: []regMask{gpspsb, gpsp, 0}}
+ gp0flagsLoad = regInfo{inputs: []regMask{gpspsb, regMask{}}}
+ gp1flagsLoad = regInfo{inputs: []regMask{gpspsb, gpsp, regMask{}}}
flagsgp = regInfo{inputs: nil, outputs: gponly}
readflags = regInfo{inputs: nil, outputs: gponly}
- flagsgpax = regInfo{inputs: nil, clobbers: ax, outputs: []regMask{gp &^ ax}}
+ flagsgpax = regInfo{inputs: nil, clobbers: ax, outputs: []regMask{gp.minus(ax)}}
- gpload = regInfo{inputs: []regMask{gpspsb, 0}, outputs: gponly}
- gp21load = regInfo{inputs: []regMask{gp, gpspsb, 0}, outputs: gponly}
- gploadidx = regInfo{inputs: []regMask{gpspsb, gpsp, 0}, outputs: gponly}
- gp21loadidx = regInfo{inputs: []regMask{gp, gpspsb, gpsp, 0}, outputs: gponly}
+ gpload = regInfo{inputs: []regMask{gpspsb, regMask{}}, outputs: gponly}
+ gp21load = regInfo{inputs: []regMask{gp, gpspsb, regMask{}}, outputs: gponly}
+ gploadidx = regInfo{inputs: []regMask{gpspsb, gpsp, regMask{}}, outputs: gponly}
+ gp21loadidx = regInfo{inputs: []regMask{gp, gpspsb, gpsp, regMask{}}, outputs: gponly}
- gpstore = regInfo{inputs: []regMask{gpspsb, gpsp, 0}}
- gpstoreconst = regInfo{inputs: []regMask{gpspsb, 0}}
- gpstoreidx = regInfo{inputs: []regMask{gpspsb, gpsp, gpsp, 0}}
- gpstoreconstidx = regInfo{inputs: []regMask{gpspsb, gpsp, 0}}
+ gpstore = regInfo{inputs: []regMask{gpspsb, gpsp, regMask{}}}
+ gpstoreconst = regInfo{inputs: []regMask{gpspsb, regMask{}}}
+ gpstoreidx = regInfo{inputs: []regMask{gpspsb, gpsp, gpsp, regMask{}}}
+ gpstoreconstidx = regInfo{inputs: []regMask{gpspsb, gpsp, regMask{}}}
fp01 = regInfo{inputs: nil, outputs: fponly}
fp21 = regInfo{inputs: []regMask{fp, fp}, outputs: fponly}
- fp21load = regInfo{inputs: []regMask{fp, gpspsb, 0}, outputs: fponly}
+ fp21load = regInfo{inputs: []regMask{fp, gpspsb, regMask{}}, outputs: fponly}
fpgp = regInfo{inputs: fponly, outputs: gponly}
gpfp = regInfo{inputs: gponly, outputs: fponly}
fp11 = regInfo{inputs: fponly, outputs: fponly}
fp2flags = regInfo{inputs: []regMask{fp, fp}}
- fpload = regInfo{inputs: []regMask{gpspsb, 0}, outputs: fponly}
- fploadidx = regInfo{inputs: []regMask{gpspsb, gpsp, 0}, outputs: fponly}
+ fpload = regInfo{inputs: []regMask{gpspsb, regMask{}}, outputs: fponly}
+ fploadidx = regInfo{inputs: []regMask{gpspsb, gpsp, regMask{}}, outputs: fponly}
- fpstore = regInfo{inputs: []regMask{gpspsb, fp, 0}}
- fpstoreidx = regInfo{inputs: []regMask{gpspsb, gpsp, fp, 0}}
+ fpstore = regInfo{inputs: []regMask{gpspsb, fp, regMask{}}}
+ fpstoreidx = regInfo{inputs: []regMask{gpspsb, gpsp, fp, regMask{}}}
)
var _386ops = []opData{
// fp ops
- {name: "ADDSS", argLength: 2, reg: fp21, asm: "ADDSS", commutative: true, resultInArg0: true}, // fp32 add
- {name: "ADDSD", argLength: 2, reg: fp21, asm: "ADDSD", commutative: true, resultInArg0: true}, // fp64 add
- {name: "SUBSS", argLength: 2, reg: fp21, asm: "SUBSS", resultInArg0: true}, // fp32 sub
- {name: "SUBSD", argLength: 2, reg: fp21, asm: "SUBSD", resultInArg0: true}, // fp64 sub
- {name: "MULSS", argLength: 2, reg: fp21, asm: "MULSS", commutative: true, resultInArg0: true}, // fp32 mul
- {name: "MULSD", argLength: 2, reg: fp21, asm: "MULSD", commutative: true, resultInArg0: true}, // fp64 mul
- {name: "DIVSS", argLength: 2, reg: fp21, asm: "DIVSS", resultInArg0: true}, // fp32 div
- {name: "DIVSD", argLength: 2, reg: fp21, asm: "DIVSD", resultInArg0: true}, // fp64 div
-
- {name: "MOVSSload", argLength: 2, reg: fpload, asm: "MOVSS", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read"}, // fp32 load
- {name: "MOVSDload", argLength: 2, reg: fpload, asm: "MOVSD", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read"}, // fp64 load
- {name: "MOVSSconst", reg: fp01, asm: "MOVSS", aux: "Float32", rematerializeable: true}, // fp32 constant
- {name: "MOVSDconst", reg: fp01, asm: "MOVSD", aux: "Float64", rematerializeable: true}, // fp64 constant
- {name: "MOVSSloadidx1", argLength: 3, reg: fploadidx, asm: "MOVSS", aux: "SymOff", symEffect: "Read"}, // fp32 load indexed by i
- {name: "MOVSSloadidx4", argLength: 3, reg: fploadidx, asm: "MOVSS", aux: "SymOff", symEffect: "Read"}, // fp32 load indexed by 4*i
- {name: "MOVSDloadidx1", argLength: 3, reg: fploadidx, asm: "MOVSD", aux: "SymOff", symEffect: "Read"}, // fp64 load indexed by i
- {name: "MOVSDloadidx8", argLength: 3, reg: fploadidx, asm: "MOVSD", aux: "SymOff", symEffect: "Read"}, // fp64 load indexed by 8*i
-
- {name: "MOVSSstore", argLength: 3, reg: fpstore, asm: "MOVSS", aux: "SymOff", faultOnNilArg0: true, symEffect: "Write"}, // fp32 store
- {name: "MOVSDstore", argLength: 3, reg: fpstore, asm: "MOVSD", aux: "SymOff", faultOnNilArg0: true, symEffect: "Write"}, // fp64 store
- {name: "MOVSSstoreidx1", argLength: 4, reg: fpstoreidx, asm: "MOVSS", aux: "SymOff", symEffect: "Write"}, // fp32 indexed by i store
- {name: "MOVSSstoreidx4", argLength: 4, reg: fpstoreidx, asm: "MOVSS", aux: "SymOff", symEffect: "Write"}, // fp32 indexed by 4i store
- {name: "MOVSDstoreidx1", argLength: 4, reg: fpstoreidx, asm: "MOVSD", aux: "SymOff", symEffect: "Write"}, // fp64 indexed by i store
- {name: "MOVSDstoreidx8", argLength: 4, reg: fpstoreidx, asm: "MOVSD", aux: "SymOff", symEffect: "Write"}, // fp64 indexed by 8i store
-
- {name: "ADDSSload", argLength: 3, reg: fp21load, asm: "ADDSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp32 arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "ADDSDload", argLength: 3, reg: fp21load, asm: "ADDSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp64 arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "SUBSSload", argLength: 3, reg: fp21load, asm: "SUBSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp32 arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "SUBSDload", argLength: 3, reg: fp21load, asm: "SUBSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp64 arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "MULSSload", argLength: 3, reg: fp21load, asm: "MULSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp32 arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "MULSDload", argLength: 3, reg: fp21load, asm: "MULSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp64 arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "DIVSSload", argLength: 3, reg: fp21load, asm: "DIVSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp32 arg0 / tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "DIVSDload", argLength: 3, reg: fp21load, asm: "DIVSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read"}, // fp64 arg0 / tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "ADDSS", argLength: 2, reg: fp21, asm: "ADDSS", commutative: true, resultInArg0: true, earlyOk: true}, // fp32 add
+ {name: "ADDSD", argLength: 2, reg: fp21, asm: "ADDSD", commutative: true, resultInArg0: true, earlyOk: true}, // fp64 add
+ {name: "SUBSS", argLength: 2, reg: fp21, asm: "SUBSS", resultInArg0: true, earlyOk: true}, // fp32 sub
+ {name: "SUBSD", argLength: 2, reg: fp21, asm: "SUBSD", resultInArg0: true, earlyOk: true}, // fp64 sub
+ {name: "MULSS", argLength: 2, reg: fp21, asm: "MULSS", commutative: true, resultInArg0: true, earlyOk: true}, // fp32 mul
+ {name: "MULSD", argLength: 2, reg: fp21, asm: "MULSD", commutative: true, resultInArg0: true, earlyOk: true}, // fp64 mul
+ {name: "DIVSS", argLength: 2, reg: fp21, asm: "DIVSS", resultInArg0: true, earlyOk: true}, // fp32 div
+ {name: "DIVSD", argLength: 2, reg: fp21, asm: "DIVSD", resultInArg0: true, earlyOk: true}, // fp64 div
+
+ {name: "MOVSSload", argLength: 2, reg: fpload, asm: "MOVSS", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // fp32 load
+ {name: "MOVSDload", argLength: 2, reg: fpload, asm: "MOVSD", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // fp64 load
+ {name: "MOVSSconst", reg: fp01, asm: "MOVSS", aux: "Float32", rematerializeable: true, earlyOk: true}, // fp32 constant
+ {name: "MOVSDconst", reg: fp01, asm: "MOVSD", aux: "Float64", rematerializeable: true, earlyOk: true}, // fp64 constant
+ {name: "MOVSSloadidx1", argLength: 3, reg: fploadidx, asm: "MOVSS", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // fp32 load indexed by i
+ {name: "MOVSSloadidx4", argLength: 3, reg: fploadidx, asm: "MOVSS", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // fp32 load indexed by 4*i
+ {name: "MOVSDloadidx1", argLength: 3, reg: fploadidx, asm: "MOVSD", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // fp64 load indexed by i
+ {name: "MOVSDloadidx8", argLength: 3, reg: fploadidx, asm: "MOVSD", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // fp64 load indexed by 8*i
+
+ {name: "MOVSSstore", argLength: 3, reg: fpstore, asm: "MOVSS", aux: "SymOff", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // fp32 store
+ {name: "MOVSDstore", argLength: 3, reg: fpstore, asm: "MOVSD", aux: "SymOff", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // fp64 store
+ {name: "MOVSSstoreidx1", argLength: 4, reg: fpstoreidx, asm: "MOVSS", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // fp32 indexed by i store
+ {name: "MOVSSstoreidx4", argLength: 4, reg: fpstoreidx, asm: "MOVSS", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // fp32 indexed by 4i store
+ {name: "MOVSDstoreidx1", argLength: 4, reg: fpstoreidx, asm: "MOVSD", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // fp64 indexed by i store
+ {name: "MOVSDstoreidx8", argLength: 4, reg: fpstoreidx, asm: "MOVSD", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // fp64 indexed by 8i store
+
+ {name: "ADDSSload", argLength: 3, reg: fp21load, asm: "ADDSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp32 arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "ADDSDload", argLength: 3, reg: fp21load, asm: "ADDSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp64 arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "SUBSSload", argLength: 3, reg: fp21load, asm: "SUBSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp32 arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "SUBSDload", argLength: 3, reg: fp21load, asm: "SUBSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp64 arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "MULSSload", argLength: 3, reg: fp21load, asm: "MULSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp32 arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "MULSDload", argLength: 3, reg: fp21load, asm: "MULSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp64 arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "DIVSSload", argLength: 3, reg: fp21load, asm: "DIVSS", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp32 arg0 / tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "DIVSDload", argLength: 3, reg: fp21load, asm: "DIVSD", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // fp64 arg0 / tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
// binary ops
- {name: "ADDL", argLength: 2, reg: gp21sp, asm: "ADDL", commutative: true, clobberFlags: true}, // arg0 + arg1
- {name: "ADDLconst", argLength: 1, reg: gp11sp, asm: "ADDL", aux: "Int32", typ: "UInt32", clobberFlags: true}, // arg0 + auxint
+ {name: "ADDL", argLength: 2, reg: gp21sp, asm: "ADDL", commutative: true, clobberFlags: true, earlyOk: true}, // arg0 + arg1
+ {name: "ADDLconst", argLength: 1, reg: gp11sp, asm: "ADDL", aux: "Int32", typ: "UInt32", clobberFlags: true, earlyOk: true}, // arg0 + auxint
- {name: "ADDLcarry", argLength: 2, reg: gp21carry, asm: "ADDL", commutative: true, resultInArg0: true}, // arg0 + arg1, generates pair
- {name: "ADDLconstcarry", argLength: 1, reg: gp11carry, asm: "ADDL", aux: "Int32", resultInArg0: true}, // arg0 + auxint, generates pair
- {name: "ADCL", argLength: 3, reg: gp2carry1, asm: "ADCL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0+arg1+carry(arg2), where arg2 is flags
- {name: "ADCLconst", argLength: 2, reg: gp1carry1, asm: "ADCL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0+auxint+carry(arg1), where arg1 is flags
+ {name: "ADDLcarry", argLength: 2, reg: gp21carry, asm: "ADDL", commutative: true, resultInArg0: true, earlyOk: true}, // arg0 + arg1, generates pair
+ {name: "ADDLconstcarry", argLength: 1, reg: gp11carry, asm: "ADDL", aux: "Int32", resultInArg0: true}, // arg0 + auxint, generates pair
+ {name: "ADCL", argLength: 3, reg: gp2carry1, asm: "ADCL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0+arg1+carry(arg2), where arg2 is flags
+ {name: "ADCLcarry", argLength: 3, reg: gp2carry1carry, asm: "ADCL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0+arg1+carry(arg2), where arg2 is flags, generates pair
+ {name: "ADCLconst", argLength: 2, reg: gp1carry1, asm: "ADCL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0+auxint+carry(arg1), where arg1 is flags
- {name: "SUBL", argLength: 2, reg: gp21, asm: "SUBL", resultInArg0: true, clobberFlags: true}, // arg0 - arg1
- {name: "SUBLconst", argLength: 1, reg: gp11, asm: "SUBL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 - auxint
+ {name: "SUBL", argLength: 2, reg: gp21, asm: "SUBL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 - arg1
+ {name: "SUBLconst", argLength: 1, reg: gp11, asm: "SUBL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 - auxint
{name: "SUBLcarry", argLength: 2, reg: gp21carry, asm: "SUBL", resultInArg0: true}, // arg0-arg1, generates pair
{name: "SUBLconstcarry", argLength: 1, reg: gp11carry, asm: "SUBL", aux: "Int32", resultInArg0: true}, // arg0-auxint, generates pair
{name: "SBBL", argLength: 3, reg: gp2carry1, asm: "SBBL", resultInArg0: true, clobberFlags: true}, // arg0-arg1-borrow(arg2), where arg2 is flags
{name: "SBBLconst", argLength: 2, reg: gp1carry1, asm: "SBBL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0-auxint-borrow(arg1), where arg1 is flags
- {name: "MULL", argLength: 2, reg: gp21, asm: "IMULL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0 * arg1
- {name: "MULLconst", argLength: 1, reg: gp11, asm: "IMUL3L", aux: "Int32", clobberFlags: true}, // arg0 * auxint
+ {name: "MULL", argLength: 2, reg: gp21, asm: "IMULL", commutative: true, resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 * arg1
+ {name: "MULLconst", argLength: 1, reg: gp11, asm: "IMUL3L", aux: "Int32", clobberFlags: true, earlyOk: true}, // arg0 * auxint
- {name: "MULLU", argLength: 2, reg: regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{ax, 0}, clobbers: dx}, typ: "(UInt32,Flags)", asm: "MULL", commutative: true, clobberFlags: true}, // Let x = arg0*arg1 (full 32x32->64 unsigned multiply). Returns uint32(x), and flags set to overflow if uint32(x) != x.
+ {name: "MULLU", argLength: 2, reg: regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{ax, regMask{}}, clobbers: dx}, typ: "(UInt32,Flags)", asm: "MULL", commutative: true, clobberFlags: true, earlyOk: true}, // Let x = arg0*arg1 (full 32x32->64 unsigned multiply). Returns uint32(x), and flags set to overflow if uint32(x) != x.
- {name: "HMULL", argLength: 2, reg: gp21hmul, commutative: true, asm: "IMULL", clobberFlags: true}, // (arg0 * arg1) >> width
- {name: "HMULLU", argLength: 2, reg: gp21hmul, commutative: true, asm: "MULL", clobberFlags: true}, // (arg0 * arg1) >> width
+ {name: "HMULL", argLength: 2, reg: gp21hmul, commutative: true, asm: "IMULL", clobberFlags: true, earlyOk: true}, // (arg0 * arg1) >> width
+ {name: "HMULLU", argLength: 2, reg: gp21hmul, commutative: true, asm: "MULL", clobberFlags: true, earlyOk: true}, // (arg0 * arg1) >> width
- {name: "MULLQU", argLength: 2, reg: gp21mul, commutative: true, asm: "MULL", clobberFlags: true}, // arg0 * arg1, high 32 in result[0], low 32 in result[1]
+ {name: "MULLQU", argLength: 2, reg: gp21mul, commutative: true, asm: "MULL", clobberFlags: true, earlyOk: true}, // arg0 * arg1, high 32 in result[0], low 32 in result[1]
- {name: "AVGLU", argLength: 2, reg: gp21, commutative: true, resultInArg0: true, clobberFlags: true}, // (arg0 + arg1) / 2 as unsigned, all 32 result bits
+ {name: "AVGLU", argLength: 2, reg: gp21, commutative: true, resultInArg0: true, clobberFlags: true, earlyOk: true}, // (arg0 + arg1) / 2 as unsigned, all 32 result bits
// For DIVL, DIVW, MODL and MODW, AuxInt non-zero means that the divisor has been proved to be not -1.
{name: "DIVL", argLength: 2, reg: gp11div, asm: "IDIVL", aux: "Bool", clobberFlags: true}, // arg0 / arg1
@@ -217,14 +219,14 @@ func init() {
{name: "MODLU", argLength: 2, reg: gp11mod, asm: "DIVL", clobberFlags: true}, // arg0 % arg1
{name: "MODWU", argLength: 2, reg: gp11mod, asm: "DIVW", clobberFlags: true}, // arg0 % arg1
- {name: "ANDL", argLength: 2, reg: gp21, asm: "ANDL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0 & arg1
- {name: "ANDLconst", argLength: 1, reg: gp11, asm: "ANDL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 & auxint
+ {name: "ANDL", argLength: 2, reg: gp21, asm: "ANDL", commutative: true, resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 & arg1
+ {name: "ANDLconst", argLength: 1, reg: gp11, asm: "ANDL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 & auxint
- {name: "ORL", argLength: 2, reg: gp21, asm: "ORL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0 | arg1
- {name: "ORLconst", argLength: 1, reg: gp11, asm: "ORL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 | auxint
+ {name: "ORL", argLength: 2, reg: gp21, asm: "ORL", commutative: true, resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 | arg1
+ {name: "ORLconst", argLength: 1, reg: gp11, asm: "ORL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 | auxint
- {name: "XORL", argLength: 2, reg: gp21, asm: "XORL", commutative: true, resultInArg0: true, clobberFlags: true}, // arg0 ^ arg1
- {name: "XORLconst", argLength: 1, reg: gp11, asm: "XORL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 ^ auxint
+ {name: "XORL", argLength: 2, reg: gp21, asm: "XORL", commutative: true, resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 ^ arg1
+ {name: "XORLconst", argLength: 1, reg: gp11, asm: "XORL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 ^ auxint
{name: "CMPL", argLength: 2, reg: gp2flags, asm: "CMPL", typ: "Flags"}, // arg0 compare to arg1
{name: "CMPW", argLength: 2, reg: gp2flags, asm: "CMPW", typ: "Flags"}, // arg0 compare to arg1
@@ -234,17 +236,17 @@ func init() {
{name: "CMPBconst", argLength: 1, reg: gp1flags, asm: "CMPB", typ: "Flags", aux: "Int8"}, // arg0 compare to auxint
// compare *(arg0+auxint+aux) to arg1 (in that order). arg2=mem.
- {name: "CMPLload", argLength: 3, reg: gp1flagsLoad, asm: "CMPL", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
- {name: "CMPWload", argLength: 3, reg: gp1flagsLoad, asm: "CMPW", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
- {name: "CMPBload", argLength: 3, reg: gp1flagsLoad, asm: "CMPB", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
+ {name: "CMPLload", argLength: 3, reg: gp1flagsLoad, asm: "CMPL", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
+ {name: "CMPWload", argLength: 3, reg: gp1flagsLoad, asm: "CMPW", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
+ {name: "CMPBload", argLength: 3, reg: gp1flagsLoad, asm: "CMPB", aux: "SymOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
// compare *(arg0+ValAndOff(AuxInt).Off()+aux) to ValAndOff(AuxInt).Val() (in that order). arg1=mem.
- {name: "CMPLconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPL", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
- {name: "CMPWconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPW", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
- {name: "CMPBconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPB", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true},
+ {name: "CMPLconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPL", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
+ {name: "CMPWconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPW", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
+ {name: "CMPBconstload", argLength: 2, reg: gp0flagsLoad, asm: "CMPB", aux: "SymValAndOff", typ: "Flags", symEffect: "Read", faultOnNilArg0: true, addrSinkArg0: true},
- {name: "UCOMISS", argLength: 2, reg: fp2flags, asm: "UCOMISS", typ: "Flags"}, // arg0 compare to arg1, f32
- {name: "UCOMISD", argLength: 2, reg: fp2flags, asm: "UCOMISD", typ: "Flags"}, // arg0 compare to arg1, f64
+ {name: "UCOMISS", argLength: 2, reg: fp2flags, asm: "UCOMISS", typ: "Flags", earlyOk: true}, // arg0 compare to arg1, f32
+ {name: "UCOMISD", argLength: 2, reg: fp2flags, asm: "UCOMISD", typ: "Flags", earlyOk: true}, // arg0 compare to arg1, f64
{name: "TESTL", argLength: 2, reg: gp2flags, commutative: true, asm: "TESTL", typ: "Flags"}, // (arg0 & arg1) compare to 0
{name: "TESTW", argLength: 2, reg: gp2flags, commutative: true, asm: "TESTW", typ: "Flags"}, // (arg0 & arg1) compare to 0
@@ -253,175 +255,175 @@ func init() {
{name: "TESTWconst", argLength: 1, reg: gp1flags, asm: "TESTW", typ: "Flags", aux: "Int16"}, // (arg0 & auxint) compare to 0
{name: "TESTBconst", argLength: 1, reg: gp1flags, asm: "TESTB", typ: "Flags", aux: "Int8"}, // (arg0 & auxint) compare to 0
- {name: "SHLL", argLength: 2, reg: gp21shift, asm: "SHLL", resultInArg0: true, clobberFlags: true}, // arg0 << arg1, shift amount is mod 32
- {name: "SHLLconst", argLength: 1, reg: gp11, asm: "SHLL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 << auxint, shift amount 0-31
+ {name: "SHLL", argLength: 2, reg: gp21shift, asm: "SHLL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 << arg1, shift amount is mod 32
+ {name: "SHLLconst", argLength: 1, reg: gp11, asm: "SHLL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 << auxint, shift amount 0-31
// Note: x86 is weird, the 16 and 8 byte shifts still use all 5 bits of shift amount!
- {name: "SHRL", argLength: 2, reg: gp21shift, asm: "SHRL", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32
- {name: "SHRW", argLength: 2, reg: gp21shift, asm: "SHRW", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32
- {name: "SHRB", argLength: 2, reg: gp21shift, asm: "SHRB", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> arg1, shift amount is mod 32
- {name: "SHRLconst", argLength: 1, reg: gp11, asm: "SHRL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-31
- {name: "SHRWconst", argLength: 1, reg: gp11, asm: "SHRW", aux: "Int16", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-15
- {name: "SHRBconst", argLength: 1, reg: gp11, asm: "SHRB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // unsigned arg0 >> auxint, shift amount 0-7
-
- {name: "SARL", argLength: 2, reg: gp21shift, asm: "SARL", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32
- {name: "SARW", argLength: 2, reg: gp21shift, asm: "SARW", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32
- {name: "SARB", argLength: 2, reg: gp21shift, asm: "SARB", resultInArg0: true, clobberFlags: true}, // signed arg0 >> arg1, shift amount is mod 32
- {name: "SARLconst", argLength: 1, reg: gp11, asm: "SARL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-31
- {name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int16", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-15
- {name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-7
-
- {name: "ROLL", argLength: 2, reg: gp21shift, asm: "ROLL", resultInArg0: true, clobberFlags: true}, // 32 bits of arg0 rotate left by arg1
- {name: "ROLW", argLength: 2, reg: gp21shift, asm: "ROLW", resultInArg0: true, clobberFlags: true}, // low 16 bits of arg0 rotate left by arg1
- {name: "ROLB", argLength: 2, reg: gp21shift, asm: "ROLB", resultInArg0: true, clobberFlags: true}, // low 8 bits of arg0 rotate left by arg1
- {name: "ROLLconst", argLength: 1, reg: gp11, asm: "ROLL", aux: "Int32", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-31
- {name: "ROLWconst", argLength: 1, reg: gp11, asm: "ROLW", aux: "Int16", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-15
- {name: "ROLBconst", argLength: 1, reg: gp11, asm: "ROLB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-7
+ {name: "SHRL", argLength: 2, reg: gp21shift, asm: "SHRL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> arg1, shift amount is mod 32
+ {name: "SHRW", argLength: 2, reg: gp21shift, asm: "SHRW", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> arg1, shift amount is mod 32
+ {name: "SHRB", argLength: 2, reg: gp21shift, asm: "SHRB", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> arg1, shift amount is mod 32
+ {name: "SHRLconst", argLength: 1, reg: gp11, asm: "SHRL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> auxint, shift amount 0-31
+ {name: "SHRWconst", argLength: 1, reg: gp11, asm: "SHRW", aux: "Int16", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> auxint, shift amount 0-15
+ {name: "SHRBconst", argLength: 1, reg: gp11, asm: "SHRB", aux: "Int8", resultInArg0: true, clobberFlags: true, earlyOk: true}, // unsigned arg0 >> auxint, shift amount 0-7
+
+ {name: "SARL", argLength: 2, reg: gp21shift, asm: "SARL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> arg1, shift amount is mod 32
+ {name: "SARW", argLength: 2, reg: gp21shift, asm: "SARW", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> arg1, shift amount is mod 32
+ {name: "SARB", argLength: 2, reg: gp21shift, asm: "SARB", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> arg1, shift amount is mod 32
+ {name: "SARLconst", argLength: 1, reg: gp11, asm: "SARL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> auxint, shift amount 0-31
+ {name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int16", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> auxint, shift amount 0-15
+ {name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true, earlyOk: true}, // signed arg0 >> auxint, shift amount 0-7
+
+ {name: "ROLL", argLength: 2, reg: gp21shift, asm: "ROLL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // 32 bits of arg0 rotate left by arg1
+ {name: "ROLW", argLength: 2, reg: gp21shift, asm: "ROLW", resultInArg0: true, clobberFlags: true, earlyOk: true}, // low 16 bits of arg0 rotate left by arg1
+ {name: "ROLB", argLength: 2, reg: gp21shift, asm: "ROLB", resultInArg0: true, clobberFlags: true, earlyOk: true}, // low 8 bits of arg0 rotate left by arg1
+ {name: "ROLLconst", argLength: 1, reg: gp11, asm: "ROLL", aux: "Int32", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 rotate left auxint, rotate amount 0-31
+ {name: "ROLWconst", argLength: 1, reg: gp11, asm: "ROLW", aux: "Int16", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 rotate left auxint, rotate amount 0-15
+ {name: "ROLBconst", argLength: 1, reg: gp11, asm: "ROLB", aux: "Int8", resultInArg0: true, clobberFlags: true, earlyOk: true}, // arg0 rotate left auxint, rotate amount 0-7
// binary-op with a memory source operand
- {name: "ADDLload", argLength: 3, reg: gp21load, asm: "ADDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "SUBLload", argLength: 3, reg: gp21load, asm: "SUBL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "MULLload", argLength: 3, reg: gp21load, asm: "IMULL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "ANDLload", argLength: 3, reg: gp21load, asm: "ANDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 & tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "ORLload", argLength: 3, reg: gp21load, asm: "ORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 | tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
- {name: "XORLload", argLength: 3, reg: gp21load, asm: "XORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read"}, // arg0 ^ tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "ADDLload", argLength: 3, reg: gp21load, asm: "ADDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 + tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "SUBLload", argLength: 3, reg: gp21load, asm: "SUBL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 - tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "MULLload", argLength: 3, reg: gp21load, asm: "IMULL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 * tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "ANDLload", argLength: 3, reg: gp21load, asm: "ANDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 & tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "ORLload", argLength: 3, reg: gp21load, asm: "ORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 | tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
+ {name: "XORLload", argLength: 3, reg: gp21load, asm: "XORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // arg0 ^ tmp, tmp loaded from arg1+auxint+aux, arg2 = mem
// binary-op with an indexed memory source operand
- {name: "ADDLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ADDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 + tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
- {name: "SUBLloadidx4", argLength: 4, reg: gp21loadidx, asm: "SUBL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 - tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
- {name: "MULLloadidx4", argLength: 4, reg: gp21loadidx, asm: "IMULL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 * tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
- {name: "ANDLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ANDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 & tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
- {name: "ORLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 | tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
- {name: "XORLloadidx4", argLength: 4, reg: gp21loadidx, asm: "XORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read"}, // arg0 ^ tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "ADDLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ADDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 + tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "SUBLloadidx4", argLength: 4, reg: gp21loadidx, asm: "SUBL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 - tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "MULLloadidx4", argLength: 4, reg: gp21loadidx, asm: "IMULL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 * tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "ANDLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ANDL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 & tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "ORLloadidx4", argLength: 4, reg: gp21loadidx, asm: "ORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 | tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
+ {name: "XORLloadidx4", argLength: 4, reg: gp21loadidx, asm: "XORL", aux: "SymOff", resultInArg0: true, clobberFlags: true, symEffect: "Read", addrSinkArg1: true}, // arg0 ^ tmp, tmp loaded from arg1+arg2*4+auxint+aux, arg3 = mem
// unary ops
- {name: "NEGL", argLength: 1, reg: gp11, asm: "NEGL", resultInArg0: true, clobberFlags: true}, // -arg0
+ {name: "NEGL", argLength: 1, reg: gp11, asm: "NEGL", resultInArg0: true, clobberFlags: true, earlyOk: true}, // -arg0
- {name: "NOTL", argLength: 1, reg: gp11, asm: "NOTL", resultInArg0: true}, // ^arg0
+ {name: "NOTL", argLength: 1, reg: gp11, asm: "NOTL", resultInArg0: true, earlyOk: true}, // ^arg0
- {name: "BSFL", argLength: 1, reg: gp11, asm: "BSFL", clobberFlags: true}, // arg0 # of low-order zeroes ; undef if zero
- {name: "BSFW", argLength: 1, reg: gp11, asm: "BSFW", clobberFlags: true}, // arg0 # of low-order zeroes ; undef if zero
- {name: "LoweredCtz32", argLength: 1, reg: gp11, clobberFlags: true}, // arg0 # of low-order zeroes
- {name: "LoweredCtz64", argLength: 2, reg: gp21, resultNotInArgs: true, clobberFlags: true}, // arg1<<32+arg0 # of low-order zeroes
+ {name: "BSFL", argLength: 1, reg: gp11, asm: "BSFL", clobberFlags: true, earlyOk: true}, // arg0 # of low-order zeroes ; undef if zero
+ {name: "BSFW", argLength: 1, reg: gp11, asm: "BSFW", clobberFlags: true, earlyOk: true}, // arg0 # of low-order zeroes ; undef if zero
+ {name: "LoweredCtz32", argLength: 1, reg: gp11, clobberFlags: true, earlyOk: true}, // arg0 # of low-order zeroes
+ {name: "LoweredCtz64", argLength: 2, reg: gp21, resultNotInArgs: true, clobberFlags: true, earlyOk: true}, // arg1<<32+arg0 # of low-order zeroes
- {name: "BSRL", argLength: 1, reg: gp11, asm: "BSRL", clobberFlags: true}, // arg0 # of high-order zeroes ; undef if zero
- {name: "BSRW", argLength: 1, reg: gp11, asm: "BSRW", clobberFlags: true}, // arg0 # of high-order zeroes ; undef if zero
+ {name: "BSRL", argLength: 1, reg: gp11, asm: "BSRL", clobberFlags: true, earlyOk: true}, // arg0 # of high-order zeroes ; undef if zero
+ {name: "BSRW", argLength: 1, reg: gp11, asm: "BSRW", clobberFlags: true, earlyOk: true}, // arg0 # of high-order zeroes ; undef if zero
- {name: "BSWAPL", argLength: 1, reg: gp11, asm: "BSWAPL", resultInArg0: true}, // arg0 swap bytes
+ {name: "BSWAPL", argLength: 1, reg: gp11, asm: "BSWAPL", resultInArg0: true, earlyOk: true}, // arg0 swap bytes
- {name: "SQRTSD", argLength: 1, reg: fp11, asm: "SQRTSD"}, // sqrt(arg0)
- {name: "SQRTSS", argLength: 1, reg: fp11, asm: "SQRTSS"}, // sqrt(arg0), float32
+ {name: "SQRTSD", argLength: 1, reg: fp11, asm: "SQRTSD", earlyOk: true}, // sqrt(arg0)
+ {name: "SQRTSS", argLength: 1, reg: fp11, asm: "SQRTSS", earlyOk: true}, // sqrt(arg0), float32
- {name: "SBBLcarrymask", argLength: 1, reg: flagsgp, asm: "SBBL"}, // (int32)(-1) if carry is set, 0 if carry is clear.
+ {name: "SBBLcarrymask", argLength: 1, reg: flagsgp, asm: "SBBL", earlyOk: true}, // (int32)(-1) if carry is set, 0 if carry is clear.
// Note: SBBW and SBBB are subsumed by SBBL
- {name: "SETEQ", argLength: 1, reg: readflags, asm: "SETEQ"}, // extract == condition from arg0
- {name: "SETNE", argLength: 1, reg: readflags, asm: "SETNE"}, // extract != condition from arg0
- {name: "SETL", argLength: 1, reg: readflags, asm: "SETLT"}, // extract signed < condition from arg0
- {name: "SETLE", argLength: 1, reg: readflags, asm: "SETLE"}, // extract signed <= condition from arg0
- {name: "SETG", argLength: 1, reg: readflags, asm: "SETGT"}, // extract signed > condition from arg0
- {name: "SETGE", argLength: 1, reg: readflags, asm: "SETGE"}, // extract signed >= condition from arg0
- {name: "SETB", argLength: 1, reg: readflags, asm: "SETCS"}, // extract unsigned < condition from arg0
- {name: "SETBE", argLength: 1, reg: readflags, asm: "SETLS"}, // extract unsigned <= condition from arg0
- {name: "SETA", argLength: 1, reg: readflags, asm: "SETHI"}, // extract unsigned > condition from arg0
- {name: "SETAE", argLength: 1, reg: readflags, asm: "SETCC"}, // extract unsigned >= condition from arg0
- {name: "SETO", argLength: 1, reg: readflags, asm: "SETOS"}, // extract if overflow flag is set from arg0
+ {name: "SETEQ", argLength: 1, reg: readflags, asm: "SETEQ", earlyOk: true}, // extract == condition from arg0
+ {name: "SETNE", argLength: 1, reg: readflags, asm: "SETNE", earlyOk: true}, // extract != condition from arg0
+ {name: "SETL", argLength: 1, reg: readflags, asm: "SETLT", earlyOk: true}, // extract signed < condition from arg0
+ {name: "SETLE", argLength: 1, reg: readflags, asm: "SETLE", earlyOk: true}, // extract signed <= condition from arg0
+ {name: "SETG", argLength: 1, reg: readflags, asm: "SETGT", earlyOk: true}, // extract signed > condition from arg0
+ {name: "SETGE", argLength: 1, reg: readflags, asm: "SETGE", earlyOk: true}, // extract signed >= condition from arg0
+ {name: "SETB", argLength: 1, reg: readflags, asm: "SETCS", earlyOk: true}, // extract unsigned < condition from arg0
+ {name: "SETBE", argLength: 1, reg: readflags, asm: "SETLS", earlyOk: true}, // extract unsigned <= condition from arg0
+ {name: "SETA", argLength: 1, reg: readflags, asm: "SETHI", earlyOk: true}, // extract unsigned > condition from arg0
+ {name: "SETAE", argLength: 1, reg: readflags, asm: "SETCC", earlyOk: true}, // extract unsigned >= condition from arg0
+ {name: "SETO", argLength: 1, reg: readflags, asm: "SETOS", earlyOk: true}, // extract if overflow flag is set from arg0
// Need different opcodes for floating point conditions because
// any comparison involving a NaN is always FALSE and thus
// the patterns for inverting conditions cannot be used.
- {name: "SETEQF", argLength: 1, reg: flagsgpax, asm: "SETEQ", clobberFlags: true}, // extract == condition from arg0
- {name: "SETNEF", argLength: 1, reg: flagsgpax, asm: "SETNE", clobberFlags: true}, // extract != condition from arg0
- {name: "SETORD", argLength: 1, reg: flagsgp, asm: "SETPC"}, // extract "ordered" (No Nan present) condition from arg0
- {name: "SETNAN", argLength: 1, reg: flagsgp, asm: "SETPS"}, // extract "unordered" (Nan present) condition from arg0
-
- {name: "SETGF", argLength: 1, reg: flagsgp, asm: "SETHI"}, // extract floating > condition from arg0
- {name: "SETGEF", argLength: 1, reg: flagsgp, asm: "SETCC"}, // extract floating >= condition from arg0
-
- {name: "MOVBLSX", argLength: 1, reg: gp11, asm: "MOVBLSX"}, // sign extend arg0 from int8 to int32
- {name: "MOVBLZX", argLength: 1, reg: gp11, asm: "MOVBLZX"}, // zero extend arg0 from int8 to int32
- {name: "MOVWLSX", argLength: 1, reg: gp11, asm: "MOVWLSX"}, // sign extend arg0 from int16 to int32
- {name: "MOVWLZX", argLength: 1, reg: gp11, asm: "MOVWLZX"}, // zero extend arg0 from int16 to int32
-
- {name: "MOVLconst", reg: gp01, asm: "MOVL", typ: "UInt32", aux: "Int32", rematerializeable: true}, // 32 low bits of auxint
-
- {name: "CVTTSD2SL", argLength: 1, reg: fpgp, asm: "CVTTSD2SL"}, // convert float64 to int32
- {name: "CVTTSS2SL", argLength: 1, reg: fpgp, asm: "CVTTSS2SL"}, // convert float32 to int32
- {name: "CVTSL2SS", argLength: 1, reg: gpfp, asm: "CVTSL2SS"}, // convert int32 to float32
- {name: "CVTSL2SD", argLength: 1, reg: gpfp, asm: "CVTSL2SD"}, // convert int32 to float64
- {name: "CVTSD2SS", argLength: 1, reg: fp11, asm: "CVTSD2SS"}, // convert float64 to float32
- {name: "CVTSS2SD", argLength: 1, reg: fp11, asm: "CVTSS2SD"}, // convert float32 to float64
-
- {name: "PXOR", argLength: 2, reg: fp21, asm: "PXOR", commutative: true, resultInArg0: true}, // exclusive or, applied to X regs for float negation.
-
- {name: "LEAL", argLength: 1, reg: gp11sb, aux: "SymOff", rematerializeable: true, symEffect: "Addr"}, // arg0 + auxint + offset encoded in aux
- {name: "LEAL1", argLength: 2, reg: gp21sb, commutative: true, aux: "SymOff", symEffect: "Addr"}, // arg0 + arg1 + auxint + aux
- {name: "LEAL2", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr"}, // arg0 + 2*arg1 + auxint + aux
- {name: "LEAL4", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr"}, // arg0 + 4*arg1 + auxint + aux
- {name: "LEAL8", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr"}, // arg0 + 8*arg1 + auxint + aux
+ {name: "SETEQF", argLength: 1, reg: flagsgpax, asm: "SETEQ", clobberFlags: true, earlyOk: true}, // extract == condition from arg0
+ {name: "SETNEF", argLength: 1, reg: flagsgpax, asm: "SETNE", clobberFlags: true, earlyOk: true}, // extract != condition from arg0
+ {name: "SETORD", argLength: 1, reg: flagsgp, asm: "SETPC", earlyOk: true}, // extract "ordered" (No Nan present) condition from arg0
+ {name: "SETNAN", argLength: 1, reg: flagsgp, asm: "SETPS", earlyOk: true}, // extract "unordered" (Nan present) condition from arg0
+
+ {name: "SETGF", argLength: 1, reg: flagsgp, asm: "SETHI", earlyOk: true}, // extract floating > condition from arg0
+ {name: "SETGEF", argLength: 1, reg: flagsgp, asm: "SETCC", earlyOk: true}, // extract floating >= condition from arg0
+
+ {name: "MOVBLSX", argLength: 1, reg: gp11, asm: "MOVBLSX", earlyOk: true}, // sign extend arg0 from int8 to int32
+ {name: "MOVBLZX", argLength: 1, reg: gp11, asm: "MOVBLZX", earlyOk: true}, // zero extend arg0 from int8 to int32
+ {name: "MOVWLSX", argLength: 1, reg: gp11, asm: "MOVWLSX", earlyOk: true}, // sign extend arg0 from int16 to int32
+ {name: "MOVWLZX", argLength: 1, reg: gp11, asm: "MOVWLZX", earlyOk: true}, // zero extend arg0 from int16 to int32
+
+ {name: "MOVLconst", reg: gp01, asm: "MOVL", typ: "UInt32", aux: "Int32", rematerializeable: true, earlyOk: true}, // 32 low bits of auxint
+
+ {name: "CVTTSD2SL", argLength: 1, reg: fpgp, asm: "CVTTSD2SL", earlyOk: true}, // convert float64 to int32
+ {name: "CVTTSS2SL", argLength: 1, reg: fpgp, asm: "CVTTSS2SL", earlyOk: true}, // convert float32 to int32
+ {name: "CVTSL2SS", argLength: 1, reg: gpfp, asm: "CVTSL2SS", earlyOk: true}, // convert int32 to float32
+ {name: "CVTSL2SD", argLength: 1, reg: gpfp, asm: "CVTSL2SD", earlyOk: true}, // convert int32 to float64
+ {name: "CVTSD2SS", argLength: 1, reg: fp11, asm: "CVTSD2SS", earlyOk: true}, // convert float64 to float32
+ {name: "CVTSS2SD", argLength: 1, reg: fp11, asm: "CVTSS2SD", earlyOk: true}, // convert float32 to float64
+
+ {name: "PXOR", argLength: 2, reg: fp21, asm: "PXOR", commutative: true, resultInArg0: true, earlyOk: true}, // exclusive or, applied to X regs for float negation.
+
+ {name: "LEAL", argLength: 1, reg: gp11sb, aux: "SymOff", rematerializeable: true, symEffect: "Addr", earlyOk: true}, // arg0 + auxint + offset encoded in aux
+ {name: "LEAL1", argLength: 2, reg: gp21sb, commutative: true, aux: "SymOff", symEffect: "Addr", earlyOk: true}, // arg0 + arg1 + auxint + aux
+ {name: "LEAL2", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr", earlyOk: true}, // arg0 + 2*arg1 + auxint + aux
+ {name: "LEAL4", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr", earlyOk: true}, // arg0 + 4*arg1 + auxint + aux
+ {name: "LEAL8", argLength: 2, reg: gp21sb, aux: "SymOff", symEffect: "Addr", earlyOk: true}, // arg0 + 8*arg1 + auxint + aux
// Note: LEAL{1,2,4,8} must not have OpSB as either argument.
// auxint+aux == add auxint and the offset of the symbol in aux (if any) to the effective address
- {name: "MOVBload", argLength: 2, reg: gpload, asm: "MOVBLZX", aux: "SymOff", typ: "UInt8", faultOnNilArg0: true, symEffect: "Read"}, // load byte from arg0+auxint+aux. arg1=mem. Zero extend.
- {name: "MOVBLSXload", argLength: 2, reg: gpload, asm: "MOVBLSX", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read"}, // ditto, sign extend to int32
- {name: "MOVWload", argLength: 2, reg: gpload, asm: "MOVWLZX", aux: "SymOff", typ: "UInt16", faultOnNilArg0: true, symEffect: "Read"}, // load 2 bytes from arg0+auxint+aux. arg1=mem. Zero extend.
- {name: "MOVWLSXload", argLength: 2, reg: gpload, asm: "MOVWLSX", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read"}, // ditto, sign extend to int32
- {name: "MOVLload", argLength: 2, reg: gpload, asm: "MOVL", aux: "SymOff", typ: "UInt32", faultOnNilArg0: true, symEffect: "Read"}, // load 4 bytes from arg0+auxint+aux. arg1=mem. Zero extend.
- {name: "MOVBstore", argLength: 3, reg: gpstore, asm: "MOVB", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store byte in arg1 to arg0+auxint+aux. arg2=mem
- {name: "MOVWstore", argLength: 3, reg: gpstore, asm: "MOVW", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store 2 bytes in arg1 to arg0+auxint+aux. arg2=mem
- {name: "MOVLstore", argLength: 3, reg: gpstore, asm: "MOVL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store 4 bytes in arg1 to arg0+auxint+aux. arg2=mem
+ {name: "MOVBload", argLength: 2, reg: gpload, asm: "MOVBLZX", aux: "SymOff", typ: "UInt8", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // load byte from arg0+auxint+aux. arg1=mem. Zero extend.
+ {name: "MOVBLSXload", argLength: 2, reg: gpload, asm: "MOVBLSX", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // ditto, sign extend to int32
+ {name: "MOVWload", argLength: 2, reg: gpload, asm: "MOVWLZX", aux: "SymOff", typ: "UInt16", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // load 2 bytes from arg0+auxint+aux. arg1=mem. Zero extend.
+ {name: "MOVWLSXload", argLength: 2, reg: gpload, asm: "MOVWLSX", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // ditto, sign extend to int32
+ {name: "MOVLload", argLength: 2, reg: gpload, asm: "MOVL", aux: "SymOff", typ: "UInt32", faultOnNilArg0: true, symEffect: "Read", addrSinkArg0: true}, // load 4 bytes from arg0+auxint+aux. arg1=mem. Zero extend.
+ {name: "MOVBstore", argLength: 3, reg: gpstore, asm: "MOVB", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store byte in arg1 to arg0+auxint+aux. arg2=mem
+ {name: "MOVWstore", argLength: 3, reg: gpstore, asm: "MOVW", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store 2 bytes in arg1 to arg0+auxint+aux. arg2=mem
+ {name: "MOVLstore", argLength: 3, reg: gpstore, asm: "MOVL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store 4 bytes in arg1 to arg0+auxint+aux. arg2=mem
// direct binary-op on memory (read-modify-write)
- {name: "ADDLmodify", argLength: 3, reg: gpstore, asm: "ADDL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+auxint+aux) += arg1, arg2=mem
- {name: "SUBLmodify", argLength: 3, reg: gpstore, asm: "SUBL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+auxint+aux) -= arg1, arg2=mem
- {name: "ANDLmodify", argLength: 3, reg: gpstore, asm: "ANDL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+auxint+aux) &= arg1, arg2=mem
- {name: "ORLmodify", argLength: 3, reg: gpstore, asm: "ORL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+auxint+aux) |= arg1, arg2=mem
- {name: "XORLmodify", argLength: 3, reg: gpstore, asm: "XORL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+auxint+aux) ^= arg1, arg2=mem
+ {name: "ADDLmodify", argLength: 3, reg: gpstore, asm: "ADDL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+auxint+aux) += arg1, arg2=mem
+ {name: "SUBLmodify", argLength: 3, reg: gpstore, asm: "SUBL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+auxint+aux) -= arg1, arg2=mem
+ {name: "ANDLmodify", argLength: 3, reg: gpstore, asm: "ANDL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+auxint+aux) &= arg1, arg2=mem
+ {name: "ORLmodify", argLength: 3, reg: gpstore, asm: "ORL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+auxint+aux) |= arg1, arg2=mem
+ {name: "XORLmodify", argLength: 3, reg: gpstore, asm: "XORL", aux: "SymOff", typ: "Mem", faultOnNilArg0: true, clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+auxint+aux) ^= arg1, arg2=mem
// direct binary-op on indexed memory (read-modify-write)
- {name: "ADDLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ADDL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+arg1*4+auxint+aux) += arg2, arg3=mem
- {name: "SUBLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "SUBL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+arg1*4+auxint+aux) -= arg2, arg3=mem
- {name: "ANDLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ANDL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+arg1*4+auxint+aux) &= arg2, arg3=mem
- {name: "ORLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ORL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+arg1*4+auxint+aux) |= arg2, arg3=mem
- {name: "XORLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "XORL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // *(arg0+arg1*4+auxint+aux) ^= arg2, arg3=mem
+ {name: "ADDLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ADDL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+arg1*4+auxint+aux) += arg2, arg3=mem
+ {name: "SUBLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "SUBL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+arg1*4+auxint+aux) -= arg2, arg3=mem
+ {name: "ANDLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ANDL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+arg1*4+auxint+aux) &= arg2, arg3=mem
+ {name: "ORLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "ORL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+arg1*4+auxint+aux) |= arg2, arg3=mem
+ {name: "XORLmodifyidx4", argLength: 4, reg: gpstoreidx, asm: "XORL", aux: "SymOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // *(arg0+arg1*4+auxint+aux) ^= arg2, arg3=mem
// direct binary-op on memory with a constant (read-modify-write)
- {name: "ADDLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ADDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write"}, // add ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
- {name: "ANDLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ANDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write"}, // and ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
- {name: "ORLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write"}, // or ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
- {name: "XORLconstmodify", argLength: 2, reg: gpstoreconst, asm: "XORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write"}, // xor ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
+ {name: "ADDLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ADDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write", addrSinkArg0: true}, // add ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
+ {name: "ANDLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ANDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write", addrSinkArg0: true}, // and ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
+ {name: "ORLconstmodify", argLength: 2, reg: gpstoreconst, asm: "ORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write", addrSinkArg0: true}, // or ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
+ {name: "XORLconstmodify", argLength: 2, reg: gpstoreconst, asm: "XORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Read,Write", addrSinkArg0: true}, // xor ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux, arg1=mem
// direct binary-op on indexed memory with a constant (read-modify-write)
- {name: "ADDLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ADDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // add ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
- {name: "ANDLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ANDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // and ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
- {name: "ORLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // or ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
- {name: "XORLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "XORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write"}, // xor ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
+ {name: "ADDLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ADDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // add ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
+ {name: "ANDLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ANDL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // and ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
+ {name: "ORLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "ORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // or ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
+ {name: "XORLconstmodifyidx4", argLength: 3, reg: gpstoreconstidx, asm: "XORL", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, symEffect: "Read,Write", addrSinkArg0: true}, // xor ValAndOff(AuxInt).Val() to arg0+arg1*4+ValAndOff(AuxInt).Off()+aux, arg2=mem
// indexed loads/stores
- {name: "MOVBloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVBLZX", aux: "SymOff", symEffect: "Read"}, // load a byte from arg0+arg1+auxint+aux. arg2=mem
- {name: "MOVWloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVWLZX", aux: "SymOff", symEffect: "Read"}, // load 2 bytes from arg0+arg1+auxint+aux. arg2=mem
- {name: "MOVWloadidx2", argLength: 3, reg: gploadidx, asm: "MOVWLZX", aux: "SymOff", symEffect: "Read"}, // load 2 bytes from arg0+2*arg1+auxint+aux. arg2=mem
- {name: "MOVLloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVL", aux: "SymOff", symEffect: "Read"}, // load 4 bytes from arg0+arg1+auxint+aux. arg2=mem
- {name: "MOVLloadidx4", argLength: 3, reg: gploadidx, asm: "MOVL", aux: "SymOff", symEffect: "Read"}, // load 4 bytes from arg0+4*arg1+auxint+aux. arg2=mem
+ {name: "MOVBloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVBLZX", aux: "SymOff", symEffect: "Read", addrSinkArg0: true, addrSinkArg1: true}, // load a byte from arg0+arg1+auxint+aux. arg2=mem
+ {name: "MOVWloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVWLZX", aux: "SymOff", symEffect: "Read", addrSinkArg0: true, addrSinkArg1: true}, // load 2 bytes from arg0+arg1+auxint+aux. arg2=mem
+ {name: "MOVWloadidx2", argLength: 3, reg: gploadidx, asm: "MOVWLZX", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // load 2 bytes from arg0+2*arg1+auxint+aux. arg2=mem
+ {name: "MOVLloadidx1", argLength: 3, reg: gploadidx, commutative: true, asm: "MOVL", aux: "SymOff", symEffect: "Read", addrSinkArg0: true, addrSinkArg1: true}, // load 4 bytes from arg0+arg1+auxint+aux. arg2=mem
+ {name: "MOVLloadidx4", argLength: 3, reg: gploadidx, asm: "MOVL", aux: "SymOff", symEffect: "Read", addrSinkArg0: true}, // load 4 bytes from arg0+4*arg1+auxint+aux. arg2=mem
// TODO: sign-extending indexed loads
- {name: "MOVBstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVB", aux: "SymOff", symEffect: "Write"}, // store byte in arg2 to arg0+arg1+auxint+aux. arg3=mem
- {name: "MOVWstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVW", aux: "SymOff", symEffect: "Write"}, // store 2 bytes in arg2 to arg0+arg1+auxint+aux. arg3=mem
- {name: "MOVWstoreidx2", argLength: 4, reg: gpstoreidx, asm: "MOVW", aux: "SymOff", symEffect: "Write"}, // store 2 bytes in arg2 to arg0+2*arg1+auxint+aux. arg3=mem
- {name: "MOVLstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVL", aux: "SymOff", symEffect: "Write"}, // store 4 bytes in arg2 to arg0+arg1+auxint+aux. arg3=mem
- {name: "MOVLstoreidx4", argLength: 4, reg: gpstoreidx, asm: "MOVL", aux: "SymOff", symEffect: "Write"}, // store 4 bytes in arg2 to arg0+4*arg1+auxint+aux. arg3=mem
+ {name: "MOVBstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVB", aux: "SymOff", symEffect: "Write", addrSinkArg0: true, addrSinkArg1: true}, // store byte in arg2 to arg0+arg1+auxint+aux. arg3=mem
+ {name: "MOVWstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVW", aux: "SymOff", symEffect: "Write", addrSinkArg0: true, addrSinkArg1: true}, // store 2 bytes in arg2 to arg0+arg1+auxint+aux. arg3=mem
+ {name: "MOVWstoreidx2", argLength: 4, reg: gpstoreidx, asm: "MOVW", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // store 2 bytes in arg2 to arg0+2*arg1+auxint+aux. arg3=mem
+ {name: "MOVLstoreidx1", argLength: 4, reg: gpstoreidx, commutative: true, asm: "MOVL", aux: "SymOff", symEffect: "Write", addrSinkArg0: true, addrSinkArg1: true}, // store 4 bytes in arg2 to arg0+arg1+auxint+aux. arg3=mem
+ {name: "MOVLstoreidx4", argLength: 4, reg: gpstoreidx, asm: "MOVL", aux: "SymOff", symEffect: "Write", addrSinkArg0: true}, // store 4 bytes in arg2 to arg0+4*arg1+auxint+aux. arg3=mem
// TODO: add size-mismatched indexed loads, like MOVBstoreidx4.
// For storeconst ops, the AuxInt field encodes both
// the value to store and an address offset of the store.
// Cast AuxInt to a ValAndOff to extract Val and Off fields.
- {name: "MOVBstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVB", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store low byte of ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux. arg1=mem
- {name: "MOVWstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store low 2 bytes of ...
- {name: "MOVLstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write"}, // store low 4 bytes of ...
+ {name: "MOVBstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVB", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store low byte of ValAndOff(AuxInt).Val() to arg0+ValAndOff(AuxInt).Off()+aux. arg1=mem
+ {name: "MOVWstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store low 2 bytes of ...
+ {name: "MOVLstoreconst", argLength: 2, reg: gpstoreconst, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", faultOnNilArg0: true, symEffect: "Write", addrSinkArg0: true}, // store low 4 bytes of ...
- {name: "MOVBstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVB", aux: "SymValAndOff", typ: "Mem", symEffect: "Write"}, // store low byte of ValAndOff(AuxInt).Val() to arg0+1*arg1+ValAndOff(AuxInt).Off()+aux. arg2=mem
- {name: "MOVWstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", symEffect: "Write"}, // store low 2 bytes of ... arg1 ...
- {name: "MOVWstoreconstidx2", argLength: 3, reg: gpstoreconstidx, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", symEffect: "Write"}, // store low 2 bytes of ... 2*arg1 ...
- {name: "MOVLstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", symEffect: "Write"}, // store low 4 bytes of ... arg1 ...
- {name: "MOVLstoreconstidx4", argLength: 3, reg: gpstoreconstidx, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", symEffect: "Write"}, // store low 4 bytes of ... 4*arg1 ...
+ {name: "MOVBstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVB", aux: "SymValAndOff", typ: "Mem", symEffect: "Write", addrSinkArg0: true}, // store low byte of ValAndOff(AuxInt).Val() to arg0+1*arg1+ValAndOff(AuxInt).Off()+aux. arg2=mem
+ {name: "MOVWstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", symEffect: "Write", addrSinkArg0: true}, // store low 2 bytes of ... arg1 ...
+ {name: "MOVWstoreconstidx2", argLength: 3, reg: gpstoreconstidx, asm: "MOVW", aux: "SymValAndOff", typ: "Mem", symEffect: "Write", addrSinkArg0: true}, // store low 2 bytes of ... 2*arg1 ...
+ {name: "MOVLstoreconstidx1", argLength: 3, reg: gpstoreconstidx, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", symEffect: "Write", addrSinkArg0: true}, // store low 4 bytes of ... arg1 ...
+ {name: "MOVLstoreconstidx4", argLength: 3, reg: gpstoreconstidx, asm: "MOVL", aux: "SymValAndOff", typ: "Mem", symEffect: "Write", addrSinkArg0: true}, // store low 4 bytes of ... 4*arg1 ...
// arg0 = pointer to start of memory to zero
// arg1 = value to store (will always be zero)
@@ -438,6 +440,7 @@ func init() {
// Note: CX is only clobbered when dynamic linking.
},
faultOnNilArg0: true,
+ addrSinkArg0: true,
},
// arg0 = address of memory to zero
@@ -453,12 +456,14 @@ func init() {
clobbers: buildReg("DI CX"),
},
faultOnNilArg0: true,
+ addrSinkArg0: true,
},
- {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
- {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true}, // tail call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
- {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("DX"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure. arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
- {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
+ {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
+ {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true}, // tail call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
+ {name: "CALLtailinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true}, // tail call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
+ {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("DX"), regMask{}}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure. arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
+ {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
// arg0 = destination pointer
// arg1 = source pointer
@@ -476,6 +481,8 @@ func init() {
clobberFlags: true,
faultOnNilArg0: true,
faultOnNilArg1: true,
+ addrSinkArg0: true,
+ addrSinkArg1: true,
},
// arg0 = destination pointer
@@ -492,6 +499,8 @@ func init() {
},
faultOnNilArg0: true,
faultOnNilArg1: true,
+ addrSinkArg0: true,
+ addrSinkArg1: true,
},
// (InvertFlags (CMPL a b)) == (CMPL b a)
@@ -520,7 +529,7 @@ func init() {
// LoweredWB invokes runtime.gcWriteBarrier. arg0=mem, auxint=# of write barrier slots
// It saves all GP registers if necessary, but may clobber others.
// Returns a pointer to a write barrier buffer in DI.
- {name: "LoweredWB", argLength: 1, reg: regInfo{clobbers: callerSave &^ gp, outputs: []regMask{buildReg("DI")}}, clobberFlags: true, aux: "Int64"},
+ {name: "LoweredWB", argLength: 1, reg: regInfo{clobbers: callerSave.minus(gp), outputs: []regMask{buildReg("DI")}}, clobberFlags: true, aux: "Int64"},
// LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail.
// the RC and CR versions are used when one of the arguments is a constant. CC is used
@@ -533,8 +542,8 @@ func init() {
{name: "LoweredPanicBoundsCC", argLength: 1, aux: "PanicBoundsCC", reg: regInfo{}, typ: "Mem", call: true}, // arg0=mem, returns memory.
// Same as above, but the x value is 64 bits.
- {name: "LoweredPanicExtendRR", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{ax | cx | dx | bx, ax | cx | dx | bx, gp}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=y, arg3=mem, returns memory.
- {name: "LoweredPanicExtendRC", argLength: 3, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{ax | cx | dx | bx, ax | cx | dx | bx}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=mem, returns memory.
+ {name: "LoweredPanicExtendRR", argLength: 4, aux: "Int64", reg: regInfo{inputs: []regMask{ax.union(cx).union(dx).union(bx), ax.union(cx).union(dx).union(bx), gp}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=y, arg3=mem, returns memory.
+ {name: "LoweredPanicExtendRC", argLength: 3, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{ax.union(cx).union(dx).union(bx), ax.union(cx).union(dx).union(bx)}}, typ: "Mem", call: true}, // arg0=x_hi, arg1=x_lo, arg2=mem, returns memory.
// Constant flag values. For any comparison, there are 5 possible
// outcomes: the three from the signed total order (<,==,>) and the
@@ -553,10 +562,10 @@ func init() {
// MOVSXconst1 loads the address of the constant-pool entry into a register.
// MOVSXconst2 loads the constant from that address.
// MOVSXconst1 returns a pointer, but we type it as uint32 because it can never point to the Go heap.
- {name: "MOVSSconst1", reg: gp01, typ: "UInt32", aux: "Float32"},
- {name: "MOVSDconst1", reg: gp01, typ: "UInt32", aux: "Float64"},
- {name: "MOVSSconst2", argLength: 1, reg: gpfp, asm: "MOVSS"},
- {name: "MOVSDconst2", argLength: 1, reg: gpfp, asm: "MOVSD"},
+ {name: "MOVSSconst1", reg: gp01, typ: "UInt32", aux: "Float32", earlyOk: true},
+ {name: "MOVSDconst1", reg: gp01, typ: "UInt32", aux: "Float64", earlyOk: true},
+ {name: "MOVSSconst2", argLength: 1, reg: gpfp, asm: "MOVSS", earlyOk: true},
+ {name: "MOVSDconst2", argLength: 1, reg: gpfp, asm: "MOVSD", earlyOk: true},
}
var _386blocks = []blockData{
diff --git a/src/cmd/compile/internal/ssa/_gen/386splitload.rules b/src/cmd/compile/internal/ssa/_gen/386splitload.rules
index 29d4f8c227f550..b4258b631fc4b9 100644
--- a/src/cmd/compile/internal/ssa/_gen/386splitload.rules
+++ b/src/cmd/compile/internal/ssa/_gen/386splitload.rules
@@ -4,7 +4,7 @@
// See the top of AMD64splitload.rules for discussion of these rules.
-(CMP(L|W|B)load {sym} [off] ptr x mem) => (CMP(L|W|B) (MOV(L|W|B)load {sym} [off] ptr mem) x)
+(CMP(L|W|B)load {sym} [off] ptr x mem) => (CMP(L|W|B) (MOV(L|W|B)load {sym} [off] ptr mem) x)
(CMPLconstload {sym} [vo] ptr mem) => (CMPLconst (MOVLload {sym} [vo.Off()] ptr mem) [vo.Val()])
(CMPWconstload {sym} [vo] ptr mem) => (CMPWconst (MOVWload {sym} [vo.Off()] ptr mem) [vo.Val16()])
diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules
index 0bea99e38de1bc..36c1e3354d6f06 100644
--- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules
+++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules
@@ -27,15 +27,10 @@
(Div8u x y) => (Select0 (DIVWU (ZeroExt8to16 x) (ZeroExt8to16 y)))
(Div(32|64)F ...) => (DIVS(S|D) ...)
-(Select0 (Add64carry x y c)) =>
- (Select0 (ADCQ x y (Select1 (NEGLflags c))))
-(Select1 (Add64carry x y c)) =>
- (NEGQ (SBBQcarrymask (Select1 (ADCQ x y (Select1 (NEGLflags c))))))
-(Select0 (Sub64borrow x y c)) =>
- (Select0 (SBBQ x y (Select1 (NEGLflags c))))
-(Select1 (Sub64borrow x y c)) =>
- (NEGQ (SBBQcarrymask (Select1 (SBBQ x y (Select1 (NEGLflags c))))))
-
+(Select0 (Add64carry x y c)) => (Select0 (ADCQ x y (Select1 (NEGLflags c))))
+(Select1 (Add64carry x y c)) => (MOVBQZX (SETB (Select1 (ADCQ x y (Select1 (NEGLflags c))))))
+(Select0 (Sub64borrow x y c)) => (Select0 (SBBQ x y (Select1 (NEGLflags c))))
+(Select1 (Sub64borrow x y c)) => (MOVBQZX (SETB (Select1 (SBBQ x y (Select1 (NEGLflags c))))))
// Optimize ADCQ and friends
(ADCQ x (MOVQconst [c]) carry) && is32Bit(c) => (ADCQconst x [int32(c)] carry)
(ADCQ x y (FlagEQ)) => (ADDQcarry x y)
@@ -46,7 +41,13 @@
(SBBQconst x [c] (FlagEQ)) => (SUBQconstborrow x [c])
(SUBQborrow x (MOVQconst [c])) && is32Bit(c) => (SUBQconstborrow x [int32(c)])
(Select1 (NEGLflags (MOVQconst [0]))) => (FlagEQ)
-(Select1 (NEGLflags (NEGQ (SBBQcarrymask x)))) => x
+(Select1 (NEGLflags (MOVBQZX (SETB x)))) => x
+// Absorb InvertFlags into ADCQ/SBBQ: CF(InvertFlags(f)) = SETA(f), so
+// re-materialize the carry/borrow through the SETA+NEGLflags path.
+(ADCQ x y (InvertFlags f)) => (ADCQ x y (Select1 (NEGLflags (MOVBQZX (SETA f)))))
+(ADCQconst x [c] (InvertFlags f)) => (ADCQconst x [c] (Select1 (NEGLflags (MOVBQZX (SETA f)))))
+(SBBQ x y (InvertFlags f)) => (SBBQ x y (Select1 (NEGLflags (MOVBQZX (SETA f)))))
+(SBBQconst x [c] (InvertFlags f)) => (SBBQconst x [c] (Select1 (NEGLflags (MOVBQZX (SETA f)))))
(Mul64uhilo ...) => (MULQU2 ...)
@@ -124,6 +125,8 @@
(Ceil x) => (ROUNDSD [2] x)
(Trunc x) => (ROUNDSD [3] x)
+(CVTSD2SS (ROUNDSD [c] (CVTSS2SD x))) => (ROUNDSS [c] x)
+
(FMA x y z) => (VFMADD231SD z x y)
// Lowering extension
@@ -385,6 +388,7 @@
(ClosureCall ...) => (CALLclosure ...)
(InterCall ...) => (CALLinter ...)
(TailCall ...) => (CALLtail ...)
+(TailCallInter ...) => (CALLtailinter ...)
// Lowering conditional moves
// If the condition is a SETxx, we can just run a CMOV from the comparison that was
@@ -397,20 +401,30 @@
(CondSelect x y (SET(EQ|NE|L|G|LE|GE|A|B|AE|BE|EQF|NEF|GF|GEF) cond)) && is16BitInt(t)
=> (CMOVW(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS|EQF|NEF|GTF|GEF) y x cond)
-// If the condition does not set the flags, we need to generate a comparison.
-(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 1
- => (CondSelect x y (MOVBQZX check))
-(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 2
- => (CondSelect x y (MOVWQZX check))
-(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 4
- => (CondSelect x y (MOVLQZX check))
-
(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && (is64BitInt(t) || isPtr(t))
=> (CMOVQNE y x (CMPQconst [0] check))
(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && is32BitInt(t)
=> (CMOVLNE y x (CMPQconst [0] check))
(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 8 && is16BitInt(t)
=> (CMOVWNE y x (CMPQconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 4 && (is64BitInt(t) || isPtr(t))
+ => (CMOVQNE y x (CMPLconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 4 && is32BitInt(t)
+ => (CMOVLNE y x (CMPLconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 4 && is16BitInt(t)
+ => (CMOVWNE y x (CMPLconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 2 && (is64BitInt(t) || isPtr(t))
+ => (CMOVQNE y x (CMPWconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 2 && is32BitInt(t)
+ => (CMOVLNE y x (CMPWconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 2 && is16BitInt(t)
+ => (CMOVWNE y x (CMPWconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 1 && (is64BitInt(t) || isPtr(t))
+ => (CMOVQNE y x (CMPBconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 1 && is32BitInt(t)
+ => (CMOVLNE y x (CMPBconst [0] check))
+(CondSelect x y check) && !check.Type.IsFlags() && check.Type.Size() == 1 && is16BitInt(t)
+ => (CMOVWNE y x (CMPBconst [0] check))
// Absorb InvertFlags
(CMOVQ(EQ|NE|LT|GT|LE|GE|HI|CS|CC|LS) x y (InvertFlags cond))
@@ -588,30 +602,30 @@
// mutandis, for UGE and SETAE, and CC and SETCC.
((NE|EQ) (TESTL (SHLL (MOVLconst [1]) x) y)) => ((ULT|UGE) (BTL x y))
((NE|EQ) (TESTQ (SHLQ (MOVQconst [1]) x) y)) => ((ULT|UGE) (BTQ x y))
-((NE|EQ) (TESTLconst [c] x)) && isUnsignedPowerOfTwo(uint32(c))
+((NE|EQ) (TESTLconst [c] x)) && isPowerOfTwo(uint32(c))
=> ((ULT|UGE) (BTLconst [int8(log32u(uint32(c)))] x))
-((NE|EQ) (TESTQconst [c] x)) && isUnsignedPowerOfTwo(uint64(c))
+((NE|EQ) (TESTQconst [c] x)) && isPowerOfTwo(uint64(c))
=> ((ULT|UGE) (BTQconst [int8(log32u(uint32(c)))] x))
-((NE|EQ) (TESTQ (MOVQconst [c]) x)) && isUnsignedPowerOfTwo(uint64(c))
+((NE|EQ) (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(uint64(c))
=> ((ULT|UGE) (BTQconst [int8(log64u(uint64(c)))] x))
(SET(NE|EQ) (TESTL (SHLL (MOVLconst [1]) x) y)) => (SET(B|AE) (BTL x y))
(SET(NE|EQ) (TESTQ (SHLQ (MOVQconst [1]) x) y)) => (SET(B|AE) (BTQ x y))
-(SET(NE|EQ) (TESTLconst [c] x)) && isUnsignedPowerOfTwo(uint32(c))
+(SET(NE|EQ) (TESTLconst [c] x)) && isPowerOfTwo(uint32(c))
=> (SET(B|AE) (BTLconst [int8(log32u(uint32(c)))] x))
-(SET(NE|EQ) (TESTQconst [c] x)) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ) (TESTQconst [c] x)) && isPowerOfTwo(uint64(c))
=> (SET(B|AE) (BTQconst [int8(log32u(uint32(c)))] x))
-(SET(NE|EQ) (TESTQ (MOVQconst [c]) x)) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ) (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(uint64(c))
=> (SET(B|AE) (BTQconst [int8(log64u(uint64(c)))] x))
// SET..store variant
(SET(NE|EQ)store [off] {sym} ptr (TESTL (SHLL (MOVLconst [1]) x) y) mem)
=> (SET(B|AE)store [off] {sym} ptr (BTL x y) mem)
(SET(NE|EQ)store [off] {sym} ptr (TESTQ (SHLQ (MOVQconst [1]) x) y) mem)
=> (SET(B|AE)store [off] {sym} ptr (BTQ x y) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTLconst [c] x) mem) && isUnsignedPowerOfTwo(uint32(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTLconst [c] x) mem) && isPowerOfTwo(uint32(c))
=> (SET(B|AE)store [off] {sym} ptr (BTLconst [int8(log32u(uint32(c)))] x) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTQconst [c] x) mem) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTQconst [c] x) mem) && isPowerOfTwo(uint64(c))
=> (SET(B|AE)store [off] {sym} ptr (BTQconst [int8(log32u(uint32(c)))] x) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTQ (MOVQconst [c]) x) mem) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTQ (MOVQconst [c]) x) mem) && isPowerOfTwo(uint64(c))
=> (SET(B|AE)store [off] {sym} ptr (BTQconst [int8(log64u(uint64(c)))] x) mem)
// Handle bit-testing in the form (a>>b)&1 != 0 by building the above rules
@@ -637,14 +651,14 @@
(XOR(Q|L) (SHL(Q|L) (MOV(Q|L)const [1]) y) x) => (BTC(Q|L) x y)
// Note: only convert OR/XOR to BTS/BTC if the constant wouldn't fit in
// the constant field of the OR/XOR instruction. See issue 61694.
-((OR|XOR)Q (MOVQconst [c]) x) && isUnsignedPowerOfTwo(uint64(c)) && uint64(c) >= 1<<31 => (BT(S|C)Qconst [int8(log64u(uint64(c)))] x)
+((OR|XOR)Q (MOVQconst [c]) x) && isPowerOfTwo(uint64(c)) && uint64(c) >= 1<<31 => (BT(S|C)Qconst [int8(log64u(uint64(c)))] x)
// Recognize bit clearing: a &^= 1< (BTR(Q|L) x y)
(ANDN(Q|L) x (SHL(Q|L) (MOV(Q|L)const [1]) y)) => (BTR(Q|L) x y)
// Note: only convert AND to BTR if the constant wouldn't fit in
// the constant field of the AND instruction. See issue 61694.
-(ANDQ (MOVQconst [c]) x) && isUnsignedPowerOfTwo(uint64(^c)) && uint64(^c) >= 1<<31 => (BTRQconst [int8(log64u(uint64(^c)))] x)
+(ANDQ (MOVQconst [c]) x) && isPowerOfTwo(uint64(^c)) && uint64(^c) >= 1<<31 => (BTRQconst [int8(log64u(uint64(^c)))] x)
// Special-case bit patterns on first/last bit.
// generic.rules changes ANDs of high-part/low-part masks into a couple of shifts,
@@ -817,10 +831,6 @@
((SHRB|SARB)const x [0]) => x
((ROLQ|ROLL|ROLW|ROLB)const x [0]) => x
-// Multi-register shifts
-(ORQ (SH(R|L)Q lo bits) (SH(L|R)Q hi (NEGQ bits))) => (SH(R|L)DQ lo hi bits)
-(ORQ (SH(R|L)XQ lo bits) (SH(L|R)XQ hi (NEGQ bits))) => (SH(R|L)DQ lo hi bits)
-
// Note: the word and byte shifts keep the low 5 bits (not the low 4 or 3 bits)
// because the x86 instructions are defined to use all 5 bits of the shift even
// for the small shifts. I don't think we'll ever generate a weird shift (e.g.
@@ -899,7 +909,8 @@
(LEA(Q|L)2 [0] {s} (ADD(Q|L) x x) x) && s == nil => (SHL(Q|L)const [2] x)
// (x + x) << 2 -> x << 3 and similar
-(SHL(Q|L)const [c] (ADD(Q|L) x x)) => (SHL(Q|L)const [c+1] x)
+(SHLQconst [c] (ADDQ x x)) && c < 63 => (SHLQconst [c+1] x)
+(SHLLconst [c] (ADDL x x)) && c < 31 => (SHLLconst [c+1] x)
// reverse ordering of compare instruction
(SETL (InvertFlags x)) => (SETG x)
@@ -1276,20 +1287,13 @@
(SETAEstore [off] {sym} ptr (FlagGT_UGT) mem) => (MOVBstore [off] {sym} ptr (MOVLconst [1]) mem)
// Remove redundant *const ops
-(ADDQconst [0] x) => x
-(ADDLconst [c] x) && c==0 => x
-(SUBQconst [0] x) => x
-(SUBLconst [c] x) && c==0 => x
-(ANDQconst [0] _) => (MOVQconst [0])
-(ANDLconst [c] _) && c==0 => (MOVLconst [0])
-(ANDQconst [-1] x) => x
-(ANDLconst [c] x) && c==-1 => x
-(ORQconst [0] x) => x
-(ORLconst [c] x) && c==0 => x
-(ORQconst [-1] _) => (MOVQconst [-1])
-(ORLconst [c] _) && c==-1 => (MOVLconst [-1])
-(XORQconst [0] x) => x
-(XORLconst [c] x) && c==0 => x
+(ADD(Q|L)const [0] x) => x
+(SUB(Q|L)const [0] x) => x
+(AND(Q|L)const [0] _) => (MOVLconst [0])
+(AND(Q|L)const [-1] x) => x
+(OR(Q|L)const [0] x) => x
+(OR(Q|L)const [-1] _) => (MOV(Q|L)const [-1])
+(XOR(Q|L)const [0] x) => x
// TODO: since we got rid of the W/B versions, we might miss
// things like (ANDLconst [0x100] x) which were formerly
// (ANDBconst [0] x). Probably doesn't happen very often.
@@ -1317,8 +1321,7 @@
(SARLconst [c] (MOVQconst [d])) => (MOVQconst [int64(int32(d))>>uint64(c)])
(SARWconst [c] (MOVQconst [d])) => (MOVQconst [int64(int16(d))>>uint64(c)])
(SARBconst [c] (MOVQconst [d])) => (MOVQconst [int64(int8(d))>>uint64(c)])
-(NEGQ (MOVQconst [c])) => (MOVQconst [-c])
-(NEGL (MOVLconst [c])) => (MOVLconst [-c])
+(NEG(Q|L) (MOV(Q|L)const [c])) => (MOV(Q|L)const [-c])
(MULQconst [c] (MOVQconst [d])) => (MOVQconst [int64(c)*d])
(MULLconst [c] (MOVLconst [d])) => (MOVLconst [c*d])
(ANDQconst [c] (MOVQconst [d])) => (MOVQconst [int64(c)&d])
@@ -1327,8 +1330,7 @@
(ORLconst [c] (MOVLconst [d])) => (MOVLconst [c|d])
(XORQconst [c] (MOVQconst [d])) => (MOVQconst [int64(c)^d])
(XORLconst [c] (MOVLconst [d])) => (MOVLconst [c^d])
-(NOTQ (MOVQconst [c])) => (MOVQconst [^c])
-(NOTL (MOVLconst [c])) => (MOVLconst [^c])
+(NOT(Q|L) (MOV(Q|L)const [c])) => (MOV(Q|L)const [^c])
(BTSQconst [c] (MOVQconst [d])) => (MOVQconst [d|(1< (MOVQconst [d&^(1< (MOVQconst [d^(1< (SUBQ x y)
-(ADDL x (NEGL y)) => (SUBL x y)
-(SUBQ x x) => (MOVQconst [0])
-(SUBL x x) => (MOVLconst [0])
-(ANDQ x x) => x
-(ANDL x x) => x
-(ORQ x x) => x
-(ORL x x) => x
-(XORQ x x) => (MOVQconst [0])
-(XORL x x) => (MOVLconst [0])
+(ADD(Q|L) x (NEG(Q|L) y)) => (SUB(Q|L) x y)
+(SUB(Q|L) x x) => (MOVLconst [0])
+(AND(Q|L) x x) => x
+(OR(Q|L) x x) => x
+(XOR(Q|L) x x) => (MOVLconst [0])
(SHLLconst [d] (MOVLconst [c])) => (MOVLconst [c << uint64(d)])
(SHLQconst [d] (MOVQconst [c])) => (MOVQconst [c << uint64(d)])
@@ -1361,10 +1358,7 @@
(MULQconst [c] (NEGQ x)) && c != -(1<<31) => (MULQconst [-c] x)
// checking AND against 0.
-(CMPQconst a:(ANDQ x y) [0]) && a.Uses == 1 => (TESTQ x y)
-(CMPLconst a:(ANDL x y) [0]) && a.Uses == 1 => (TESTL x y)
-(CMPWconst a:(ANDL x y) [0]) && a.Uses == 1 => (TESTW x y)
-(CMPBconst a:(ANDL x y) [0]) && a.Uses == 1 => (TESTB x y)
+(CMP(Q|L|W|B)const a:(AND(Q|L|L|L) x y) [0]) && a.Uses == 1 => (TEST(Q|L|W|B) x y)
(CMPQconst a:(ANDQconst [c] x) [0]) && a.Uses == 1 => (TESTQconst [c] x)
(CMPLconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 => (TESTLconst [c] x)
(CMPWconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 => (TESTWconst [int16(c)] x)
@@ -1377,10 +1371,7 @@
(TESTB (MOVLconst [c]) x) => (TESTBconst [int8(c)] x)
// TEST %reg,%reg is shorter than CMP
-(CMPQconst x [0]) => (TESTQ x x)
-(CMPLconst x [0]) => (TESTL x x)
-(CMPWconst x [0]) => (TESTW x x)
-(CMPBconst x [0]) => (TESTB x x)
+(CMP(Q|L|W|B)const x [0]) => (TEST(Q|L|W|B) x x)
(TESTQconst [-1] x) && x.Op != OpAMD64MOVQconst => (TESTQ x x)
(TESTLconst [-1] x) && x.Op != OpAMD64MOVLconst => (TESTL x x)
(TESTWconst [-1] x) && x.Op != OpAMD64MOVLconst => (TESTW x x)
@@ -1568,7 +1559,8 @@
(MOVBload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read8(sym, int64(off)))])
(MOVWload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read16(sym, int64(off), config.ctxt.Arch.ByteOrder))])
-(MOVLload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read32(sym, int64(off), config.ctxt.Arch.ByteOrder))])
+(MOVLload [off] {sym} (SB) _) && symIsRO(sym) && is32BitInt(t) => (MOVLconst [int32(read32(sym, int64(off), config.ctxt.Arch.ByteOrder))])
+(MOVLload