net/textproto: add Referer to commonHeader

Change-Id: Id4c95138788d90779aa7b9a54e58f9e4cb271163
GitHub-Last-Rev: ee43b95269
GitHub-Pull-Request: golang/go#65985
Reviewed-on: https://go-review.googlesource.com/c/go/+/567539
Reviewed-by: Sean Liao <sean@liao.dev>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Damien Neil <dneil@google.com>
Reviewed-by: Junyang Shao <shaojunyang@google.com>

VERSION

@@ -1,2 +0,0 @@
-go1.26rc1
-time 2025-12-15T22:30:00Z

api/except.txt

@@ -503,6 +503,7 @@ pkg unicode, const Version = "10.0.0"
 pkg unicode, const Version = "11.0.0"
 pkg unicode, const Version = "12.0.0"
 pkg unicode, const Version = "13.0.0"
+pkg unicode, const Version = "15.0.0"
 pkg unicode, const Version = "6.2.0"
 pkg unicode, const Version = "6.3.0"
 pkg unicode, const Version = "7.0.0"

api/next/74958.txt

@@ -0,0 +1 @@
+pkg go/scanner, method (*Scanner) End() token.Pos #74958

api/next/75500.txt

@@ -0,0 +1 @@
+pkg net/http, type Server struct, DisableClientPriority bool #75500

api/next/77169.txt

@@ -0,0 +1 @@
+pkg testing/synctest, func Sleep(time.Duration) #77169

api/next/77266.txt

@@ -0,0 +1,16 @@
+pkg unicode, const Version = "17.0.0" #77266
+pkg unicode, var Beria_Erfe *RangeTable #77266
+pkg unicode, var Garay *RangeTable #77266
+pkg unicode, var Gurung_Khema *RangeTable #77266
+pkg unicode, var IDS_Unary_Operator *RangeTable #77266
+pkg unicode, var ID_Compat_Math_Continue *RangeTable #77266
+pkg unicode, var ID_Compat_Math_Start *RangeTable #77266
+pkg unicode, var Kirat_Rai *RangeTable #77266
+pkg unicode, var Modifier_Combining_Mark *RangeTable #77266
+pkg unicode, var Ol_Onal *RangeTable #77266
+pkg unicode, var Sidetic *RangeTable #77266
+pkg unicode, var Sunuwar *RangeTable #77266
+pkg unicode, var Tai_Yo *RangeTable #77266
+pkg unicode, var Todhri *RangeTable #77266
+pkg unicode, var Tolong_Siki *RangeTable #77266
+pkg unicode, var Tulu_Tigalari *RangeTable #77266

codereview.cfg

@@ -1,2 +1 @@
-branch: release-branch.go1.26
-parent-branch: master
+branch: master

doc/go_spec.html

@@ -1,6 +1,6 @@
 <!--{
 	"Title": "The Go Programming Language Specification",
-	"Subtitle": "Language version go1.26 (Dec 2, 2025)",
+	"Subtitle": "Language version go1.26 (Jan 12, 2026)",
 	"Path": "/ref/spec"
 }-->
 
@@ -3143,7 +3143,8 @@ math.Sin // denotes the Sin function in package math
 <p>
 Composite literals construct new values for structs, arrays, slices, and maps
 each time they are evaluated.
-They consist of the type of the literal followed by a brace-bound list of elements.
+They consist of the type of the literal followed by a (possibly empty)
+brace-bound list of elements.
 Each element may optionally be preceded by a corresponding key.
 </p>
 
@@ -3168,35 +3169,53 @@ as a TypeName).
 If the LiteralType is a type parameter, all types in its type set
 must have the same underlying type which must be
 a valid composite literal type.
-The types of the elements and keys must be <a href="#Assignability">assignable</a>
-to the respective field, element, and key types of type <code>T</code>;
-there is no additional conversion.
-The key is interpreted as a field name for struct literals,
-an index for array and slice literals, and a key for map literals.
-For map literals, all elements must have a key. It is an error
-to specify multiple elements with the same field name or
-constant key value. For non-constant map keys, see the section on
-<a href="#Order_of_evaluation">evaluation order</a>.
 </p>
 
 <p>
-For struct literals the following rules apply:
+The types of the elements and keys must be <a href="#Assignability">assignable</a>
+to the respective field, element, and key types of the LiteralType;
+there is no additional conversion.
+The key is interpreted as a field name for struct literals,
+an index for array and slice literals, and a key for map literals.
+It is an error to specify multiple elements with the same field name
+or constant key value.
+A literal may omit the element list; such a literal evaluates
+to the zero value for its type.
+</p>
+
+<p>
+A parsing ambiguity arises when a composite literal using the
+TypeName form of the LiteralType appears as an operand between the
+<a href="#Keywords">keyword</a> and the opening brace of the block
+of an "if", "for", or "switch" statement, and the composite literal
+is not enclosed in parentheses, square brackets, or curly braces.
+In this rare case, the opening brace of the literal is erroneously parsed
+as the one introducing the block of statements. To resolve the ambiguity,
+the composite literal must appear within parentheses.
+</p>
+
+<pre>
+if x == (T{a,b,c}[i]) { … }
+if (x == T{a,b,c}[i]) { … }
+</pre>
+
+<h4>Struct literals</h4>
+
+<p>
+For struct literals without keys, the element list must contain an element
+for each struct field in the order in which the fields are declared.
+</p>
+
+<p>
+For struct literals with keys the following rules apply:
 </p>
 <ul>
-	<li>A key must be a field name declared in the struct type.
+	<li>Every element must have a key.
 	</li>
-	<li>An element list that does not contain any keys must
-	    list an element for each struct field in the
-	    order in which the fields are declared.
+	<li>Each key must be a field name declared in the struct type.
 	</li>
-	<li>If any element has a key, every element must have a key.
-	</li>
-	<li>An element list that contains keys does not need to
-	    have an element for each struct field. Omitted fields
-	    get the zero value for that field.
-	</li>
-	<li>A literal may omit the element list; such a literal evaluates
-	    to the zero value for its type.
+	<li>The element list does not need to have an element for each struct field.
+	    Omitted fields get the zero value for that field.
 	</li>
 	<li>It is an error to specify an element for a non-exported
 	    field of a struct belonging to a different package.
@@ -3220,6 +3239,8 @@ origin := Point3D{}                            // zero value for Point3D
 line := Line{origin, Point3D{y: -4, z: 12.3}}  // zero value for line.q.x
 </pre>
 
+<h4>Array and slice literals</h4>
+
 <p>
 For array and slice literals the following rules apply:
 </p>
@@ -3295,6 +3316,16 @@ tmp := [n]T{x1, x2, … xn}
 tmp[0 : n]
 </pre>
 
+<h4>Map literals</h4>
+
+<p>
+For map literals, each element must have a key.
+For non-constant map keys, see the section on
+<a href="#Order_of_evaluation">evaluation order</a>.
+</p>
+
+<h4>Elision of element types</h4>
+
 <p>
 Within a composite literal of array, slice, or map type <code>T</code>,
 elements or map keys that are themselves composite literals may elide the respective
@@ -3315,22 +3346,6 @@ type PPoint *Point
 [2]PPoint{{1.5, -3.5}, {}}          // same as [2]PPoint{PPoint(&amp;Point{1.5, -3.5}), PPoint(&amp;Point{})}
 </pre>
 
-<p>
-A parsing ambiguity arises when a composite literal using the
-TypeName form of the LiteralType appears as an operand between the
-<a href="#Keywords">keyword</a> and the opening brace of the block
-of an "if", "for", or "switch" statement, and the composite literal
-is not enclosed in parentheses, square brackets, or curly braces.
-In this rare case, the opening brace of the literal is erroneously parsed
-as the one introducing the block of statements. To resolve the ambiguity,
-the composite literal must appear within parentheses.
-</p>
-
-<pre>
-if x == (T{a,b,c}[i]) { … }
-if (x == T{a,b,c}[i]) { … }
-</pre>
-
 <p>
 Examples of valid array, slice, and map literals:
 </p>

doc/godebug.md

@@ -163,6 +163,13 @@ will fail early. The default value is `httpcookiemaxnum=3000`. Setting
 number of cookies. To avoid denial of service attacks, this setting and default
 was backported to Go 1.25.2 and Go 1.24.8.
 
+Go 1.26 added a new `urlmaxqueryparams` setting that controls the maximum number
+of query parameters that net/url will accept when parsing a URL-encoded query string.
+If the number of parameters exceeds the number set in `urlmaxqueryparams`,
+parsing will fail early. The default value is `urlmaxqueryparams=10000`.
+Setting `urlmaxqueryparams=0` disables the limit. To avoid denial of service
+attacks, this setting and default was backported to Go 1.25.6 and Go 1.24.12.
+
 Go 1.26 added a new `urlstrictcolons` setting that controls whether `net/url.Parse`
 allows malformed hostnames containing colons outside of a bracketed IPv6 address.
 The default `urlstrictcolons=1` rejects URLs such as `http://localhost:1:2` or `http://::1/`.

doc/next/1-intro.md

@@ -0,0 +1,8 @@
+<style>
+  main ul li { margin: 0.5em 0; }
+</style>
+
+## DRAFT RELEASE NOTES — Introduction to Go 1.27 {#introduction}
+
+**Go 1.27 is not yet released. These are work-in-progress release notes.
+Go 1.27 is expected to be released in August 2026.**

doc/next/2-language.md

@@ -0,0 +1,3 @@
+## Changes to the language {#language}
+
+

doc/next/3-tools.md

@@ -0,0 +1,6 @@
+## Tools {#tools}
+
+### Go command {#go-command}
+
+### Cgo {#cgo}
+

doc/next/4-runtime.md

@@ -0,0 +1 @@
+## Runtime {#runtime}

doc/next/5-toolchain.md

@@ -0,0 +1,7 @@
+## Compiler {#compiler}
+
+## Assembler {#assembler}
+
+## Linker {#linker}
+
+

doc/next/6-stdlib/0-heading.md

@@ -0,0 +1,2 @@
+## Standard library {#library}
+

doc/next/6-stdlib/99-minor/0-heading.md

@@ -0,0 +1 @@
+### Minor changes to the library {#minor_library_changes}

doc/next/6-stdlib/99-minor/README

@@ -0,0 +1 @@
+API changes and other small changes to the standard library go here.

doc/next/6-stdlib/99-minor/go/scanner/74958.md

@@ -0,0 +1 @@
+The scanner now allows retrieving the end position of a token via the new [Scanner.End] method.

doc/next/6-stdlib/99-minor/net/http/75500.md

@@ -0,0 +1,4 @@
+HTTP/2 server now accepts client priority signals, as defined in RFC 9218,
+allowing it to prioritize serving HTTP/2 streams with higher priority. If the
+old behavior is preferred, where streams are served in a round-robin manner
+regardless of priority, [Server.DisableClientPriority] can be set to `true`.

doc/next/6-stdlib/99-minor/testing/synctest/77169.md

@@ -0,0 +1 @@
+The new [Sleep] helper function combines [time.Sleep] and [testing/synctest.Wait].

doc/next/6-stdlib/99-minor/unicode/77266.md

@@ -0,0 +1,4 @@
+The unicode package and associated support throughout the system has been upgraded from Unicode 15 to Unicode 17.
+See the [Unicode 16.0.0](https://www.unicode.org/versions/Unicode16.0.0/) and
+[Unicode 17.0.0](https://www.unicode.org/versions/Unicode17.0.0/)
+release notes for information about the changes.

doc/next/7-ports.md

@@ -0,0 +1,8 @@
+## Ports {#ports}
+
+### Darwin {#darwin}
+
+<!-- go.dev/issue/75836 -->
+As [announced](go1.26#darwin) in the Go 1.26 release notes,
+Go 1.27 requires macOS 13 Ventura or later;
+support for previous versions has been discontinued.

lib/fips140/fips140.sum

@@ -10,4 +10,4 @@
 #	go test cmd/go/internal/fips140 -update
 #
 v1.0.0-c2097c7c.zip daf3614e0406f67ae6323c902db3f953a1effb199142362a039e7526dfb9368b
-v1.1.0-rc1.zip ea94f8c3885294c9efe1bd8f9b6e86daeb25b6aff2aeb20707cd9a5101f6f54e
+v1.26.0.zip 9b28f847fdf1db4a36cb2b2f8ec09443c039383f085630a03ecfaddf6db7ea23

src/README.vendor

@@ -39,12 +39,14 @@ Go from source and use that 'go' binary to update its source tree.
 
 Requirements may be added, updated, and removed with 'go get'.
 The vendor directory may be updated with 'go mod vendor'.
+Tree inconsistencies are reported by 'go test cmd/internal/moddeps'.
 A typical sequence might be:
 
     cd src  # or src/cmd
     go get golang.org/x/net@master
     go mod tidy
     go mod vendor
+    go test cmd/internal/moddeps
 
 Use caution when passing '-u' to 'go get'. The '-u' flag updates
 modules providing all transitively imported packages, not only
@@ -53,3 +55,22 @@ the module providing the target package.
 Note that 'go mod vendor' only copies packages that are transitively
 imported by packages in the current module. If a new package is needed,
 it should be imported before running 'go mod vendor'.
+
+Go release cycle considerations
+===============================
+
+Applying changes to packages that are vendored follows the considerations
+written down at go.dev/s/release.
+
+When the Go tree is open for development, a specific change may be pulled in
+at any time that it is needed. During the release freeze, the bar for changes
+in vendored packages is the same as it is for changes in non-vendored packages.
+After a major release is out, minor Go releases follow a more involved process
+documented at go.dev/wiki/MinorReleases#cherry-pick-cls-for-vendored-golangorgx-packages.
+
+In addition to individual updates that happen on demand, all dependencies in
+the std and cmd modules are updated to their latest available versions at least
+twice during every major release cycle. This is done to avoid the possibility of
+some dependencies being left on very old versions and in turn make their eventual
+update more disruptive. This recurring process is tracked in go.dev/issue/36905.
+The golang.org/x/build/cmd/updatestd command exists to assist with that process.

src/archive/tar/reader_test.go

@@ -787,7 +787,7 @@ type readBadSeeker struct{ io.ReadSeeker }
 
 func (rbs *readBadSeeker) Seek(int64, int) (int64, error) { return 0, fmt.Errorf("illegal seek") }
 
-// TestReadTruncation test the ending condition on various truncated files and
+// TestReadTruncation tests the ending condition on various truncated files and
 // that truncated files are still detected even if the underlying io.Reader
 // satisfies io.Seeker.
 func TestReadTruncation(t *testing.T) {

src/archive/tar/stat_unix.go

@@ -19,7 +19,7 @@ func init() {
 	sysStat = statUnix
 }
 
-// userMap and groupMap caches UID and GID lookups for performance reasons.
+// userMap and groupMap cache UID and GID lookups for performance reasons.
 // The downside is that renaming uname or gname by the OS never takes effect.
 var userMap, groupMap sync.Map // map[int]string
 

src/archive/tar/strconv.go

@@ -312,7 +312,7 @@ func formatPAXRecord(k, v string) (string, error) {
 //	"%d %s=%s\n" % (size, key, value)
 //
 // Keys and values should be UTF-8, but the number of bad writers out there
-// forces us to be a more liberal.
+// forces us to be more liberal.
 // Thus, we only reject all keys with NUL, and only reject NULs in values
 // for the PAX version of the USTAR string fields.
 // The key must not contain an '=' character.

src/archive/zip/reader.go

@@ -834,7 +834,16 @@ func (r *Reader) initFileList() {
 				continue
 			}
 
-			for dir := path.Dir(name); dir != "."; dir = path.Dir(dir) {
+			dir := name
+			for {
+				if idx := strings.LastIndex(dir, "/"); idx < 0 {
+					break
+				} else {
+					dir = dir[:idx]
+				}
+				if dirs[dir] {
+					break
+				}
 				dirs[dir] = true
 			}
 

src/archive/zip/reader_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"internal/obscuretestdata"
 	"io"
 	"io/fs"
@@ -1874,3 +1875,83 @@ func TestBaseOffsetPlusOverflow(t *testing.T) {
 	// as the section reader offset & size were < 0.
 	NewReader(bytes.NewReader(data), int64(len(data))+1875)
 }
+
+func BenchmarkReaderOneDeepDir(b *testing.B) {
+	var buf bytes.Buffer
+	zw := NewWriter(&buf)
+
+	for i := range 4000 {
+		name := strings.Repeat("a/", i) + "data"
+		zw.CreateHeader(&FileHeader{
+			Name:   name,
+			Method: Store,
+		})
+	}
+
+	if err := zw.Close(); err != nil {
+		b.Fatal(err)
+	}
+	data := buf.Bytes()
+
+	for b.Loop() {
+		zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			b.Fatal(err)
+		}
+		zr.Open("does-not-exist")
+	}
+}
+
+func BenchmarkReaderManyDeepDirs(b *testing.B) {
+	var buf bytes.Buffer
+	zw := NewWriter(&buf)
+
+	for i := range 2850 {
+		name := fmt.Sprintf("%x", i)
+		name = strings.Repeat("/"+name, i+1)[1:]
+
+		zw.CreateHeader(&FileHeader{
+			Name:   name,
+			Method: Store,
+		})
+	}
+
+	if err := zw.Close(); err != nil {
+		b.Fatal(err)
+	}
+	data := buf.Bytes()
+
+	for b.Loop() {
+		zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			b.Fatal(err)
+		}
+		zr.Open("does-not-exist")
+	}
+}
+
+func BenchmarkReaderManyShallowFiles(b *testing.B) {
+	var buf bytes.Buffer
+	zw := NewWriter(&buf)
+
+	for i := range 310000 {
+		name := fmt.Sprintf("%v", i)
+		zw.CreateHeader(&FileHeader{
+			Name:   name,
+			Method: Store,
+		})
+	}
+
+	if err := zw.Close(); err != nil {
+		b.Fatal(err)
+	}
+	data := buf.Bytes()
+
+	for b.Loop() {
+		zr, err := NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			b.Fatal(err)
+		}
+		zr.Open("does-not-exist")
+	}
+}

src/bytes/bytes.go

@@ -246,7 +246,7 @@ func IndexAny(s []byte, chars string) int {
 		}
 		return IndexRune(s, r)
 	}
-	if len(s) > 8 {
+	if shouldUseASCIISet(len(s)) {
 		if as, isASCII := makeASCIISet(chars); isASCII {
 			for i, c := range s {
 				if as.contains(c) {
@@ -301,7 +301,7 @@ func LastIndexAny(s []byte, chars string) int {
 		// Avoid scanning all of s.
 		return -1
 	}
-	if len(s) > 8 {
+	if shouldUseASCIISet(len(s)) {
 		if as, isASCII := makeASCIISet(chars); isASCII {
 			for i := len(s) - 1; i >= 0; i-- {
 				if as.contains(s[i]) {
@@ -935,15 +935,22 @@ func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int {
 	return -1
 }
 
-// asciiSet is a 32-byte value, where each bit represents the presence of a
-// given ASCII character in the set. The 128-bits of the lower 16 bytes,
-// starting with the least-significant bit of the lowest word to the
-// most-significant bit of the highest word, map to the full range of all
-// 128 ASCII characters. The 128-bits of the upper 16 bytes will be zeroed,
-// ensuring that any non-ASCII character will be reported as not in the set.
-// This allocates a total of 32 bytes even though the upper half
-// is unused to avoid bounds checks in asciiSet.contains.
-type asciiSet [8]uint32
+// asciiSet is a 256-byte lookup table for fast ASCII character membership testing.
+// Each element corresponds to an ASCII character value, with true indicating the
+// character is in the set. Using bool instead of byte allows the compiler to
+// eliminate the comparison instruction, as bool values are guaranteed to be 0 or 1.
+//
+// The full 256-element table is used rather than a 128-element table to avoid
+// additional operations in the lookup path. Alternative approaches were tested:
+//   - [128]bool with explicit bounds check (if c >= 128): introduces branches
+//     that cause pipeline stalls, resulting in ~70% slower performance
+//   - [128]bool with masking (c&0x7f): eliminates bounds checks but the AND
+//     operation still costs ~10% performance compared to direct indexing
+//
+// The 256-element array allows direct indexing with no bounds checks, no branches,
+// and no masking operations, providing optimal performance. The additional 128 bytes
+// of memory is a worthwhile tradeoff for the simpler, faster code.
+type asciiSet [256]bool
 
 // makeASCIISet creates a set of ASCII characters and reports whether all
 // characters in chars are ASCII.
@@ -953,14 +960,24 @@ func makeASCIISet(chars string) (as asciiSet, ok bool) {
 		if c >= utf8.RuneSelf {
 			return as, false
 		}
-		as[c/32] |= 1 << (c % 32)
+		as[c] = true
 	}
 	return as, true
 }
 
 // contains reports whether c is inside the set.
 func (as *asciiSet) contains(c byte) bool {
-	return (as[c/32] & (1 << (c % 32))) != 0
+	return as[c]
+}
+
+// shouldUseASCIISet returns whether to use the lookup table optimization.
+// The threshold of 8 bytes balances initialization cost against per-byte
+// search cost, performing well across all charset sizes.
+//
+// More complex heuristics (e.g., different thresholds per charset size)
+// add branching overhead that eats away any theoretical improvements.
+func shouldUseASCIISet(bufLen int) bool {
+	return bufLen > 8
 }
 
 // containsRune is a simplified version of strings.ContainsRune

src/bytes/bytes_test.go

@@ -961,7 +961,7 @@ func TestSplit(t *testing.T) {
 		if tt.n < 0 {
 			b := sliceOfString(Split([]byte(tt.s), []byte(tt.sep)))
 			if !slices.Equal(result, b) {
-				t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
+				t.Errorf("Split disagrees with SplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
 			}
 		}
 		if len(a) > 0 {
@@ -1023,7 +1023,7 @@ func TestSplitAfter(t *testing.T) {
 		if tt.n < 0 {
 			b := sliceOfString(SplitAfter([]byte(tt.s), []byte(tt.sep)))
 			if !slices.Equal(result, b) {
-				t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
+				t.Errorf("SplitAfter disagrees with SplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a)
 			}
 		}
 	}

src/cmd/asm/internal/asm/endtoend_test.go

@@ -199,6 +199,11 @@ Diff:
 	}
 	obj.Flushplist(ctxt, pList, nil)
 
+	if !ok {
+		// If we've encountered errors, the output is unlikely to be sane.
+		t.FailNow()
+	}
+
 	for p := top; p != nil; p = p.Link {
 		if p.As == obj.ATEXT {
 			text = p.From.Sym
@@ -486,16 +491,35 @@ func TestPPC64EndToEnd(t *testing.T) {
 	}
 }
 
-func TestRISCVEndToEnd(t *testing.T) {
-	testEndToEnd(t, "riscv64", "riscv64")
+func testRISCV64AllProfiles(t *testing.T, testFn func(t *testing.T)) {
+	t.Helper()
+
+	defer func(orig int) { buildcfg.GORISCV64 = orig }(buildcfg.GORISCV64)
+
+	for _, goriscv64 := range []int{20, 22, 23} {
+		t.Run(fmt.Sprintf("rva%vu64", goriscv64), func(t *testing.T) {
+			buildcfg.GORISCV64 = goriscv64
+			testFn(t)
+		})
+	}
 }
 
-func TestRISCVErrors(t *testing.T) {
-	testErrors(t, "riscv64", "riscv64error")
+func TestRISCV64EndToEnd(t *testing.T) {
+	testRISCV64AllProfiles(t, func(t *testing.T) {
+		testEndToEnd(t, "riscv64", "riscv64")
+	})
 }
 
-func TestRISCVValidation(t *testing.T) {
-	testErrors(t, "riscv64", "riscv64validation")
+func TestRISCV64Errors(t *testing.T) {
+	testRISCV64AllProfiles(t, func(t *testing.T) {
+		testErrors(t, "riscv64", "riscv64error")
+	})
+}
+
+func TestRISCV64Validation(t *testing.T) {
+	testRISCV64AllProfiles(t, func(t *testing.T) {
+		testErrors(t, "riscv64", "riscv64validation")
+	})
 }
 
 func TestS390XEndToEnd(t *testing.T) {

src/cmd/cgo/ast.go

@@ -301,17 +301,12 @@ func (f *File) saveExport(x any, context astContext) {
 			error_(c.Pos(), "export comment has wrong name %q, want %q", name, n.Name.Name)
 		}
 
-		doc := ""
-		for _, c1 := range n.Doc.List {
-			if c1 != c {
-				doc += c1.Text + "\n"
-			}
-		}
-
 		f.ExpFunc = append(f.ExpFunc, &ExpFunc{
 			Func:    n,
 			ExpName: name,
-			Doc:     doc,
+			// Caution: Do not set the Doc field on purpose
+			// to ensure that there are no unintended artifacts
+			// in the binary. See https://go.dev/issue/76697.
 		})
 		break
 	}

src/cmd/cgo/gcc.go

@@ -24,6 +24,7 @@ import (
 	"math"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
@@ -1795,7 +1796,7 @@ var n atomic.Int64
 
 func gccTmp() string {
 	c := strconv.Itoa(int(n.Add(1)))
-	return *objDir + "_cgo_" + c + ".o"
+	return filepath.Join(outputDir(), "_cgo_"+c+".o")
 }
 
 // gccCmd returns the gcc command line to use for compiling

src/cmd/cgo/internal/test/issue76861.go

@@ -0,0 +1,12 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build cgo
+
+package cgotest
+
+// Issue 43639: No runtime test needed, make sure package
+// cmd/cgo/internal/test/issue76861 compiles without error.
+
+import _ "cmd/cgo/internal/test/issue76861"

src/cmd/cgo/internal/test/issue76861/a.go

@@ -0,0 +1,13 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package issue76861
+
+// #cgo CFLAGS: -Wall -Werror
+// void issue76861(void) {}
+import "C"
+
+func Issue76861() {
+	C.issue76861()
+}

src/cmd/cgo/internal/testerrors/errors_test.go

@@ -72,7 +72,7 @@ func expect(t *testing.T, errors []*regexp.Regexp, files ...string) {
 	defer os.RemoveAll(dir)
 
 	dst := filepath.Join(dir, strings.TrimSuffix(files[0], ".go"))
-	args := []string{"build", "-gcflags=-L -e", "-o=" + dst} // TODO(gri) no need for -gcflags=-L if go tool is adjusted
+	args := []string{"build", "-gcflags=-e", "-o=" + dst}
 	for _, file := range files {
 		args = append(args, path(file))
 	}

src/cmd/cgo/internal/testout/out_test.go

@@ -18,6 +18,32 @@ import (
 	"testing"
 )
 
+// TestDisallowSmuggledCode tests that
+// docstrings do not smuggle code into
+// files generated by Cgo.
+func TestDisallowSmuggledCode(t *testing.T) {
+	testenv.MustHaveGoRun(t)
+	testenv.MustHaveCGO(t)
+	objDir := cgo(t, "comments.go")
+
+	file, err := os.Open(filepath.Join(objDir, "_cgo_export.h"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if strings.Contains(line, `"Hello, I am exploiting CVE-2025-61732!\n"`) {
+			t.Fatalf(`got %q, want ""`, line)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 type methodAlign struct {
 	Method string
 	Align  int
@@ -43,23 +69,7 @@ var wantAligns = map[string]int{
 func TestAligned(t *testing.T) {
 	testenv.MustHaveGoRun(t)
 	testenv.MustHaveCGO(t)
-
-	testdata, err := filepath.Abs("testdata")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	objDir := t.TempDir()
-
-	cmd := testenv.Command(t, testenv.GoToolPath(t), "tool", "cgo",
-		"-objdir", objDir,
-		filepath.Join(testdata, "aligned.go"))
-	cmd.Stderr = new(bytes.Buffer)
-
-	err = cmd.Run()
-	if err != nil {
-		t.Fatalf("%#q: %v\n%s", cmd, err, cmd.Stderr)
-	}
+	objDir := cgo(t, "aligned.go")
 
 	haveAligns, err := parseAlign(filepath.Join(objDir, "_cgo_export.c"))
 	if err != nil {
@@ -84,6 +94,28 @@ func TestAligned(t *testing.T) {
 	}
 }
 
+// cgo executes 'go tool cgo' on testFile
+// and returns the objdir containing the
+// generated files.
+func cgo(t *testing.T, testFile string) string {
+	objDir := t.TempDir()
+	testdata, err := filepath.Abs("testdata")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmd := testenv.Command(t, testenv.GoToolPath(t), "tool", "cgo",
+		"-objdir", objDir,
+		filepath.Join(testdata, testFile))
+
+	cmd.Stderr = new(bytes.Buffer)
+	if err = cmd.Run(); err != nil {
+		t.Fatalf("%#q: %v\n%s", cmd, err, cmd.Stderr)
+	}
+
+	return objDir
+}
+
 func parseAlign(filename string) ([]methodAlign, error) {
 	file, err := os.Open(filename)
 	if err != nil {

src/cmd/cgo/internal/testout/testdata/comments.go

@@ -0,0 +1,47 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+/*
+#include <stdio.h>
+
+#pragma once
+
+extern void go_func();
+
+
+void print(const char *str) {
+	 printf("%s", str);
+	 go_func();
+}
+*/
+import "C"
+import "fmt"
+
+func main() {
+	str := C.CString("Hello from C\n")
+	C.print(str)
+}
+
+// \
+/*
+
+#ifndef AUTO_PRINT_H
+#define AUTO_PRINT_H
+
+#include <stdio.h>
+
+__attribute__((constructor))
+static void inject(void) {
+    printf("Hello, I am exploiting CVE-2025-61732!\n");
+}
+
+#endif
+
+/* */
+//export go_func
+func go_func() {
+	fmt.Println("Hello from Go")
+}

src/cmd/cgo/main.go

@@ -320,6 +320,10 @@ func main() {
 		conf.Mode &^= printer.SourcePos
 	}
 
+	if *objDir == "" {
+		*objDir = "_obj"
+	}
+
 	args := flag.Args()
 	if len(args) < 1 {
 		usage()
@@ -446,14 +450,6 @@ func main() {
 
 	cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6])
 
-	if *objDir == "" {
-		*objDir = "_obj"
-	}
-	// make sure that `objDir` directory exists, so that we can write
-	// all the output files there.
-	os.MkdirAll(*objDir, 0o700)
-	*objDir += string(filepath.Separator)
-
 	for i, input := range goFiles {
 		f := fs[i]
 		p.Translate(f)

src/cmd/cgo/out.go

@@ -33,15 +33,15 @@ var (
 // writeDefs creates output files to be compiled by gc and gcc.
 func (p *Package) writeDefs() {
 	var fgo2, fc io.Writer
-	f := creat(*objDir + "_cgo_gotypes.go")
+	f := creat("_cgo_gotypes.go")
 	defer f.Close()
 	fgo2 = f
 	if *gccgo {
-		f := creat(*objDir + "_cgo_defun.c")
+		f := creat("_cgo_defun.c")
 		defer f.Close()
 		fc = f
 	}
-	fm := creat(*objDir + "_cgo_main.c")
+	fm := creat("_cgo_main.c")
 
 	var gccgoInit strings.Builder
 
@@ -50,7 +50,7 @@ func (p *Package) writeDefs() {
 			fmt.Fprintf(fgo2, "//go:cgo_ldflag %q\n", arg)
 		}
 	} else {
-		fflg := creat(*objDir + "_cgo_flags")
+		fflg := creat("_cgo_flags")
 		for _, arg := range p.LdFlags {
 			fmt.Fprintf(fflg, "_CGO_LDFLAGS=%s\n", arg)
 		}
@@ -242,8 +242,8 @@ func (p *Package) writeDefs() {
 		}
 	}
 
-	fgcc := creat(*objDir + "_cgo_export.c")
-	fgcch := creat(*objDir + "_cgo_export.h")
+	fgcc := creat("_cgo_export.c")
+	fgcch := creat("_cgo_export.h")
 	if *gccgo {
 		p.writeGccgoExports(fgo2, fm, fgcc, fgcch)
 	} else {
@@ -263,8 +263,11 @@ func (p *Package) writeDefs() {
 	}
 
 	if *exportHeader != "" && len(p.ExpFunc) > 0 {
-		fexp := creat(*exportHeader)
-		fgcch, err := os.Open(*objDir + "_cgo_export.h")
+		fexp, err := os.Create(*exportHeader)
+		if err != nil {
+			fatalf("%s", err)
+		}
+		fgcch, err := os.Open(filepath.Join(outputDir(), "_cgo_export.h"))
 		if err != nil {
 			fatalf("%s", err)
 		}
@@ -697,8 +700,8 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 	base := srcfile
 	base = strings.TrimSuffix(base, ".go")
 	base = filepath.Base(base)
-	fgo1 := creat(*objDir + base + ".cgo1.go")
-	fgcc := creat(*objDir + base + ".cgo2.c")
+	fgo1 := creat(base + ".cgo1.go")
+	fgcc := creat(base + ".cgo2.c")
 
 	p.GoFiles = append(p.GoFiles, base+".cgo1.go")
 	p.GccFiles = append(p.GccFiles, base+".cgo2.c")
@@ -783,13 +786,13 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 	// We're trying to write a gcc struct that matches gc's layout.
 	// Use packed attribute to force no padding in this struct in case
 	// gcc has different packing requirements.
-	fmt.Fprintf(fgcc, "\t%s %v *_cgo_a = v;\n", ctype, p.packedAttribute())
-	if n.FuncType.Result != nil {
+	tr := n.FuncType.Result
+	if (n.Kind != "macro" && len(n.FuncType.Params) > 0) || tr != nil {
+		fmt.Fprintf(fgcc, "\t%s %v *_cgo_a = v;\n", ctype, p.packedAttribute())
+	}
+	if tr != nil {
 		// Save the stack top for use below.
 		fmt.Fprintf(fgcc, "\tchar *_cgo_stktop = _cgo_topofstack();\n")
-	}
-	tr := n.FuncType.Result
-	if tr != nil {
 		fmt.Fprintf(fgcc, "\t__typeof__(_cgo_a->r) _cgo_r;\n")
 	}
 	fmt.Fprintf(fgcc, "\t_cgo_tsan_acquire();\n")
@@ -819,7 +822,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 		fmt.Fprintf(fgcc, "\t_cgo_errno = errno;\n")
 	}
 	fmt.Fprintf(fgcc, "\t_cgo_tsan_release();\n")
-	if n.FuncType.Result != nil {
+	if tr != nil {
 		// The cgo call may have caused a stack copy (via a callback).
 		// Adjust the return value pointer appropriately.
 		fmt.Fprintf(fgcc, "\t_cgo_a = (void*)((char*)_cgo_a + (_cgo_topofstack() - _cgo_stktop));\n")
@@ -1395,7 +1398,7 @@ func gccgoToSymbol(ppath string) string {
 				fatalf("unable to locate gccgo: %v", err)
 			}
 		}
-		gccgoMangler, err = pkgpath.ToSymbolFunc(cmd, *objDir)
+		gccgoMangler, err = pkgpath.ToSymbolFunc(cmd, outputDir())
 		if err != nil {
 			fatalf("%v", err)
 		}

src/cmd/cgo/util.go

@@ -10,6 +10,7 @@ import (
 	"go/token"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"slices"
 )
 
@@ -97,10 +98,17 @@ func error_(pos token.Pos, msg string, args ...any) {
 	fmt.Fprintf(os.Stderr, "\n")
 }
 
+// create creates a file in the output directory.
 func creat(name string) *os.File {
-	f, err := os.Create(name)
+	f, err := os.Create(filepath.Join(outputDir(), name))
 	if err != nil {
 		fatalf("%s", err)
 	}
 	return f
 }
+
+// outputDir returns the output directory, making sure that it exists.
+func outputDir() string {
+	os.MkdirAll(*objDir, 0o700)
+	return *objDir
+}

src/cmd/compile/abi-internal.md

@@ -643,8 +643,6 @@ Registers R20 - R21, R23 – R28, R30 - R31, F16 – F31 are permanent scratch r
 
 Register R2 is reserved and never used.
 
-Register R20, R21 is Used by runtime.duffcopy, runtime.duffzero.
-
 Special-purpose registers used within Go generated code and Go assembly code
 are as follows:
 
@@ -653,7 +651,6 @@ are as follows:
 | R0 | Zero value | Same | Same |
 | R1 | Link register | Link register | Scratch |
 | R3 | Stack pointer | Same | Same |
-| R20,R21 | Scratch | Scratch | Used by duffcopy, duffzero |
 | R22 | Current goroutine | Same | Same |
 | R29 | Closure context pointer | Same | Same |
 | R30, R31 | used by the assembler | Same | Same |

src/cmd/compile/internal/amd64/simdssa.go

@@ -1,4 +1,4 @@
-// Code generated by x/arch/internal/simdgen using 'go run . -xedPath $XED_PATH -o godefs -goroot $GOROOT go.yaml types.yaml categories.yaml'; DO NOT EDIT.
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
 
 package amd64
 
@@ -25,23 +25,23 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPABSQ128,
 		ssa.OpAMD64VPABSQ256,
 		ssa.OpAMD64VPABSQ512,
-		ssa.OpAMD64VBROADCASTSS128,
 		ssa.OpAMD64VPBROADCASTQ128,
-		ssa.OpAMD64VPBROADCASTB128,
-		ssa.OpAMD64VPBROADCASTW128,
-		ssa.OpAMD64VPBROADCASTD128,
-		ssa.OpAMD64VBROADCASTSS256,
+		ssa.OpAMD64VBROADCASTSS128,
 		ssa.OpAMD64VBROADCASTSD256,
-		ssa.OpAMD64VPBROADCASTB256,
-		ssa.OpAMD64VPBROADCASTW256,
-		ssa.OpAMD64VPBROADCASTD256,
+		ssa.OpAMD64VPBROADCASTD128,
 		ssa.OpAMD64VPBROADCASTQ256,
-		ssa.OpAMD64VBROADCASTSS512,
+		ssa.OpAMD64VBROADCASTSS256,
 		ssa.OpAMD64VBROADCASTSD512,
-		ssa.OpAMD64VPBROADCASTB512,
-		ssa.OpAMD64VPBROADCASTW512,
-		ssa.OpAMD64VPBROADCASTD512,
+		ssa.OpAMD64VPBROADCASTW128,
+		ssa.OpAMD64VPBROADCASTD256,
 		ssa.OpAMD64VPBROADCASTQ512,
+		ssa.OpAMD64VBROADCASTSS512,
+		ssa.OpAMD64VPBROADCASTB128,
+		ssa.OpAMD64VPBROADCASTW256,
+		ssa.OpAMD64VPBROADCASTD512,
+		ssa.OpAMD64VPBROADCASTB256,
+		ssa.OpAMD64VPBROADCASTW512,
+		ssa.OpAMD64VPBROADCASTB512,
 		ssa.OpAMD64VCVTPD2PSX128,
 		ssa.OpAMD64VCVTPD2PSY128,
 		ssa.OpAMD64VCVTPD2PS256,
@@ -175,7 +175,15 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMOVSQD128_128,
 		ssa.OpAMD64VPMOVSQD128_256,
 		ssa.OpAMD64VPMOVSQD256,
+		ssa.OpAMD64VPMOVUSWB128_128,
+		ssa.OpAMD64VPMOVUSWB128_256,
 		ssa.OpAMD64VPMOVUSWB256,
+		ssa.OpAMD64VPMOVUSDB128_128,
+		ssa.OpAMD64VPMOVUSDB128_256,
+		ssa.OpAMD64VPMOVUSDB128_512,
+		ssa.OpAMD64VPMOVUSQB128_128,
+		ssa.OpAMD64VPMOVUSQB128_256,
+		ssa.OpAMD64VPMOVUSQB128_512,
 		ssa.OpAMD64VPMOVUSDW128_128,
 		ssa.OpAMD64VPMOVUSDW128_256,
 		ssa.OpAMD64VPMOVUSDW256,
@@ -242,12 +250,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPADDQ256,
 		ssa.OpAMD64VPADDQ512,
 		ssa.OpAMD64VHADDPS128,
-		ssa.OpAMD64VHADDPS256,
 		ssa.OpAMD64VHADDPD128,
-		ssa.OpAMD64VHADDPD256,
 		ssa.OpAMD64VPHADDW128,
-		ssa.OpAMD64VPHADDW256,
 		ssa.OpAMD64VPHADDD128,
+		ssa.OpAMD64VHADDPS256,
+		ssa.OpAMD64VHADDPD256,
+		ssa.OpAMD64VPHADDW256,
 		ssa.OpAMD64VPHADDD256,
 		ssa.OpAMD64VPHADDSW128,
 		ssa.OpAMD64VPHADDSW256,
@@ -512,12 +520,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPSUBQ256,
 		ssa.OpAMD64VPSUBQ512,
 		ssa.OpAMD64VHSUBPS128,
-		ssa.OpAMD64VHSUBPS256,
 		ssa.OpAMD64VHSUBPD128,
-		ssa.OpAMD64VHSUBPD256,
 		ssa.OpAMD64VPHSUBW128,
-		ssa.OpAMD64VPHSUBW256,
 		ssa.OpAMD64VPHSUBD128,
+		ssa.OpAMD64VHSUBPS256,
+		ssa.OpAMD64VHSUBPD256,
+		ssa.OpAMD64VPHSUBW256,
 		ssa.OpAMD64VPHSUBD256,
 		ssa.OpAMD64VPHSUBSW128,
 		ssa.OpAMD64VPHSUBSW256,
@@ -731,12 +739,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPRORVQMasked128,
 		ssa.OpAMD64VPRORVQMasked256,
 		ssa.OpAMD64VPRORVQMasked512,
-		ssa.OpAMD64VPACKSSDWMasked128,
 		ssa.OpAMD64VPACKSSDWMasked256,
 		ssa.OpAMD64VPACKSSDWMasked512,
-		ssa.OpAMD64VPACKUSDWMasked128,
+		ssa.OpAMD64VPACKSSDWMasked128,
 		ssa.OpAMD64VPACKUSDWMasked256,
 		ssa.OpAMD64VPACKUSDWMasked512,
+		ssa.OpAMD64VPACKUSDWMasked128,
 		ssa.OpAMD64VSCALEFPSMasked128,
 		ssa.OpAMD64VSCALEFPSMasked256,
 		ssa.OpAMD64VSCALEFPSMasked512,
@@ -824,23 +832,23 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPABSQMasked128,
 		ssa.OpAMD64VPABSQMasked256,
 		ssa.OpAMD64VPABSQMasked512,
-		ssa.OpAMD64VBROADCASTSSMasked128,
 		ssa.OpAMD64VPBROADCASTQMasked128,
-		ssa.OpAMD64VPBROADCASTBMasked128,
-		ssa.OpAMD64VPBROADCASTWMasked128,
-		ssa.OpAMD64VPBROADCASTDMasked128,
-		ssa.OpAMD64VBROADCASTSSMasked256,
+		ssa.OpAMD64VBROADCASTSSMasked128,
 		ssa.OpAMD64VBROADCASTSDMasked256,
-		ssa.OpAMD64VPBROADCASTBMasked256,
-		ssa.OpAMD64VPBROADCASTWMasked256,
-		ssa.OpAMD64VPBROADCASTDMasked256,
+		ssa.OpAMD64VPBROADCASTDMasked128,
 		ssa.OpAMD64VPBROADCASTQMasked256,
-		ssa.OpAMD64VBROADCASTSSMasked512,
+		ssa.OpAMD64VBROADCASTSSMasked256,
 		ssa.OpAMD64VBROADCASTSDMasked512,
-		ssa.OpAMD64VPBROADCASTBMasked512,
-		ssa.OpAMD64VPBROADCASTWMasked512,
-		ssa.OpAMD64VPBROADCASTDMasked512,
+		ssa.OpAMD64VPBROADCASTWMasked128,
+		ssa.OpAMD64VPBROADCASTDMasked256,
 		ssa.OpAMD64VPBROADCASTQMasked512,
+		ssa.OpAMD64VBROADCASTSSMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked128,
+		ssa.OpAMD64VPBROADCASTWMasked256,
+		ssa.OpAMD64VPBROADCASTDMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked256,
+		ssa.OpAMD64VPBROADCASTWMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked512,
 		ssa.OpAMD64VCOMPRESSPSMasked128,
 		ssa.OpAMD64VCOMPRESSPSMasked256,
 		ssa.OpAMD64VCOMPRESSPSMasked512,
@@ -1010,7 +1018,15 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMOVSQDMasked128_128,
 		ssa.OpAMD64VPMOVSQDMasked128_256,
 		ssa.OpAMD64VPMOVSQDMasked256,
+		ssa.OpAMD64VPMOVUSWBMasked128_128,
+		ssa.OpAMD64VPMOVUSWBMasked128_256,
 		ssa.OpAMD64VPMOVUSWBMasked256,
+		ssa.OpAMD64VPMOVUSDBMasked128_128,
+		ssa.OpAMD64VPMOVUSDBMasked128_256,
+		ssa.OpAMD64VPMOVUSDBMasked128_512,
+		ssa.OpAMD64VPMOVUSQBMasked128_128,
+		ssa.OpAMD64VPMOVUSQBMasked128_256,
+		ssa.OpAMD64VPMOVUSQBMasked128_512,
 		ssa.OpAMD64VPMOVUSDWMasked128_128,
 		ssa.OpAMD64VPMOVUSDWMasked128_256,
 		ssa.OpAMD64VPMOVUSDWMasked256,
@@ -1308,12 +1324,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPERMI2Q256,
 		ssa.OpAMD64VPERMI2PD512,
 		ssa.OpAMD64VPERMI2Q512,
-		ssa.OpAMD64VPDPBUSD128,
-		ssa.OpAMD64VPDPBUSD256,
-		ssa.OpAMD64VPDPBUSD512,
-		ssa.OpAMD64VPDPBUSDS128,
-		ssa.OpAMD64VPDPBUSDS256,
-		ssa.OpAMD64VPDPBUSDS512,
 		ssa.OpAMD64VFMADD213PS128,
 		ssa.OpAMD64VFMADD213PS256,
 		ssa.OpAMD64VFMADD213PS512,
@@ -1430,12 +1440,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMADDUBSWMasked128Merging,
 		ssa.OpAMD64VPMADDUBSWMasked256Merging,
 		ssa.OpAMD64VPMADDUBSWMasked512Merging,
-		ssa.OpAMD64VPDPBUSDMasked128,
-		ssa.OpAMD64VPDPBUSDMasked256,
-		ssa.OpAMD64VPDPBUSDMasked512,
-		ssa.OpAMD64VPDPBUSDSMasked128,
-		ssa.OpAMD64VPDPBUSDSMasked256,
-		ssa.OpAMD64VPDPBUSDSMasked512,
 		ssa.OpAMD64VGF2P8MULBMasked128Merging,
 		ssa.OpAMD64VGF2P8MULBMasked256Merging,
 		ssa.OpAMD64VGF2P8MULBMasked512Merging,
@@ -1559,12 +1563,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPRORVQMasked128Merging,
 		ssa.OpAMD64VPRORVQMasked256Merging,
 		ssa.OpAMD64VPRORVQMasked512Merging,
-		ssa.OpAMD64VPACKSSDWMasked128Merging,
 		ssa.OpAMD64VPACKSSDWMasked256Merging,
 		ssa.OpAMD64VPACKSSDWMasked512Merging,
-		ssa.OpAMD64VPACKUSDWMasked128Merging,
+		ssa.OpAMD64VPACKSSDWMasked128Merging,
 		ssa.OpAMD64VPACKUSDWMasked256Merging,
 		ssa.OpAMD64VPACKUSDWMasked512Merging,
+		ssa.OpAMD64VPACKUSDWMasked128Merging,
 		ssa.OpAMD64VSCALEFPSMasked128Merging,
 		ssa.OpAMD64VSCALEFPSMasked256Merging,
 		ssa.OpAMD64VSCALEFPSMasked512Merging,
@@ -1955,25 +1959,11 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPERMI2Q256load,
 		ssa.OpAMD64VPERMI2PD512load,
 		ssa.OpAMD64VPERMI2Q512load,
-		ssa.OpAMD64VPDPBUSD512load,
-		ssa.OpAMD64VPDPBUSDS512load,
-		ssa.OpAMD64VFMADD213PS128load,
-		ssa.OpAMD64VFMADD213PS256load,
 		ssa.OpAMD64VFMADD213PS512load,
-		ssa.OpAMD64VFMADD213PD128load,
-		ssa.OpAMD64VFMADD213PD256load,
 		ssa.OpAMD64VFMADD213PD512load,
-		ssa.OpAMD64VFMADDSUB213PS128load,
-		ssa.OpAMD64VFMADDSUB213PS256load,
 		ssa.OpAMD64VFMADDSUB213PS512load,
-		ssa.OpAMD64VFMADDSUB213PD128load,
-		ssa.OpAMD64VFMADDSUB213PD256load,
 		ssa.OpAMD64VFMADDSUB213PD512load,
-		ssa.OpAMD64VFMSUBADD213PS128load,
-		ssa.OpAMD64VFMSUBADD213PS256load,
 		ssa.OpAMD64VFMSUBADD213PS512load,
-		ssa.OpAMD64VFMSUBADD213PD128load,
-		ssa.OpAMD64VFMSUBADD213PD256load,
 		ssa.OpAMD64VFMSUBADD213PD512load,
 		ssa.OpAMD64VPSHLDVD128load,
 		ssa.OpAMD64VPSHLDVD256load,
@@ -2004,12 +1994,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPERMI2QMasked256load,
 		ssa.OpAMD64VPERMI2PDMasked512load,
 		ssa.OpAMD64VPERMI2QMasked512load,
-		ssa.OpAMD64VPDPBUSDMasked128load,
-		ssa.OpAMD64VPDPBUSDMasked256load,
-		ssa.OpAMD64VPDPBUSDMasked512load,
-		ssa.OpAMD64VPDPBUSDSMasked128load,
-		ssa.OpAMD64VPDPBUSDSMasked256load,
-		ssa.OpAMD64VPDPBUSDSMasked512load,
 		ssa.OpAMD64VFMADD213PSMasked128load,
 		ssa.OpAMD64VFMADD213PSMasked256load,
 		ssa.OpAMD64VFMADD213PSMasked512load,
@@ -2146,12 +2130,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPRORVQMasked128load,
 		ssa.OpAMD64VPRORVQMasked256load,
 		ssa.OpAMD64VPRORVQMasked512load,
-		ssa.OpAMD64VPACKSSDWMasked128load,
 		ssa.OpAMD64VPACKSSDWMasked256load,
 		ssa.OpAMD64VPACKSSDWMasked512load,
-		ssa.OpAMD64VPACKUSDWMasked128load,
+		ssa.OpAMD64VPACKSSDWMasked128load,
 		ssa.OpAMD64VPACKUSDWMasked256load,
 		ssa.OpAMD64VPACKUSDWMasked512load,
+		ssa.OpAMD64VPACKUSDWMasked128load,
 		ssa.OpAMD64VSCALEFPSMasked128load,
 		ssa.OpAMD64VSCALEFPSMasked256load,
 		ssa.OpAMD64VSCALEFPSMasked512load,
@@ -2464,23 +2448,23 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPABSQMasked128Merging,
 		ssa.OpAMD64VPABSQMasked256Merging,
 		ssa.OpAMD64VPABSQMasked512Merging,
-		ssa.OpAMD64VBROADCASTSSMasked128Merging,
 		ssa.OpAMD64VPBROADCASTQMasked128Merging,
-		ssa.OpAMD64VPBROADCASTBMasked128Merging,
-		ssa.OpAMD64VPBROADCASTWMasked128Merging,
-		ssa.OpAMD64VPBROADCASTDMasked128Merging,
-		ssa.OpAMD64VBROADCASTSSMasked256Merging,
+		ssa.OpAMD64VBROADCASTSSMasked128Merging,
 		ssa.OpAMD64VBROADCASTSDMasked256Merging,
-		ssa.OpAMD64VPBROADCASTBMasked256Merging,
-		ssa.OpAMD64VPBROADCASTWMasked256Merging,
-		ssa.OpAMD64VPBROADCASTDMasked256Merging,
+		ssa.OpAMD64VPBROADCASTDMasked128Merging,
 		ssa.OpAMD64VPBROADCASTQMasked256Merging,
-		ssa.OpAMD64VBROADCASTSSMasked512Merging,
+		ssa.OpAMD64VBROADCASTSSMasked256Merging,
 		ssa.OpAMD64VBROADCASTSDMasked512Merging,
-		ssa.OpAMD64VPBROADCASTBMasked512Merging,
-		ssa.OpAMD64VPBROADCASTWMasked512Merging,
-		ssa.OpAMD64VPBROADCASTDMasked512Merging,
+		ssa.OpAMD64VPBROADCASTWMasked128Merging,
+		ssa.OpAMD64VPBROADCASTDMasked256Merging,
 		ssa.OpAMD64VPBROADCASTQMasked512Merging,
+		ssa.OpAMD64VBROADCASTSSMasked512Merging,
+		ssa.OpAMD64VPBROADCASTBMasked128Merging,
+		ssa.OpAMD64VPBROADCASTWMasked256Merging,
+		ssa.OpAMD64VPBROADCASTDMasked512Merging,
+		ssa.OpAMD64VPBROADCASTBMasked256Merging,
+		ssa.OpAMD64VPBROADCASTWMasked512Merging,
+		ssa.OpAMD64VPBROADCASTBMasked512Merging,
 		ssa.OpAMD64VRNDSCALEPSMasked128Merging,
 		ssa.OpAMD64VRNDSCALEPSMasked256Merging,
 		ssa.OpAMD64VRNDSCALEPSMasked512Merging,
@@ -2638,7 +2622,15 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMOVSQDMasked128_128Merging,
 		ssa.OpAMD64VPMOVSQDMasked128_256Merging,
 		ssa.OpAMD64VPMOVSQDMasked256Merging,
+		ssa.OpAMD64VPMOVUSWBMasked128_128Merging,
+		ssa.OpAMD64VPMOVUSWBMasked128_256Merging,
 		ssa.OpAMD64VPMOVUSWBMasked256Merging,
+		ssa.OpAMD64VPMOVUSDBMasked128_128Merging,
+		ssa.OpAMD64VPMOVUSDBMasked128_256Merging,
+		ssa.OpAMD64VPMOVUSDBMasked128_512Merging,
+		ssa.OpAMD64VPMOVUSQBMasked128_128Merging,
+		ssa.OpAMD64VPMOVUSQBMasked128_256Merging,
+		ssa.OpAMD64VPMOVUSQBMasked128_512Merging,
 		ssa.OpAMD64VPMOVUSDWMasked128_128Merging,
 		ssa.OpAMD64VPMOVUSDWMasked128_256Merging,
 		ssa.OpAMD64VPMOVUSDWMasked256Merging,
@@ -2813,23 +2805,23 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPAVGWMasked128,
 		ssa.OpAMD64VPAVGWMasked256,
 		ssa.OpAMD64VPAVGWMasked512,
-		ssa.OpAMD64VBROADCASTSSMasked128,
 		ssa.OpAMD64VPBROADCASTQMasked128,
-		ssa.OpAMD64VPBROADCASTBMasked128,
-		ssa.OpAMD64VPBROADCASTWMasked128,
-		ssa.OpAMD64VPBROADCASTDMasked128,
-		ssa.OpAMD64VBROADCASTSSMasked256,
+		ssa.OpAMD64VBROADCASTSSMasked128,
 		ssa.OpAMD64VBROADCASTSDMasked256,
-		ssa.OpAMD64VPBROADCASTBMasked256,
-		ssa.OpAMD64VPBROADCASTWMasked256,
-		ssa.OpAMD64VPBROADCASTDMasked256,
+		ssa.OpAMD64VPBROADCASTDMasked128,
 		ssa.OpAMD64VPBROADCASTQMasked256,
-		ssa.OpAMD64VBROADCASTSSMasked512,
+		ssa.OpAMD64VBROADCASTSSMasked256,
 		ssa.OpAMD64VBROADCASTSDMasked512,
-		ssa.OpAMD64VPBROADCASTBMasked512,
-		ssa.OpAMD64VPBROADCASTWMasked512,
-		ssa.OpAMD64VPBROADCASTDMasked512,
+		ssa.OpAMD64VPBROADCASTWMasked128,
+		ssa.OpAMD64VPBROADCASTDMasked256,
 		ssa.OpAMD64VPBROADCASTQMasked512,
+		ssa.OpAMD64VBROADCASTSSMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked128,
+		ssa.OpAMD64VPBROADCASTWMasked256,
+		ssa.OpAMD64VPBROADCASTDMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked256,
+		ssa.OpAMD64VPBROADCASTWMasked512,
+		ssa.OpAMD64VPBROADCASTBMasked512,
 		ssa.OpAMD64VRNDSCALEPSMasked128,
 		ssa.OpAMD64VRNDSCALEPSMasked128load,
 		ssa.OpAMD64VRNDSCALEPSMasked256,
@@ -3021,18 +3013,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMADDUBSWMasked128,
 		ssa.OpAMD64VPMADDUBSWMasked256,
 		ssa.OpAMD64VPMADDUBSWMasked512,
-		ssa.OpAMD64VPDPBUSDMasked128,
-		ssa.OpAMD64VPDPBUSDMasked128load,
-		ssa.OpAMD64VPDPBUSDMasked256,
-		ssa.OpAMD64VPDPBUSDMasked256load,
-		ssa.OpAMD64VPDPBUSDMasked512,
-		ssa.OpAMD64VPDPBUSDMasked512load,
-		ssa.OpAMD64VPDPBUSDSMasked128,
-		ssa.OpAMD64VPDPBUSDSMasked128load,
-		ssa.OpAMD64VPDPBUSDSMasked256,
-		ssa.OpAMD64VPDPBUSDSMasked256load,
-		ssa.OpAMD64VPDPBUSDSMasked512,
-		ssa.OpAMD64VPDPBUSDSMasked512load,
 		ssa.OpAMD64VEXPANDPSMasked128,
 		ssa.OpAMD64VEXPANDPSMasked256,
 		ssa.OpAMD64VEXPANDPSMasked512,
@@ -3415,12 +3395,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMOVSQBMasked128_128,
 		ssa.OpAMD64VPMOVSQBMasked128_256,
 		ssa.OpAMD64VPMOVSQBMasked128_512,
-		ssa.OpAMD64VPACKSSDWMasked128,
-		ssa.OpAMD64VPACKSSDWMasked128load,
 		ssa.OpAMD64VPACKSSDWMasked256,
 		ssa.OpAMD64VPACKSSDWMasked256load,
 		ssa.OpAMD64VPACKSSDWMasked512,
 		ssa.OpAMD64VPACKSSDWMasked512load,
+		ssa.OpAMD64VPACKSSDWMasked128,
+		ssa.OpAMD64VPACKSSDWMasked128load,
 		ssa.OpAMD64VPMOVSDWMasked128_128,
 		ssa.OpAMD64VPMOVSDWMasked128_256,
 		ssa.OpAMD64VPMOVSDWMasked256,
@@ -3430,13 +3410,21 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool {
 		ssa.OpAMD64VPMOVSQDMasked128_128,
 		ssa.OpAMD64VPMOVSQDMasked128_256,
 		ssa.OpAMD64VPMOVSQDMasked256,
+		ssa.OpAMD64VPMOVUSWBMasked128_128,
+		ssa.OpAMD64VPMOVUSWBMasked128_256,
 		ssa.OpAMD64VPMOVUSWBMasked256,
-		ssa.OpAMD64VPACKUSDWMasked128,
-		ssa.OpAMD64VPACKUSDWMasked128load,
+		ssa.OpAMD64VPMOVUSDBMasked128_128,
+		ssa.OpAMD64VPMOVUSDBMasked128_256,
+		ssa.OpAMD64VPMOVUSDBMasked128_512,
+		ssa.OpAMD64VPMOVUSQBMasked128_128,
+		ssa.OpAMD64VPMOVUSQBMasked128_256,
+		ssa.OpAMD64VPMOVUSQBMasked128_512,
 		ssa.OpAMD64VPACKUSDWMasked256,
 		ssa.OpAMD64VPACKUSDWMasked256load,
 		ssa.OpAMD64VPACKUSDWMasked512,
 		ssa.OpAMD64VPACKUSDWMasked512load,
+		ssa.OpAMD64VPACKUSDWMasked128,
+		ssa.OpAMD64VPACKUSDWMasked128load,
 		ssa.OpAMD64VPMOVUSDWMasked128_128,
 		ssa.OpAMD64VPMOVUSDWMasked128_256,
 		ssa.OpAMD64VPMOVUSDWMasked256,

src/cmd/compile/internal/amd64/ssa.go

@@ -43,6 +43,10 @@ func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
 	}
 }
 
+func isGPReg(r int16) bool {
+	return x86.REG_AL <= r && r <= x86.REG_R15
+}
+
 func isFPReg(r int16) bool {
 	return x86.REG_X0 <= r && r <= x86.REG_Z31
 }
@@ -1225,14 +1229,23 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		if v.Type.IsMemory() {
 			return
 		}
-		x := v.Args[0].Reg()
+		arg := v.Args[0]
+		x := arg.Reg()
 		y := v.Reg()
 		if v.Type.IsSIMD() {
-			x = simdOrMaskReg(v.Args[0])
+			x = simdOrMaskReg(arg)
 			y = simdOrMaskReg(v)
 		}
 		if x != y {
-			opregreg(s, moveByRegsWidth(y, x, v.Type.Size()), y, x)
+			width := v.Type.Size()
+			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg, 3) {
+				// The source was naturally zext-ed from 32 to 64 bits,
+				// but we are asked to do a full 64-bit copy.
+				// Save the REX prefix byte in I-CACHE by using a 32-bit move,
+				// since it zeroes the upper 32 bits anyway.
+				width = 4
+			}
+			opregreg(s, moveByRegsWidth(y, x, width), y, x)
 		}
 	case ssa.OpLoadReg:
 		if v.Type.IsFlags() {
@@ -1845,7 +1858,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpAMD64VPMOVVec32x16ToM,
 		ssa.OpAMD64VPMOVVec64x2ToM,
 		ssa.OpAMD64VPMOVVec64x4ToM,
-		ssa.OpAMD64VPMOVVec64x8ToM:
+		ssa.OpAMD64VPMOVVec64x8ToM,
+		ssa.OpAMD64VPMOVMSKB128,
+		ssa.OpAMD64VPMOVMSKB256,
+		ssa.OpAMD64VMOVMSKPS128,
+		ssa.OpAMD64VMOVMSKPS256,
+		ssa.OpAMD64VMOVMSKPD128,
+		ssa.OpAMD64VMOVMSKPD256:
 		p := s.Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_REG
 		p.From.Reg = simdReg(v.Args[0])

src/cmd/compile/internal/base/debug.go

@@ -18,6 +18,7 @@ var Debug DebugFlags
 type DebugFlags struct {
 	AlignHot              int    `help:"enable hot block alignment (currently requires -pgo)" concurrent:"ok"`
 	Append                int    `help:"print information about append compilation"`
+	AstDump               string `help:"for specified function/method, dump AST/IR at interesting points in compilation, to file pkg.func.ast. Use leading ~ for regular expression match."`
 	Checkptr              int    `help:"instrument unsafe pointer conversions\n0: instrumentation disabled\n1: conversions involving unsafe.Pointer are instrumented\n2: conversions to unsafe.Pointer force heap allocation" concurrent:"ok"`
 	Closure               int    `help:"print information about closure compilation"`
 	CompressInstructions  int    `help:"use compressed instructions when possible (if supported by architecture)"`

src/cmd/compile/internal/bloop/bloop.go

@@ -39,7 +39,6 @@ package bloop
 import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
-	"cmd/compile/internal/reflectdata"
 	"cmd/compile/internal/typecheck"
 	"cmd/compile/internal/types"
 	"cmd/internal/src"
@@ -81,39 +80,43 @@ func getAddressableNameFromNode(n ir.Node) *ir.Name {
 	return nil
 }
 
+// getKeepAliveNodes analyzes an IR node and returns a list of nodes that must be kept alive.
+func getKeepAliveNodes(pos src.XPos, n ir.Node) ir.Nodes {
+	name := getAddressableNameFromNode(n)
+	if name != nil {
+		debugName(name, pos)
+		return ir.Nodes{name}
+	} else if deref := n.(*ir.StarExpr); deref != nil {
+		if base.Flag.LowerM > 1 {
+			base.WarnfAt(pos, "dereference will be kept alive")
+		}
+		return ir.Nodes{deref}
+	} else if base.Flag.LowerM > 1 {
+		base.WarnfAt(pos, "expr is unknown to bloop pass")
+	}
+	return nil
+}
+
 // keepAliveAt returns a statement that is either curNode, or a
 // block containing curNode followed by a call to runtime.KeepAlive for each
 // node in ns. These calls ensure that nodes in ns will be live until
 // after curNode's execution.
-func keepAliveAt(ns []ir.Node, curNode ir.Node) ir.Node {
+func keepAliveAt(ns ir.Nodes, curNode ir.Node) ir.Node {
 	if len(ns) == 0 {
 		return curNode
 	}
 
 	pos := curNode.Pos()
-	calls := []ir.Node{curNode}
+	calls := ir.Nodes{curNode}
 	for _, n := range ns {
-		if n == nil {
-			continue
-		}
-		if n.Sym() == nil {
-			continue
-		}
-		if n.Sym().IsBlank() {
+		if n == nil || n.Sym() == nil || n.Sym().IsBlank() {
 			continue
 		}
 		if !ir.IsAddressable(n) {
 			base.FatalfAt(n.Pos(), "keepAliveAt: node %v is not addressable", n)
 		}
 		arg := ir.NewConvExpr(pos, ir.OCONV, types.Types[types.TUNSAFEPTR], typecheck.NodAddr(n))
-		if !n.Type().IsInterface() {
-			srcRType0 := reflectdata.TypePtrAt(pos, n.Type())
-			arg.TypeWord = srcRType0
-			arg.SrcRType = srcRType0
-		}
-		callExpr := typecheck.Call(pos,
-			typecheck.LookupRuntime("KeepAlive"),
-			[]ir.Node{arg}, false).(*ir.CallExpr)
+		callExpr := typecheck.Call(pos, typecheck.LookupRuntime("KeepAlive"), ir.Nodes{arg}, false).(*ir.CallExpr)
 		callExpr.IsCompilerVarLive = true
 		callExpr.NoInline = true
 		calls = append(calls, callExpr)
@@ -132,135 +135,105 @@ func debugName(name *ir.Name, pos src.XPos) {
 	}
 }
 
+// preserveCallResults assigns the results of a call statement to temporary variables to ensure they remain alive.
+func preserveCallResults(curFn *ir.Func, call *ir.CallExpr) ir.Node {
+	var ns ir.Nodes
+	lhs := make(ir.Nodes, call.Fun.Type().NumResults())
+	for i, res := range call.Fun.Type().Results() {
+		tmp := typecheck.TempAt(call.Pos(), curFn, res.Type)
+		lhs[i] = tmp
+		ns = append(ns, tmp)
+	}
+
+	if base.Flag.LowerM > 1 {
+		plural := ""
+		if call.Fun.Type().NumResults() > 1 {
+			plural = "s"
+		}
+		base.WarnfAt(call.Pos(), "function result%s will be kept alive", plural)
+	}
+
+	assign := typecheck.AssignExpr(ir.NewAssignListStmt(call.Pos(), ir.OAS2, lhs, ir.Nodes{call})).(*ir.AssignListStmt)
+	assign.Def = true
+	for _, tmp := range lhs {
+		// Place temp declarations in the loop body to help escape analysis.
+		assign.PtrInit().Append(typecheck.Stmt(ir.NewDecl(assign.Pos(), ir.ODCL, tmp.(*ir.Name))))
+	}
+	return keepAliveAt(ns, assign)
+}
+
+// preserveCallArgs ensures the arguments of a call statement are kept alive by transforming them into temporaries if necessary.
+func preserveCallArgs(curFn *ir.Func, call *ir.CallExpr) ir.Node {
+	var argTmps ir.Nodes
+	var names ir.Nodes
+	preserveTmp := func(pos src.XPos, n ir.Node) ir.Node {
+		tmp := typecheck.TempAt(pos, curFn, n.Type())
+		assign := ir.NewAssignStmt(pos, tmp, n)
+		assign.Def = true
+		// Place temp declarations in the loop body to help escape analysis.
+		assign.PtrInit().Append(typecheck.Stmt(ir.NewDecl(assign.Pos(), ir.ODCL, tmp)))
+		argTmps = append(argTmps, typecheck.AssignExpr(assign))
+		names = append(names, tmp)
+		if base.Flag.LowerM > 1 {
+			base.WarnfAt(call.Pos(), "function arg will be kept alive")
+		}
+		return tmp
+	}
+	for i, a := range call.Args {
+		if name := getAddressableNameFromNode(a); name != nil {
+			// If they are name, keep them alive directly.
+			debugName(name, call.Pos())
+			names = append(names, name)
+		} else if a.Op() == ir.OSLICELIT {
+			// variadic args are encoded as slice literal.
+			s := a.(*ir.CompLitExpr)
+			var ns ir.Nodes
+			for i, elem := range s.List {
+				if name := getAddressableNameFromNode(elem); name != nil {
+					debugName(name, call.Pos())
+					ns = append(ns, name)
+				} else {
+					// We need a temporary to save this arg.
+					s.List[i] = preserveTmp(elem.Pos(), elem)
+				}
+			}
+			names = append(names, ns...)
+		} else {
+			// expressions, we need to assign them to temps and change the original arg to reference them.
+			call.Args[i] = preserveTmp(call.Pos(), a)
+		}
+	}
+	if len(argTmps) > 0 {
+		argTmps = append(argTmps, call)
+		return keepAliveAt(names, ir.NewBlockStmt(call.Pos(), argTmps))
+	}
+	return keepAliveAt(names, call)
+}
+
 // preserveStmt transforms stmt so that any names defined/assigned within it
 // are used after stmt's execution, preventing their dead code elimination
 // and dead store elimination. The return value is the transformed statement.
-func preserveStmt(curFn *ir.Func, stmt ir.Node) (ret ir.Node) {
-	ret = stmt
+func preserveStmt(curFn *ir.Func, stmt ir.Node) ir.Node {
 	switch n := stmt.(type) {
 	case *ir.AssignStmt:
-		// Peel down struct and slice indexing to get the names
-		name := getAddressableNameFromNode(n.X)
-		if name != nil {
-			debugName(name, n.Pos())
-			ret = keepAliveAt([]ir.Node{name}, n)
-		} else if deref := n.X.(*ir.StarExpr); deref != nil {
-			ret = keepAliveAt([]ir.Node{deref}, n)
-			if base.Flag.LowerM > 1 {
-				base.WarnfAt(n.Pos(), "dereference will be kept alive")
-			}
-		} else if base.Flag.LowerM > 1 {
-			base.WarnfAt(n.Pos(), "expr is unknown to bloop pass")
-		}
+		return keepAliveAt(getKeepAliveNodes(n.Pos(), n.X), n)
 	case *ir.AssignListStmt:
-		ns := []ir.Node{}
+		var ns ir.Nodes
 		for _, lhs := range n.Lhs {
-			name := getAddressableNameFromNode(lhs)
-			if name != nil {
-				debugName(name, n.Pos())
-				ns = append(ns, name)
-			} else if deref := lhs.(*ir.StarExpr); deref != nil {
-				ns = append(ns, deref)
-				if base.Flag.LowerM > 1 {
-					base.WarnfAt(n.Pos(), "dereference will be kept alive")
-				}
-			} else if base.Flag.LowerM > 1 {
-				base.WarnfAt(n.Pos(), "expr is unknown to bloop pass")
-			}
+			ns = append(ns, getKeepAliveNodes(n.Pos(), lhs)...)
 		}
-		ret = keepAliveAt(ns, n)
+		return keepAliveAt(ns, n)
 	case *ir.AssignOpStmt:
-		name := getAddressableNameFromNode(n.X)
-		if name != nil {
-			debugName(name, n.Pos())
-			ret = keepAliveAt([]ir.Node{name}, n)
-		} else if deref := n.X.(*ir.StarExpr); deref != nil {
-			ret = keepAliveAt([]ir.Node{deref}, n)
-			if base.Flag.LowerM > 1 {
-				base.WarnfAt(n.Pos(), "dereference will be kept alive")
-			}
-		} else if base.Flag.LowerM > 1 {
-			base.WarnfAt(n.Pos(), "expr is unknown to bloop pass")
-		}
+		return keepAliveAt(getKeepAliveNodes(n.Pos(), n.X), n)
 	case *ir.CallExpr:
-		curNode := stmt
+		// The function's results are not assigned, preserve them.
 		if n.Fun != nil && n.Fun.Type() != nil && n.Fun.Type().NumResults() != 0 {
-			ns := []ir.Node{}
-			// This function's results are not assigned, assign them to
-			// auto tmps and then keepAliveAt these autos.
-			// Note: markStmt assumes the context that it's called - this CallExpr is
-			// not within another OAS2, which is guaranteed by the case above.
-			results := n.Fun.Type().Results()
-			lhs := make([]ir.Node, len(results))
-			for i, res := range results {
-				tmp := typecheck.TempAt(n.Pos(), curFn, res.Type)
-				lhs[i] = tmp
-				ns = append(ns, tmp)
-			}
-
-			// Create an assignment statement.
-			assign := typecheck.AssignExpr(
-				ir.NewAssignListStmt(n.Pos(), ir.OAS2, lhs,
-					[]ir.Node{n})).(*ir.AssignListStmt)
-			assign.Def = true
-			curNode = assign
-			plural := ""
-			if len(results) > 1 {
-				plural = "s"
-			}
-			if base.Flag.LowerM > 1 {
-				base.WarnfAt(n.Pos(), "function result%s will be kept alive", plural)
-			}
-			ret = keepAliveAt(ns, curNode)
-		} else {
-			// This function probably doesn't return anything, keep its args alive.
-			argTmps := []ir.Node{}
-			names := []ir.Node{}
-			for i, a := range n.Args {
-				if name := getAddressableNameFromNode(a); name != nil {
-					// If they are name, keep them alive directly.
-					debugName(name, n.Pos())
-					names = append(names, name)
-				} else if a.Op() == ir.OSLICELIT {
-					// variadic args are encoded as slice literal.
-					s := a.(*ir.CompLitExpr)
-					ns := []ir.Node{}
-					for i, elem := range s.List {
-						if name := getAddressableNameFromNode(elem); name != nil {
-							debugName(name, n.Pos())
-							ns = append(ns, name)
-						} else {
-							// We need a temporary to save this arg.
-							tmp := typecheck.TempAt(elem.Pos(), curFn, elem.Type())
-							argTmps = append(argTmps, typecheck.AssignExpr(ir.NewAssignStmt(elem.Pos(), tmp, elem)))
-							names = append(names, tmp)
-							s.List[i] = tmp
-							if base.Flag.LowerM > 1 {
-								base.WarnfAt(n.Pos(), "function arg will be kept alive")
-							}
-						}
-					}
-					names = append(names, ns...)
-				} else {
-					// expressions, we need to assign them to temps and change the original arg to reference
-					// them.
-					tmp := typecheck.TempAt(n.Pos(), curFn, a.Type())
-					argTmps = append(argTmps, typecheck.AssignExpr(ir.NewAssignStmt(n.Pos(), tmp, a)))
-					names = append(names, tmp)
-					n.Args[i] = tmp
-					if base.Flag.LowerM > 1 {
-						base.WarnfAt(n.Pos(), "function arg will be kept alive")
-					}
-				}
-			}
-			if len(argTmps) > 0 {
-				argTmps = append(argTmps, n)
-				curNode = ir.NewBlockStmt(n.Pos(), argTmps)
-			}
-			ret = keepAliveAt(names, curNode)
+			return preserveCallResults(curFn, n)
 		}
+		// This function doesn't return anything, keep its args alive.
+		return preserveCallArgs(curFn, n)
 	}
-	return
+	return stmt
 }
 
 func preserveStmts(curFn *ir.Func, list ir.Nodes) {
@@ -325,7 +298,7 @@ func (e editor) edit(n ir.Node) ir.Node {
 	return n
 }
 
-// BloopWalk performs a walk on all functions in the package
+// Walk performs a walk on all functions in the package
 // if it imports testing and wrap the results of all qualified
 // statements in a runtime.KeepAlive intrinsic call. See package
 // doc for more details.
@@ -333,7 +306,7 @@ func (e editor) edit(n ir.Node) ir.Node {
 //	for b.Loop() {...}
 //
 // loop's body.
-func BloopWalk(pkg *ir.Package) {
+func Walk(pkg *ir.Package) {
 	hasTesting := false
 	for _, i := range pkg.Imports {
 		if i.Path == "testing" {
@@ -347,5 +320,9 @@ func BloopWalk(pkg *ir.Package) {
 	for _, fn := range pkg.Funcs {
 		e := editor{false, fn}
 		ir.EditChildren(fn, e.edit)
+		if ir.MatchAstDump(fn, "bloop") {
+			ir.AstDump(fn, "bloop, "+ir.FuncName(fn))
+		}
 	}
+
 }

src/cmd/compile/internal/compare/compare.go

@@ -257,8 +257,8 @@ func EqStruct(t *types.Type, np, nq ir.Node) ([]ir.Node, bool) {
 func EqString(s, t ir.Node) (eqlen *ir.BinaryExpr, eqmem *ir.CallExpr) {
 	s = typecheck.Conv(s, types.Types[types.TSTRING])
 	t = typecheck.Conv(t, types.Types[types.TSTRING])
-	sptr := ir.NewUnaryExpr(base.Pos, ir.OSPTR, s)
-	tptr := ir.NewUnaryExpr(base.Pos, ir.OSPTR, t)
+	sptr := ir.NewConvExpr(base.Pos, ir.OCONVNOP, types.Types[types.TUNSAFEPTR], ir.NewUnaryExpr(base.Pos, ir.OSPTR, s))
+	tptr := ir.NewConvExpr(base.Pos, ir.OCONVNOP, types.Types[types.TUNSAFEPTR], ir.NewUnaryExpr(base.Pos, ir.OSPTR, t))
 	slen := typecheck.Conv(ir.NewUnaryExpr(base.Pos, ir.OLEN, s), types.Types[types.TUINTPTR])
 	tlen := typecheck.Conv(ir.NewUnaryExpr(base.Pos, ir.OLEN, t), types.Types[types.TUINTPTR])
 
@@ -293,7 +293,7 @@ func EqString(s, t ir.Node) (eqlen *ir.BinaryExpr, eqmem *ir.CallExpr) {
 		cmplen = tlen
 	}
 
-	fn := typecheck.LookupRuntime("memequal", types.Types[types.TUINT8], types.Types[types.TUINT8])
+	fn := typecheck.LookupRuntime("memequal")
 	call := typecheck.Call(base.Pos, fn, []ir.Node{sptr, tptr, ir.Copy(cmplen)}, false).(*ir.CallExpr)
 
 	cmp := ir.NewBinaryExpr(base.Pos, ir.OEQ, slen, tlen)

src/cmd/compile/internal/deadlocals/deadlocals.go

@@ -23,7 +23,11 @@ func Funcs(fns []*ir.Func) {
 	zero := ir.NewBasicLit(base.AutogeneratedPos, types.Types[types.TINT], constant.MakeInt64(0))
 
 	for _, fn := range fns {
+
 		if fn.IsClosure() {
+			if ir.MatchAstDump(fn, "deadlocals closure") {
+				ir.AstDump(fn, "deadlocals closure skipped, "+ir.FuncName(fn))
+			}
 			continue
 		}
 
@@ -50,6 +54,9 @@ func Funcs(fns []*ir.Func) {
 				k.Defn = nil
 			}
 		}
+		if ir.MatchAstDump(fn, "deadlocals") {
+			ir.AstDump(fn, "deadLocals, "+ir.FuncName(fn))
+		}
 	}
 }
 

src/cmd/compile/internal/escape/escape.go

@@ -380,6 +380,11 @@ func (b *batch) finish(fns []*ir.Func) {
 		}
 	}
 
+	for _, fn := range fns {
+		if ir.MatchAstDump(fn, "escape") {
+			ir.AstDump(fn, "escape, "+ir.FuncName(fn))
+		}
+	}
 }
 
 // inMutualBatch reports whether function fn is in the batch of

src/cmd/compile/internal/gc/compile.go

@@ -121,6 +121,9 @@ func prepareFunc(fn *ir.Func) {
 
 	ir.CurFunc = fn
 	walk.Walk(fn)
+	if ir.MatchAstDump(fn, "walk") {
+		ir.AstDump(fn, "walk, "+ir.FuncName(fn))
+	}
 	ir.CurFunc = nil // enforce no further uses of CurFunc
 
 	base.Ctxt.DwTextCount++
@@ -142,73 +145,47 @@ func compileFunctions(profile *pgoir.Profile) {
 		// Compile the longest functions first,
 		// since they're most likely to be the slowest.
 		// This helps avoid stragglers.
+		// Since we remove from the end of the slice queue,
+		// that means shortest to longest.
 		slices.SortFunc(compilequeue, func(a, b *ir.Func) int {
-			return cmp.Compare(len(b.Body), len(a.Body))
+			return cmp.Compare(len(a.Body), len(b.Body))
 		})
 	}
 
-	// By default, we perform work right away on the current goroutine
-	// as the solo worker.
-	queue := func(work func(int)) {
-		work(0)
-	}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	mu.Lock()
 
-	if nWorkers := base.Flag.LowerC; nWorkers > 1 {
-		// For concurrent builds, we allow the work queue
-		// to grow arbitrarily large, but only nWorkers work items
-		// can be running concurrently.
-		workq := make(chan func(int))
-		done := make(chan int)
+	for workerId := range base.Flag.LowerC {
+		// TODO: replace with wg.Go when the oldest bootstrap has it.
+		// With the current policy, that'd be go1.27.
+		wg.Add(1)
 		go func() {
-			ids := make([]int, nWorkers)
-			for i := range ids {
-				ids[i] = i
-			}
-			var pending []func(int)
+			defer wg.Done()
+			var closures []*ir.Func
 			for {
-				select {
-				case work := <-workq:
-					pending = append(pending, work)
-				case id := <-done:
-					ids = append(ids, id)
-				}
-				for len(pending) > 0 && len(ids) > 0 {
-					work := pending[len(pending)-1]
-					id := ids[len(ids)-1]
-					pending = pending[:len(pending)-1]
-					ids = ids[:len(ids)-1]
-					go func() {
-						work(id)
-						done <- id
-					}()
+				mu.Lock()
+				compilequeue = append(compilequeue, closures...)
+				remaining := len(compilequeue)
+				if remaining == 0 {
+					mu.Unlock()
+					return
 				}
+				fn := compilequeue[len(compilequeue)-1]
+				compilequeue = compilequeue[:len(compilequeue)-1]
+				mu.Unlock()
+				ssagen.Compile(fn, workerId, profile)
+				closures = fn.Closures
 			}
 		}()
-		queue = func(work func(int)) {
-			workq <- work
-		}
-	}
-
-	var wg sync.WaitGroup
-	var compile func([]*ir.Func)
-	compile = func(fns []*ir.Func) {
-		wg.Add(len(fns))
-		for _, fn := range fns {
-			fn := fn
-			queue(func(worker int) {
-				ssagen.Compile(fn, worker, profile)
-				compile(fn.Closures)
-				wg.Done()
-			})
-		}
 	}
 
 	types.CalcSizeDisabled = true // not safe to calculate sizes concurrently
 	base.Ctxt.InParallel = true
 
-	compile(compilequeue)
-	compilequeue = nil
+	mu.Unlock()
 	wg.Wait()
+	compilequeue = nil
 
 	base.Ctxt.InParallel = false
 	types.CalcSizeDisabled = false

src/cmd/compile/internal/gc/main.go

@@ -47,6 +47,7 @@ import (
 // already been some compiler errors). It may also be invoked from the explicit panic in
 // hcrash(), in which case, we pass the panic on through.
 func handlePanic() {
+	ir.CloseHTMLWriters()
 	if err := recover(); err != nil {
 		if err == "-h" {
 			// Force real panic now with -h option (hcrash) - the error
@@ -243,13 +244,25 @@ func Main(archInit func(*ssagen.ArchInfo)) {
 		}
 	}
 
+	for _, fn := range typecheck.Target.Funcs {
+		if ir.MatchAstDump(fn, "start") {
+			ir.AstDump(fn, "start, "+ir.FuncName(fn))
+		}
+	}
+
 	// Apply bloop markings.
-	bloop.BloopWalk(typecheck.Target)
+	bloop.Walk(typecheck.Target)
 
 	// Interleaved devirtualization and inlining.
 	base.Timer.Start("fe", "devirtualize-and-inline")
 	interleaved.DevirtualizeAndInlinePackage(typecheck.Target, profile)
 
+	for _, fn := range typecheck.Target.Funcs {
+		if ir.MatchAstDump(fn, "devirtualize-and-inline") {
+			ir.AstDump(fn, "devirtualize-and-inline, "+ir.FuncName(fn))
+		}
+	}
+
 	noder.MakeWrappers(typecheck.Target) // must happen after inlining
 
 	// Get variable capture right in for loops.

src/cmd/compile/internal/inline/inl.go

@@ -286,6 +286,8 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) {
 	// locals, and we use this map to produce a pruned Inline.Dcl
 	// list. See issue 25459 for more context.
 
+	dbg := ir.MatchAstDump(fn, "inline")
+
 	visitor := hairyVisitor{
 		curFunc:       fn,
 		debug:         isDebugFn(fn),
@@ -294,10 +296,17 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) {
 		maxBudget:     budget,
 		extraCallCost: cc,
 		profile:       profile,
+		dbg:           dbg, // Useful for downstream debugging
 	}
+
 	if visitor.tooHairy(fn) {
 		reason = visitor.reason
+		if dbg {
+			ir.AstDump(fn, "inline, too hairy because "+visitor.reason+", "+ir.FuncName(fn))
+		}
 		return
+	} else if dbg {
+		ir.AstDump(fn, "inline, OK, "+ir.FuncName(fn))
 	}
 
 	n.Func.Inl = &ir.Inline{
@@ -441,6 +450,7 @@ type hairyVisitor struct {
 	usedLocals    ir.NameSet
 	do            func(ir.Node) bool
 	profile       *pgoir.Profile
+	dbg           bool
 }
 
 func isDebugFn(fn *ir.Func) bool {
@@ -516,6 +526,9 @@ opSwitch:
 						break opSwitch
 					case "panicrangestate":
 						cheap = true
+					case "deferrangefunc":
+						v.reason = "defer call in range func"
+						return true
 					}
 				}
 			}

src/cmd/compile/internal/ir/dump.go

@@ -9,11 +9,16 @@
 package ir
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"reflect"
 	"regexp"
+	"strings"
+	"sync"
 
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/types"
@@ -63,6 +68,164 @@ func FDumpAny(w io.Writer, root any, filter string, depth int) {
 	p.printf("\n")
 }
 
+// MatchAstDump returns true if the fn matches the value
+// of the astdump debug flag.  Fn matches in the following
+// cases:
+//
+//   - astdump == name(fn)
+//   - astdump == pkgname(fn).name(fn)
+//   - astdump == afterslash(pkgname(fn)).name(fn)
+//   - astdump begins with a "~" and what follows "~" is a
+//     regular expression matching pkgname(fn).name(fn)
+//
+// If MatchAstDump returns true, it also prints to os.Stderr
+//
+//	\nir.Match(<fn>, <astdump>) for <where>\n
+func MatchAstDump(fn *Func, where string) bool {
+	if len(base.Debug.AstDump) == 0 {
+		return false
+	}
+	return matchForDump(fn, base.Ctxt.Pkgpath, where)
+}
+
+var dbgRE *regexp.Regexp
+var onceDbgRE sync.Once
+
+func matchForDump(fn *Func, pkgPath, where string) bool {
+	dbg := false
+	flag := base.Debug.AstDump
+	if flag[0] == '~' {
+		onceDbgRE.Do(func() { dbgRE = regexp.MustCompile(flag[1:]) })
+		dbg = dbgRE.MatchString(pkgPath + "." + FuncName(fn))
+	} else {
+		dbg = matchPkgFn(pkgPath, FuncName(fn), flag)
+	}
+	return dbg
+}
+
+// matchPkgFn returns true if pkg and fnName "match" toMatch.
+// "aFunc" matches "aFunc" (in any package)
+// "aPkg.aFunc" matches "aPkg.aFunc"
+// "aPkg/subPkg.aFunc" matches "subPkg.aFunc"
+func matchPkgFn(pkgName, fnName, toMatch string) bool {
+	if fnName == toMatch {
+		return true
+	}
+	matchPkgDotName := func(pkg string) bool {
+		// Allocation-free equality check for toMatch == base.Ctxt.Pkgpath + "." + fnName
+		return len(toMatch) == len(pkg)+1+len(fnName) &&
+			strings.HasPrefix(toMatch, pkg) && toMatch[len(pkg)] == '.' && strings.HasSuffix(toMatch, fnName)
+	}
+	if matchPkgDotName(pkgName) {
+		return true
+	}
+	if l := strings.LastIndexByte(pkgName, '/'); l > 0 && matchPkgDotName(pkgName[l+1:]) {
+		return true
+	}
+
+	return false
+}
+
+// AstDump appends the ast dump for fn to the ast dump file for fn.
+// The generated file name is
+//
+//	url.PathEscape(PkgFuncName(fn)) + ".ast"
+//
+// It also prints
+//
+//	Writing ast output to <astfilename>\n
+//
+// to os.Stderr.
+func AstDump(fn *Func, why string) {
+	err := withLockAndFile(
+		fn,
+		func(w io.Writer) {
+			FDump(w, why, fn)
+		},
+	)
+	// strip text following comma, for phase names.
+	comma := strings.Index(why, ",")
+	if comma > 0 {
+		why = why[:comma]
+	}
+	DumpNodeHTML(fn, why, fn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Dump returned error %v\n", err)
+	}
+}
+
+var mu sync.Mutex
+var astDumpFiles = make(map[string]bool)
+
+// escapedFileName constructs a file name from fn and suffix,
+// url-path-escaping the function part of the name and replacing it
+// with a hash if it is too long.  The suffix is neither escaped
+// nor including in the length calculation, so an excessively
+// creative suffix will result in problems.
+func escapedFileName(fn *Func, suffix string) string {
+	name := url.PathEscape(PkgFuncName(fn))
+	if len(name) > 125 { // arbitrary limit on file names, as if anyone types these in by hand
+		hash := sha256.Sum256([]byte(name))
+		name = hex.EncodeToString(hash[:8])
+	}
+	return name + suffix
+}
+
+// withLockAndFile manages ast dump files for various function names
+// and invokes a dumping function to write output, under a lock.
+func withLockAndFile(fn *Func, dump func(io.Writer)) (err error) {
+	name := escapedFileName(fn, ".ast")
+
+	// Ensure that debugging output is not scrambled and is written promptly
+	mu.Lock()
+	defer mu.Unlock()
+	mode := os.O_APPEND | os.O_RDWR
+	if !astDumpFiles[name] {
+		astDumpFiles[name] = true
+		mode = os.O_CREATE | os.O_TRUNC | os.O_RDWR
+		fmt.Fprintf(os.Stderr, "Writing text ast output for %s to %s\n", PkgFuncName(fn), name)
+	}
+
+	fi, err := os.OpenFile(name, mode, 0777)
+	if err != nil {
+		return err
+	}
+	defer func() { err = fi.Close() }()
+	dump(fi)
+	return
+}
+
+var htmlWriters = make(map[*Func]*HTMLWriter)
+var orderedFuncs = []*Func{}
+
+// DumpNodeHTML dumps the node n to the HTML writer for fn.
+// It uses the same phase name as the text dump.
+func DumpNodeHTML(fn *Func, why string, n Node) {
+	mu.Lock()
+	defer mu.Unlock()
+	w, ok := htmlWriters[fn]
+	if !ok {
+		name := escapedFileName(fn, ".html")
+		w = NewHTMLWriter(name, fn, "")
+		htmlWriters[fn] = w
+		orderedFuncs = append(orderedFuncs, fn)
+	}
+	w.WritePhase(why, why)
+}
+
+// CloseHTMLWriter closes the HTML writer for fn, if one exists.
+func CloseHTMLWriters() {
+	mu.Lock()
+	defer mu.Unlock()
+	for _, fn := range orderedFuncs {
+		if w, ok := htmlWriters[fn]; ok {
+			w.Close()
+			delete(htmlWriters, fn)
+		}
+	}
+	orderedFuncs = nil
+}
+
 type dumper struct {
 	output  io.Writer
 	fieldrx *regexp.Regexp  // field name filter

src/cmd/compile/internal/ir/dump_test.go

@@ -0,0 +1,44 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+	"testing"
+)
+
+func testMatch(t *testing.T, pkgName, fnName, toMatch string, match bool) {
+	if matchPkgFn(pkgName, fnName, toMatch) != match {
+		t.Errorf("%v != matchPkgFn(%s, %s, %s)", match, pkgName, fnName, toMatch)
+	}
+}
+
+func TestMatchPkgFn(t *testing.T) {
+	// "aFunc" matches "aFunc" (in any package)
+	// "aPkg.aFunc" matches "aPkg.aFunc"
+	// "aPkg/subPkg.aFunc" matches "subPkg.aFunc"
+
+	match := func(pkgName, fnName, toMatch string) {
+		if !matchPkgFn(pkgName, fnName, toMatch) {
+			t.Errorf("matchPkgFn(%s, %s, %s) did not match", pkgName, fnName, toMatch)
+		}
+	}
+	match("aPkg", "AFunc", "AFunc")
+	match("aPkg", "AFunc", "AFunc")
+	match("aPkg", "AFunc", "aPkg.AFunc")
+	match("aPkg/sPkg", "AFunc", "aPkg/sPkg.AFunc")
+	match("aPkg/sPkg", "AFunc", "sPkg.AFunc")
+
+	notmatch := func(pkgName, fnName, toMatch string) {
+		if matchPkgFn(pkgName, fnName, toMatch) {
+			t.Errorf("matchPkgFn(%s, %s, %s) should not match", pkgName, fnName, toMatch)
+		}
+	}
+	notmatch("aPkg", "AFunc", "BFunc")
+	notmatch("aPkg", "AFunc", "aPkg.BFunc")
+	notmatch("aPkg", "AFunc", "bPkg.AFunc")
+	notmatch("aPkg", "AFunc", "aPkg_AFunc")
+	notmatch("aPkg/sPkg", "AFunc", "aPkg/ssPkg.AFunc")
+	notmatch("aPkg/sPkg", "AFunc", "XPkg.AFunc")
+}

src/cmd/compile/internal/ir/fmt.go

@@ -897,11 +897,19 @@ func (l Nodes) Format(s fmt.State, verb rune) {
 // Dump
 
 // Dump prints the message s followed by a debug dump of n.
+// This includes all the recursive structure under n.
 func Dump(s string, n Node) {
 	fmt.Printf("%s%+v\n", s, n)
 }
 
+// Fdump prints to w the message s followed by a debug dump of n.
+// This includes all the recursive structure under n.
+func FDump(w io.Writer, s string, n Node) {
+	fmt.Fprintf(w, "%s%+v\n", s, n)
+}
+
 // DumpList prints the message s followed by a debug dump of each node in the list.
+// This includes all the recursive structure under each node in the list.
 func DumpList(s string, list Nodes) {
 	var buf bytes.Buffer
 	FDumpList(&buf, s, list)
@@ -909,6 +917,7 @@ func DumpList(s string, list Nodes) {
 }
 
 // FDumpList prints to w the message s followed by a debug dump of each node in the list.
+// This includes all the recursive structure under each node in the list.
 func FDumpList(w io.Writer, s string, list Nodes) {
 	io.WriteString(w, s)
 	dumpNodes(w, list, 1)

src/cmd/compile/internal/ir/func.go

@@ -315,7 +315,9 @@ func PkgFuncName(f *Func) string {
 	}
 	s := f.Sym()
 	pkg := s.Pkg
-
+	if pkg == nil {
+		return "<nil>." + s.Name
+	}
 	return pkg.Path + "." + s.Name
 }
 

src/cmd/compile/internal/ir/html_test.go

@@ -0,0 +1,104 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ir
+
+import (
+	"cmd/compile/internal/base"
+	"cmd/compile/internal/types"
+	"cmd/internal/obj"
+	"cmd/internal/src"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestHTMLWriter(t *testing.T) {
+	// Initialize base.Ctxt to avoid panics
+	base.Ctxt = new(obj.Link)
+
+	// Setup a temporary directory for output
+	tmpDir := t.TempDir()
+
+	// Mock func
+	fn := &Func{
+		Nname: &Name{
+			sym: &types.Sym{Name: "TestFunc"},
+		},
+	}
+	// Func embeds miniExpr, so we might need to set op if checked
+	fn.op = ODCLFUNC
+
+	// Create HTMLWriter
+	outFile := filepath.Join(tmpDir, "test.html")
+	w := NewHTMLWriter(outFile, fn, "")
+	if w == nil {
+		t.Fatalf("Failed to create HTMLWriter")
+	}
+
+	// Write a phase
+	w.WritePhase("phase1", "Phase 1")
+
+	// Register a file/line
+	posBase := src.NewFileBase("test.go", "test.go")
+	// base.Ctxt.PosTable.Register(posBase) -- Not needed/doesn't exist
+	pos := src.MakePos(posBase, 10, 1)
+
+	// Create a dummy node
+	n := &Name{
+		sym:   &types.Sym{Name: "VarX"},
+		Class: PAUTO,
+	}
+	n.op = ONAME
+	n.pos = base.Ctxt.PosTable.XPos(pos)
+
+	// Add another phase which actually dumps something interesting
+	fn.Body = []Node{n}
+	w.WritePhase("phase2", "Phase 2")
+
+	// Test escaping
+	n2 := &Name{
+		sym:   &types.Sym{Name: "<Bad>"},
+		Class: PAUTO,
+	}
+	n2.op = ONAME
+	fn.Body = []Node{n2}
+	w.WritePhase("phase3", "Phase 3")
+
+	w.Close()
+
+	// Verify file exists and has content
+	content, err := os.ReadFile(outFile)
+	if err != nil {
+		t.Fatalf("Failed to read output file: %v", err)
+	}
+
+	s := string(content)
+	if len(s) == 0 {
+		t.Errorf("Output file is empty")
+	}
+
+	// Check for Expected strings
+	expected := []string{
+		"<html>",
+		"Phase 1",
+		"Phase 2",
+		"Phase 2",
+		"VarX",
+		"NAME",
+		"&lt;Bad&gt;",
+		"resizer",
+		"loc-",
+		"line-number",
+		"sym-",
+		"variable-name",
+	}
+
+	for _, e := range expected {
+		if !strings.Contains(s, e) {
+			t.Errorf("Output missing %q", e)
+		}
+	}
+}

src/cmd/compile/internal/ir/node.go

@@ -18,6 +18,9 @@ import (
 // A Node is the abstract interface to an IR node.
 type Node interface {
 	// Formatting
+	// For debugging output, use one of
+	//  Dump/FDump/DumpList/FDumplist (in fmt.go)
+	//  DumpAny/FDumpAny (in dump.go)
 	Format(s fmt.State, verb rune)
 
 	// Source position.

src/cmd/compile/internal/liveness/plive.go

@@ -427,7 +427,7 @@ func newliveness(fn *ir.Func, f *ssa.Func, vars []*ir.Name, idx map[*ir.Name]int
 
 	nblocks := int32(len(f.Blocks))
 	nvars := int32(len(vars))
-	bulk := bitvec.NewBulk(nvars, nblocks*7, fn.Pos())
+	bulk := bitvec.NewBulk(nvars, nblocks*4, fn.Pos())
 	for _, b := range f.Blocks {
 		be := lv.blockEffects(b)
 

src/cmd/compile/internal/loong64/ssa.go

@@ -529,6 +529,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpLOONG64BITREV4B,
 		ssa.OpLOONG64BITREVW,
 		ssa.OpLOONG64BITREVV,
+		ssa.OpLOONG64ABSF,
 		ssa.OpLOONG64ABSD:
 		p := s.Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_REG

src/cmd/compile/internal/loopvar/loopvar.go

@@ -448,6 +448,11 @@ func ForCapture(fn *ir.Func) []VarAndLoop {
 		}
 	}
 	ir.WithFunc(fn, forCapture)
+
+	if ir.MatchAstDump(fn, "loopvar") {
+		ir.AstDump(fn, "loopvar, "+ir.FuncName(fn))
+	}
+
 	return transformed
 }
 

src/cmd/compile/internal/loopvar/loopvar_test.go

@@ -6,6 +6,7 @@ package loopvar_test
 
 import (
 	"internal/testenv"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
@@ -381,3 +382,62 @@ func TestLoopVarVersionDisableGoBuild(t *testing.T) {
 		t.Errorf("err=%v == nil", err)
 	}
 }
+
+// TestLoopVarLineDirective tests that loopvar version detection works correctly
+// with line directives. This is a regression test for a bug where FileBase() was
+// used instead of Base(), causing incorrect version lookup when line directives
+// were present.
+func TestLoopVarLineDirective(t *testing.T) {
+	switch runtime.GOOS {
+	case "linux", "darwin":
+	default:
+		t.Skipf("Slow test, usually avoid it, os=%s not linux or darwin", runtime.GOOS)
+	}
+	switch runtime.GOARCH {
+	case "amd64", "arm64":
+	default:
+		t.Skipf("Slow test, usually avoid it, arch=%s not amd64 or arm64", runtime.GOARCH)
+	}
+
+	testenv.MustHaveGoBuild(t)
+	gocmd := testenv.GoToolPath(t)
+	tmpdir := t.TempDir()
+	output := filepath.Join(tmpdir, "foo.exe")
+
+	// Create a go.mod file with Go 1.21 to test compatibility behavior.
+	// When building with a higher Go compiler, the loopvar should be created per-loop.
+	gomodPath := filepath.Join(tmpdir, "go.mod")
+	if err := os.WriteFile(gomodPath, []byte("module test\n\ngo 1.21\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Copy the test file (with line directive) to the temporary module
+	testFile := "range_esc_closure_linedir.go"
+	srcPath := filepath.Join("testdata", testFile)
+	dstPath := filepath.Join(tmpdir, testFile)
+	src, err := os.ReadFile(srcPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(dstPath, src, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Build the module (not as a single file, so go.mod is respected)
+	cmd := testenv.Command(t, gocmd, "build", "-o", output, ".")
+	cmd.Dir = tmpdir
+	b, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("build output: %s", b)
+		t.Fatal(err)
+	}
+	t.Logf("build output: %s", b)
+
+	cmd = testenv.Command(t, output)
+	b, err = cmd.CombinedOutput()
+	t.Logf("run output: %s", b)
+
+	if err != nil {
+		t.Errorf("expected success (exit code 0), got: %v", err)
+	}
+}

src/cmd/compile/internal/loopvar/testdata/range_esc_closure_linedir.go

@@ -0,0 +1,24 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//line range_esc_closure_linedir.go:5
+package main
+
+import "fmt"
+
+var is []func() int
+
+func main() {
+	var ints = []int{0, 0, 0}
+	for i := range ints {
+		is = append(is, func() int { return i })
+	}
+
+	for _, f := range is {
+		fmt.Println(f())
+		if f() != 2 {
+			panic("loop variable i: expected shared per-loop, but got distinct per-iteration")
+		}
+	}
+}

src/cmd/compile/internal/mips/ssa.go

@@ -197,7 +197,6 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpMIPSANDconst,
 		ssa.OpMIPSORconst,
 		ssa.OpMIPSXORconst,
-		ssa.OpMIPSNORconst,
 		ssa.OpMIPSSLLconst,
 		ssa.OpMIPSSRLconst,
 		ssa.OpMIPSSRAconst,

src/cmd/compile/internal/mips64/ssa.go

@@ -191,7 +191,6 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpMIPS64ANDconst,
 		ssa.OpMIPS64ORconst,
 		ssa.OpMIPS64XORconst,
-		ssa.OpMIPS64NORconst,
 		ssa.OpMIPS64SLLVconst,
 		ssa.OpMIPS64SRLVconst,
 		ssa.OpMIPS64SRAVconst,

src/cmd/compile/internal/noder/irgen.go

@@ -56,7 +56,6 @@ func checkFiles(m posMap, noders []*noder) (*types2.Package, *types2.Info, map[*
 		IgnoreBranchErrors: true, // parser already checked via syntax.CheckBranches mode
 		Importer:           &importer,
 		Sizes:              types2.SizesFor("gc", buildcfg.GOARCH),
-		EnableAlias:        true,
 	}
 	if base.Flag.ErrorURL {
 		conf.ErrorURL = " [go.dev/e/%s]"

src/cmd/compile/internal/noder/reader.go

@@ -3342,6 +3342,9 @@ func (r *reader) pkgInitOrder(target *ir.Package) {
 	// Outline (if legal/profitable) global map inits.
 	staticinit.OutlineMapInits(fn)
 
+	// Split large init function.
+	staticinit.SplitLargeInit(fn)
+
 	target.Inits = append(target.Inits, fn)
 }
 

src/cmd/compile/internal/noder/writer.go

@@ -1557,7 +1557,7 @@ func (w *writer) forStmt(stmt *syntax.ForStmt) {
 
 func (w *writer) distinctVars(stmt *syntax.ForStmt) bool {
 	lv := base.Debug.LoopVar
-	fileVersion := w.p.info.FileVersions[stmt.Pos().Base()]
+	fileVersion := w.p.info.FileVersions[stmt.Pos().FileBase()]
 	is122 := fileVersion == "" || version.Compare(fileVersion, "go1.22") >= 0
 
 	// Turning off loopvar for 1.22 is only possible with loopvarhash=qn

src/cmd/compile/internal/pkginit/init.go

@@ -87,10 +87,7 @@ func MakeTask() {
 
 	// Record user init functions.
 	for _, fn := range typecheck.Target.Inits {
-		if fn.Sym().Name == "init" {
-			// Synthetic init function for initialization of package-scope
-			// variables. We can use staticinit to optimize away static
-			// assignments.
+		if staticinit.CanOptimize(fn) {
 			s := staticinit.Schedule{
 				Plans: make(map[ir.Node]*staticinit.Plan),
 				Temps: make(map[ir.Node]*ir.Name),

src/cmd/compile/internal/reflectdata/map.go

@@ -271,10 +271,10 @@ func writeMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) {
 
 	slotTyp := gtyp.Field(1).Type.Elem()
 	elemOff := slotTyp.Field(1).Offset
-	if AlgType(t.Key()) == types.AMEM64 && elemOff != 8 {
+	if types.AlgType(t.Key()) == types.AMEM && t.Key().Size() == 8 && elemOff != 8 {
 		base.Fatalf("runtime assumes elemOff for 8-byte keys is 8, got %d", elemOff)
 	}
-	if AlgType(t.Key()) == types.ASTRING && elemOff != int64(2*types.PtrSize) {
+	if types.AlgType(t.Key()) == types.ASTRING && elemOff != int64(2*types.PtrSize) {
 		base.Fatalf("runtime assumes elemOff for string keys is %d, got %d", 2*types.PtrSize, elemOff)
 	}
 

src/cmd/compile/internal/reflectdata/reflect.go

@@ -756,6 +756,9 @@ func writeType(t *types.Type) *obj.LSym {
 	// | method list, if any            |   dextratype
 	// +--------------------------------+                            - E
 
+	// runtime.moduleTypelinks is aware of this type layout,
+	// and must be changed if the layout change.
+
 	// UncommonType section is included if we have a name or a method.
 	extra := t.Sym() != nil || len(methods(t)) != 0
 

src/cmd/compile/internal/slice/slice.go

@@ -124,6 +124,11 @@ func Funcs(all []*ir.Func) {
 	for _, fn := range all {
 		analyze(fn)
 	}
+	for _, fn := range all {
+		if ir.MatchAstDump(fn, "slice") {
+			ir.AstDump(fn, "slice, "+ir.FuncName(fn))
+		}
+	}
 }
 
 func analyze(fn *ir.Func) {

src/cmd/compile/internal/ssa/_gen/AMD64.rules

@@ -598,30 +598,30 @@
 // mutandis, for UGE and SETAE, and CC and SETCC.
 ((NE|EQ) (TESTL (SHLL (MOVLconst [1]) x) y)) => ((ULT|UGE) (BTL x y))
 ((NE|EQ) (TESTQ (SHLQ (MOVQconst [1]) x) y)) => ((ULT|UGE) (BTQ x y))
-((NE|EQ) (TESTLconst [c] x)) && isUnsignedPowerOfTwo(uint32(c))
+((NE|EQ) (TESTLconst [c] x)) && isPowerOfTwo(uint32(c))
     => ((ULT|UGE) (BTLconst [int8(log32u(uint32(c)))] x))
-((NE|EQ) (TESTQconst [c] x)) && isUnsignedPowerOfTwo(uint64(c))
+((NE|EQ) (TESTQconst [c] x)) && isPowerOfTwo(uint64(c))
     => ((ULT|UGE) (BTQconst [int8(log32u(uint32(c)))] x))
-((NE|EQ) (TESTQ (MOVQconst [c]) x)) && isUnsignedPowerOfTwo(uint64(c))
+((NE|EQ) (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(uint64(c))
     => ((ULT|UGE) (BTQconst [int8(log64u(uint64(c)))] x))
 (SET(NE|EQ) (TESTL (SHLL (MOVLconst [1]) x) y)) => (SET(B|AE)  (BTL x y))
 (SET(NE|EQ) (TESTQ (SHLQ (MOVQconst [1]) x) y)) => (SET(B|AE)  (BTQ x y))
-(SET(NE|EQ) (TESTLconst [c] x)) && isUnsignedPowerOfTwo(uint32(c))
+(SET(NE|EQ) (TESTLconst [c] x)) && isPowerOfTwo(uint32(c))
     => (SET(B|AE)  (BTLconst [int8(log32u(uint32(c)))] x))
-(SET(NE|EQ) (TESTQconst [c] x)) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ) (TESTQconst [c] x)) && isPowerOfTwo(uint64(c))
     => (SET(B|AE)  (BTQconst [int8(log32u(uint32(c)))] x))
-(SET(NE|EQ) (TESTQ (MOVQconst [c]) x)) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ) (TESTQ (MOVQconst [c]) x)) && isPowerOfTwo(uint64(c))
     => (SET(B|AE)  (BTQconst [int8(log64u(uint64(c)))] x))
 // SET..store variant
 (SET(NE|EQ)store [off] {sym} ptr (TESTL (SHLL (MOVLconst [1]) x) y) mem)
     => (SET(B|AE)store  [off] {sym} ptr (BTL x y) mem)
 (SET(NE|EQ)store [off] {sym} ptr (TESTQ (SHLQ (MOVQconst [1]) x) y) mem)
     => (SET(B|AE)store  [off] {sym} ptr (BTQ x y) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTLconst [c] x) mem) && isUnsignedPowerOfTwo(uint32(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTLconst [c] x) mem) && isPowerOfTwo(uint32(c))
     => (SET(B|AE)store  [off] {sym} ptr (BTLconst [int8(log32u(uint32(c)))] x) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTQconst [c] x) mem) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTQconst [c] x) mem) && isPowerOfTwo(uint64(c))
     => (SET(B|AE)store  [off] {sym} ptr (BTQconst [int8(log32u(uint32(c)))] x) mem)
-(SET(NE|EQ)store [off] {sym} ptr (TESTQ (MOVQconst [c]) x) mem) && isUnsignedPowerOfTwo(uint64(c))
+(SET(NE|EQ)store [off] {sym} ptr (TESTQ (MOVQconst [c]) x) mem) && isPowerOfTwo(uint64(c))
     => (SET(B|AE)store  [off] {sym} ptr (BTQconst [int8(log64u(uint64(c)))] x) mem)
 
 // Handle bit-testing in the form (a>>b)&1 != 0 by building the above rules
@@ -647,14 +647,14 @@
 (XOR(Q|L) (SHL(Q|L) (MOV(Q|L)const [1]) y) x) => (BTC(Q|L) x y)
 // Note: only convert OR/XOR to BTS/BTC if the constant wouldn't fit in
 // the constant field of the OR/XOR instruction. See issue 61694.
-((OR|XOR)Q (MOVQconst [c]) x) && isUnsignedPowerOfTwo(uint64(c)) && uint64(c) >= 1<<31 => (BT(S|C)Qconst [int8(log64u(uint64(c)))] x)
+((OR|XOR)Q (MOVQconst [c]) x) && isPowerOfTwo(uint64(c)) && uint64(c) >= 1<<31 => (BT(S|C)Qconst [int8(log64u(uint64(c)))] x)
 
 // Recognize bit clearing: a &^= 1<<b
 (AND(Q|L) (NOT(Q|L) (SHL(Q|L) (MOV(Q|L)const [1]) y)) x) => (BTR(Q|L) x y)
 (ANDN(Q|L) x (SHL(Q|L) (MOV(Q|L)const [1]) y)) => (BTR(Q|L) x y)
 // Note: only convert AND to BTR if the constant wouldn't fit in
 // the constant field of the AND instruction. See issue 61694.
-(ANDQ (MOVQconst [c]) x) && isUnsignedPowerOfTwo(uint64(^c)) && uint64(^c) >= 1<<31 => (BTRQconst [int8(log64u(uint64(^c)))] x)
+(ANDQ (MOVQconst [c]) x) && isPowerOfTwo(uint64(^c)) && uint64(^c) >= 1<<31 => (BTRQconst [int8(log64u(uint64(^c)))] x)
 
 // Special-case bit patterns on first/last bit.
 // generic.rules changes ANDs of high-part/low-part masks into a couple of shifts,
@@ -1679,21 +1679,21 @@
 (Cvt8toMask64x8 <t> x) => (VPMOVMToVec64x8 <types.TypeVec512> (KMOVBk <t> x))
 
 // masks to integers
-(CvtMask8x16to16 <t> x) => (KMOVWi <t> (VPMOVVec8x16ToM <types.TypeMask> x))
-(CvtMask8x32to32 <t> x) => (KMOVDi <t> (VPMOVVec8x32ToM <types.TypeMask> x))
-(CvtMask8x64to64 <t> x) => (KMOVQi <t> (VPMOVVec8x64ToM <types.TypeMask> x))
+(CvtMask8x16to16 ...) => (VPMOVMSKB128 ...)
+(CvtMask8x32to32 ...) => (VPMOVMSKB256 ...)
+(CvtMask8x64to64 x) => (KMOVQi (VPMOVVec8x64ToM <types.TypeMask> x))
 
-(CvtMask16x8to8 <t> x) => (KMOVBi <t> (VPMOVVec16x8ToM <types.TypeMask> x))
-(CvtMask16x16to16 <t> x) => (KMOVWi <t> (VPMOVVec16x16ToM <types.TypeMask> x))
-(CvtMask16x32to32 <t> x) => (KMOVDi <t> (VPMOVVec16x32ToM <types.TypeMask> x))
+(CvtMask16x8to8 x) => (KMOVBi (VPMOVVec16x8ToM <types.TypeMask> x))
+(CvtMask16x16to16 x) => (KMOVWi (VPMOVVec16x16ToM <types.TypeMask> x))
+(CvtMask16x32to32 x) => (KMOVDi (VPMOVVec16x32ToM <types.TypeMask> x))
 
-(CvtMask32x4to8 <t> x) => (KMOVBi <t> (VPMOVVec32x4ToM <types.TypeMask> x))
-(CvtMask32x8to8 <t> x) => (KMOVBi <t> (VPMOVVec32x8ToM <types.TypeMask> x))
-(CvtMask32x16to16 <t> x) => (KMOVWi <t> (VPMOVVec32x16ToM <types.TypeMask> x))
+(CvtMask32x4to8 ...) => (VMOVMSKPS128 ...)
+(CvtMask32x8to8 ...) => (VMOVMSKPS256 ...)
+(CvtMask32x16to16 x) => (KMOVWi (VPMOVVec32x16ToM <types.TypeMask> x))
 
-(CvtMask64x2to8 <t> x) => (KMOVBi <t> (VPMOVVec64x2ToM <types.TypeMask> x))
-(CvtMask64x4to8 <t> x) => (KMOVBi <t> (VPMOVVec64x4ToM <types.TypeMask> x))
-(CvtMask64x8to8 <t> x) => (KMOVBi <t> (VPMOVVec64x8ToM <types.TypeMask> x))
+(CvtMask64x2to8 ...) => (VMOVMSKPD128 ...)
+(CvtMask64x4to8 ...) => (VMOVMSKPD256 ...)
+(CvtMask64x8to8 x) => (KMOVBi (VPMOVVec64x8ToM <types.TypeMask> x))
 
 // optimizations
 (MOVBstore [off] {sym} ptr (KMOVBi mask) mem) => (KMOVBstore [off] {sym} ptr mask mem)
@@ -1730,6 +1730,13 @@
 // Misc
 (IsZeroVec x) => (SETEQ (VPTEST x x))
 
+(IsNaNFloat32x4  x) => (VCMPPS128 [3] x x)
+(IsNaNFloat32x8  x) => (VCMPPS256 [3] x x)
+(IsNaNFloat32x16 x) => (VPMOVMToVec32x16 (VCMPPS512 [3] x x))
+(IsNaNFloat64x2  x) => (VCMPPD128 [3] x x)
+(IsNaNFloat64x4  x) => (VCMPPD256 [3] x x)
+(IsNaNFloat64x8  x) => (VPMOVMToVec64x8 (VCMPPD512 [3] x x))
+
 // SIMD vector K-masked loads and stores
 
 (LoadMasked64 <t> ptr mask mem) && t.Size() == 64 => (VPMASK64load512 ptr (VPMOVVec64x8ToM  <types.TypeMask> mask) mem)
@@ -1818,10 +1825,10 @@
 (EQ (VPTEST x:(VPANDN(128|256) j k) y) yes no) && x == y && x.Uses == 2 => (ULT (VPTEST k j) yes no) // AndNot has swapped its operand order
 (EQ (VPTEST x:(VPANDN(D|Q)512 j k) y) yes no) && x == y && x.Uses == 2 => (ULT (VPTEST k j) yes no) // AndNot has swapped its operand order
 
-// DotProductQuadruple optimizations
-(VPADDD128 (VPDPBUSD128 (Zero128 <t>) x y) z) => (VPDPBUSD128 <t> z x y)
-(VPADDD256 (VPDPBUSD256 (Zero256 <t>) x y) z) => (VPDPBUSD256 <t> z x y)
-(VPADDD512 (VPDPBUSD512 (Zero512 <t>) x y) z) => (VPDPBUSD512 <t> z x y)
-(VPADDD128 (VPDPBUSDS128 (Zero128 <t>) x y) z) => (VPDPBUSDS128 <t> z x y)
-(VPADDD256 (VPDPBUSDS256 (Zero256 <t>) x y) z) => (VPDPBUSDS256 <t> z x y)
-(VPADDD512 (VPDPBUSDS512 (Zero512 <t>) x y) z) => (VPDPBUSDS512 <t> z x y)
+// optimize x.IsNaN().Or(y.IsNaN())
+(VPOR128 (VCMPP(S|D)128 [3] x x) (VCMPP(S|D)128 [3] y y)) => (VCMPP(S|D)128 [3] x y)
+(VPOR256 (VCMPP(S|D)256 [3] x x) (VCMPP(S|D)256 [3] y y)) => (VCMPP(S|D)256 [3] x y)
+(VPORD512 (VPMOVMToVec32x16 (VCMPPS512 [3] x x)) (VPMOVMToVec32x16 (VCMPPS512 [3] y y))) =>
+	(VPMOVMToVec32x16 (VCMPPS512 [3] x y))
+(VPORD512 (VPMOVMToVec64x8  (VCMPPD512 [3] x x)) (VPMOVMToVec64x8  (VCMPPD512 [3] y y))) =>
+	(VPMOVMToVec64x8  (VCMPPD512 [3] x y))

src/cmd/compile/internal/ssa/_gen/AMD64Ops.go

@@ -1368,6 +1368,7 @@ func init() {
 		{name: "VPMASK64load512", argLength: 3, reg: vloadk, asm: "VMOVDQU64", aux: "SymOff", faultOnNilArg0: true, symEffect: "Read"},    // load from arg0+auxint+aux, arg1=k mask, arg2 = mem
 		{name: "VPMASK64store512", argLength: 4, reg: vstorek, asm: "VMOVDQU64", aux: "SymOff", faultOnNilArg0: true, symEffect: "Write"}, // store, *(arg0+auxint+aux) = arg2, arg1=k mask, arg3 = mem
 
+		// AVX512 moves between int-vector and mask registers
 		{name: "VPMOVMToVec8x16", argLength: 1, reg: kv, asm: "VPMOVM2B"},
 		{name: "VPMOVMToVec8x32", argLength: 1, reg: kv, asm: "VPMOVM2B"},
 		{name: "VPMOVMToVec8x64", argLength: 1, reg: kw, asm: "VPMOVM2B"},
@@ -1400,6 +1401,14 @@ func init() {
 		{name: "VPMOVVec64x4ToM", argLength: 1, reg: vk, asm: "VPMOVQ2M"},
 		{name: "VPMOVVec64x8ToM", argLength: 1, reg: wk, asm: "VPMOVQ2M"},
 
+		// AVX1/2 moves from int-vector to bitmask (extracting sign bits)
+		{name: "VPMOVMSKB128", argLength: 1, reg: vgp, asm: "VPMOVMSKB"},
+		{name: "VPMOVMSKB256", argLength: 1, reg: vgp, asm: "VPMOVMSKB"},
+		{name: "VMOVMSKPS128", argLength: 1, reg: vgp, asm: "VMOVMSKPS"},
+		{name: "VMOVMSKPS256", argLength: 1, reg: vgp, asm: "VMOVMSKPS"},
+		{name: "VMOVMSKPD128", argLength: 1, reg: vgp, asm: "VMOVMSKPD"},
+		{name: "VMOVMSKPD256", argLength: 1, reg: vgp, asm: "VMOVMSKPD"},
+
 		// X15 is the zero register up to 128-bit. For larger values, we zero it on the fly.
 		{name: "Zero128", argLength: 0, reg: x15only, zeroWidth: true, fixedReg: true},
 		{name: "Zero256", argLength: 0, reg: v01, asm: "VPXOR"},

src/cmd/compile/internal/ssa/_gen/AMD64latelower.rules

@@ -8,6 +8,6 @@
 (SHR(Q|L) x y) && buildcfg.GOAMD64 >= 3 => (SHRX(Q|L) x y)
 
 // See comments in ARM64latelower.rules for why these are here.
-(MOVLQZX x) && zeroUpper32Bits(x,3) => x
-(MOVWQZX x) && zeroUpper48Bits(x,3) => x
-(MOVBQZX x) && zeroUpper56Bits(x,3) => x
+(MOVLQZX x) && ZeroUpper32Bits(x,3) => x
+(MOVWQZX x) && ZeroUpper48Bits(x,3) => x
+(MOVBQZX x) && ZeroUpper56Bits(x,3) => x

src/cmd/compile/internal/ssa/_gen/ARM64latelower.rules

@@ -29,7 +29,7 @@
 (MOVBUreg x:((Equal|NotEqual|LessThan|LessThanU|LessThanF|LessEqual|LessEqualU|LessEqualF|GreaterThan|GreaterThanU|GreaterThanF|GreaterEqual|GreaterEqualU|GreaterEqualF) _)) => x
 
 // omit unsigned extension
-(MOVWUreg x) && zeroUpper32Bits(x, 3) => x
+(MOVWUreg x) && ZeroUpper32Bits(x, 3) => x
 
 // don't extend after proper load
 (MOVBreg  x:(MOVBload  _ _)) => (MOVDreg x)

src/cmd/compile/internal/ssa/_gen/LOONG64.rules

@@ -773,6 +773,9 @@
 (SUBF (NEGF (MULF x y)) z) && z.Block.Func.useFMA(v) => (FNMADDF x y z)
 (SUBD (NEGD (MULD x y)) z) && z.Block.Func.useFMA(v) => (FNMADDD x y z)
 
+// Absorb conversion between 32 bit and 64 bit if both src and dst are 32 bit.
+(MOVDF ((ABS|SQRT)D (MOVFD x))) => ((ABS|SQRT)F x)
+
 // generic simplifications
 (ADDV x (NEGV y)) => (SUBV x y)
 (SUBV x (NEGV y)) => (ADDV x y)
@@ -843,6 +846,11 @@
 
 (MOVBUreg (ANDconst [c] x)) => (ANDconst [c&0xff] x)
 
+// Avoid extending when already sufficiently shifted.
+(MOVBUreg x:(SRLconst [c] y)) && c >= 24 => x
+(MOVHUreg x:(SRLconst [c] y)) && c >= 16 => x
+(MOVWUreg x:(SRLconst [c] y)) => x
+
 // Avoid extending when already sufficiently masked.
 (MOVBreg  x:(ANDconst [c] y)) && c >= 0 && int64(int8(c)) == c => x
 (MOVHreg  x:(ANDconst [c] y)) && c >= 0 && int64(int16(c)) == c => x

src/cmd/compile/internal/ssa/_gen/LOONG64Ops.go

@@ -169,6 +169,7 @@ func init() {
 		{name: "SQRTD", argLength: 1, reg: fp11, asm: "SQRTD"}, // sqrt(arg0), float64
 		{name: "SQRTF", argLength: 1, reg: fp11, asm: "SQRTF"}, // sqrt(arg0), float32
 
+		{name: "ABSF", argLength: 1, reg: fp11, asm: "ABSF"}, // abs(arg0), float32
 		{name: "ABSD", argLength: 1, reg: fp11, asm: "ABSD"}, // abs(arg0), float64
 
 		{name: "CLZW", argLength: 1, reg: gp11, asm: "CLZW"}, // Count leading (high order) zeroes (returns 0-32)

src/cmd/compile/internal/ssa/_gen/MIPS.rules

@@ -127,7 +127,7 @@
 (Neg(32|16|8) ...) => (NEG ...)
 (Neg(32|64)F ...) => (NEG(F|D) ...)
 
-(Com(32|16|8) x) => (NORconst [0] x)
+(Com(32|16|8) x) => (NOR (MOVWconst [0]) x)
 
 (Sqrt ...) => (SQRTD ...)
 (Sqrt32 ...) => (SQRTF ...)
@@ -382,7 +382,7 @@
 		(OR <typ.UInt32> (SLL <typ.UInt32> (ZeroExt8to32 val)
 			(SLLconst <typ.UInt32> [3]
 				(ANDconst  <typ.UInt32> [3] ptr)))
-		(NORconst [0] <typ.UInt32> (SLL <typ.UInt32>
+		(NOR (MOVWconst [0]) <typ.UInt32> (SLL <typ.UInt32>
 			(MOVWconst [0xff]) (SLLconst <typ.UInt32> [3]
 				(ANDconst <typ.UInt32> [3] ptr))))) mem)
 
@@ -401,7 +401,7 @@
 			(SLLconst <typ.UInt32> [3]
 				(ANDconst  <typ.UInt32> [3]
 					(XORconst <typ.UInt32> [3] ptr))))
-		(NORconst [0] <typ.UInt32> (SLL <typ.UInt32>
+		(NOR (MOVWconst [0]) <typ.UInt32> (SLL <typ.UInt32>
 			(MOVWconst [0xff]) (SLLconst <typ.UInt32> [3]
 				(ANDconst <typ.UInt32> [3]
 					(XORconst <typ.UInt32> [3] ptr)))))) mem)
@@ -599,7 +599,6 @@
 (AND x (MOVWconst [c])) => (ANDconst [c] x)
 (OR  x (MOVWconst [c])) => (ORconst  [c] x)
 (XOR x (MOVWconst [c])) => (XORconst [c] x)
-(NOR x (MOVWconst [c])) => (NORconst [c] x)
 
 (SLL x (MOVWconst [c])) => (SLLconst x [c&31])
 (SRL x (MOVWconst [c])) => (SRLconst x [c&31])
@@ -617,13 +616,13 @@
 (Select0 (MULTU (MOVWconst [1])  _ )) => (MOVWconst [0])
 (Select1 (MULTU (MOVWconst [-1]) x )) => (NEG <x.Type> x)
 (Select0 (MULTU (MOVWconst [-1]) x )) => (CMOVZ (ADDconst <x.Type> [-1] x) (MOVWconst [0]) x)
-(Select1 (MULTU (MOVWconst [c])  x )) && isUnsignedPowerOfTwo(uint32(c)) => (SLLconst [int32(log32u(uint32(c)))] x)
-(Select0 (MULTU (MOVWconst [c])  x )) && isUnsignedPowerOfTwo(uint32(c)) => (SRLconst [int32(32-log32u(uint32(c)))] x)
+(Select1 (MULTU (MOVWconst [c])  x )) && isPowerOfTwo(uint32(c)) => (SLLconst [int32(log32u(uint32(c)))] x)
+(Select0 (MULTU (MOVWconst [c])  x )) && isPowerOfTwo(uint32(c)) => (SRLconst [int32(32-log32u(uint32(c)))] x)
 
 (MUL (MOVWconst [0])  _ ) => (MOVWconst [0])
 (MUL (MOVWconst [1])  x ) => x
 (MUL (MOVWconst [-1]) x ) => (NEG x)
-(MUL (MOVWconst [c]) x ) && isUnsignedPowerOfTwo(uint32(c)) => (SLLconst [int32(log32u(uint32(c)))] x)
+(MUL (MOVWconst [c]) x ) && isPowerOfTwo(uint32(c)) => (SLLconst [int32(log32u(uint32(c)))] x)
 
 // generic simplifications
 (ADD x (NEG y)) => (SUB x y)
@@ -648,7 +647,6 @@
 (ORconst  [0]  x) => x
 (ORconst  [-1] _) => (MOVWconst [-1])
 (XORconst [0]  x) => x
-(XORconst [-1] x) => (NORconst [0] x)
 
 // generic constant folding
 (ADDconst [c] (MOVWconst [d]))  => (MOVWconst [int32(c+d)])
@@ -673,7 +671,6 @@
 (ORconst [c] (ORconst [d] x)) => (ORconst [c|d] x)
 (XORconst [c] (MOVWconst [d])) => (MOVWconst [c^d])
 (XORconst [c] (XORconst [d] x)) => (XORconst [c^d] x)
-(NORconst [c] (MOVWconst [d])) => (MOVWconst [^(c|d)])
 (NEG (MOVWconst [c])) => (MOVWconst [-c])
 (MOVBreg  (MOVWconst [c])) => (MOVWconst [int32(int8(c))])
 (MOVBUreg (MOVWconst [c])) => (MOVWconst [int32(uint8(c))])

src/cmd/compile/internal/ssa/_gen/MIPS64.rules

@@ -433,7 +433,7 @@
 		(OR <typ.UInt64> (SLLV <typ.UInt32> (ZeroExt8to32 val)
 			(SLLVconst <typ.UInt64> [3]
 				(ANDconst  <typ.UInt64> [3] ptr)))
-		(NORconst [0] <typ.UInt64> (SLLV <typ.UInt64>
+		(NOR (MOVVconst [0]) <typ.UInt64> (SLLV <typ.UInt64>
 			(MOVVconst [0xff]) (SLLVconst <typ.UInt64> [3]
 				(ANDconst <typ.UInt64> [3] ptr))))) mem)
 
@@ -452,7 +452,7 @@
 			(SLLVconst <typ.UInt64> [3]
 				(ANDconst  <typ.UInt64> [3]
 					(XORconst <typ.UInt64> [3] ptr))))
-		(NORconst [0] <typ.UInt64> (SLLV <typ.UInt64>
+		(NOR (MOVVconst [0]) <typ.UInt64> (SLLV <typ.UInt64>
 			(MOVVconst [0xff]) (SLLVconst <typ.UInt64> [3]
 				(ANDconst <typ.UInt64> [3]
 					(XORconst <typ.UInt64> [3] ptr)))))) mem)
@@ -668,7 +668,6 @@
 (AND x (MOVVconst [c])) && is32Bit(c) => (ANDconst [c] x)
 (OR  x (MOVVconst [c])) && is32Bit(c) => (ORconst  [c] x)
 (XOR x (MOVVconst [c])) && is32Bit(c) => (XORconst [c] x)
-(NOR x (MOVVconst [c])) && is32Bit(c) => (NORconst [c] x)
 
 (SLLV _ (MOVVconst [c])) && uint64(c)>=64 => (MOVVconst [0])
 (SRLV _ (MOVVconst [c])) && uint64(c)>=64 => (MOVVconst [0])
@@ -711,7 +710,6 @@
 (ORconst  [0]  x) => x
 (ORconst  [-1] _) => (MOVVconst [-1])
 (XORconst [0]  x) => x
-(XORconst [-1] x) => (NORconst [0] x)
 
 // generic constant folding
 (ADDVconst [c] (MOVVconst [d]))  => (MOVVconst [c+d])
@@ -734,7 +732,6 @@
 (ORconst [c] (ORconst [d] x)) && is32Bit(c|d) => (ORconst [c|d] x)
 (XORconst [c] (MOVVconst [d])) => (MOVVconst [c^d])
 (XORconst [c] (XORconst [d] x)) && is32Bit(c^d) => (XORconst [c^d] x)
-(NORconst [c] (MOVVconst [d])) => (MOVVconst [^(c|d)])
 (NEGV (MOVVconst [c])) => (MOVVconst [-c])
 (MOVBreg  (MOVVconst [c])) => (MOVVconst [int64(int8(c))])
 (MOVBUreg (MOVVconst [c])) => (MOVVconst [int64(uint8(c))])

src/cmd/compile/internal/ssa/_gen/MIPS64Ops.go

@@ -189,7 +189,6 @@ func init() {
 		{name: "XOR", argLength: 2, reg: gp21, asm: "XOR", commutative: true, typ: "UInt64"}, // arg0 ^ arg1
 		{name: "XORconst", argLength: 1, reg: gp11, asm: "XOR", aux: "Int64", typ: "UInt64"}, // arg0 ^ auxInt
 		{name: "NOR", argLength: 2, reg: gp21, asm: "NOR", commutative: true},                // ^(arg0 | arg1)
-		{name: "NORconst", argLength: 1, reg: gp11, asm: "NOR", aux: "Int64"},                // ^(arg0 | auxInt)
 
 		{name: "NEGV", argLength: 1, reg: gp11},                // -arg0
 		{name: "NEGF", argLength: 1, reg: fp11, asm: "NEGF"},   // -arg0, float32

src/cmd/compile/internal/ssa/_gen/MIPSOps.go

@@ -173,7 +173,6 @@ func init() {
 		{name: "XOR", argLength: 2, reg: gp21, asm: "XOR", commutative: true, typ: "UInt32"}, // arg0 ^ arg1
 		{name: "XORconst", argLength: 1, reg: gp11, asm: "XOR", aux: "Int32", typ: "UInt32"}, // arg0 ^ auxInt
 		{name: "NOR", argLength: 2, reg: gp21, asm: "NOR", commutative: true},                // ^(arg0 | arg1)
-		{name: "NORconst", argLength: 1, reg: gp11, asm: "NOR", aux: "Int32"},                // ^(arg0 | auxInt)
 
 		{name: "NEG", argLength: 1, reg: gp11},                 // -arg0
 		{name: "NEGF", argLength: 1, reg: fp11, asm: "NEGF"},   // -arg0, float32

src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go

@@ -118,6 +118,7 @@ func init() {
 	regCtxt := regNamed["X26"]
 	callerSave := gpMask | fpMask | regNamed["g"]
 	r5toR6 := regNamed["X5"] | regNamed["X6"]
+	regX5 := regNamed["X5"]
 
 	var (
 		gpstore  = regInfo{inputs: []regMask{gpspsbMask, gpspMask, 0}} // SB in first input so we can load from a global, but not in second to avoid using SB as a temporary register
@@ -142,9 +143,13 @@ func init() {
 		fpload  = regInfo{inputs: []regMask{gpspsbMask, 0}, outputs: []regMask{fpMask}}
 		fp2gp   = regInfo{inputs: []regMask{fpMask, fpMask}, outputs: []regMask{gpMask}}
 
-		call        = regInfo{clobbers: callerSave}
-		callClosure = regInfo{inputs: []regMask{gpspMask, regCtxt, 0}, clobbers: callerSave}
-		callInter   = regInfo{inputs: []regMask{gpMask}, clobbers: callerSave}
+		call = regInfo{clobbers: callerSave}
+		// Avoid using X5 as the source register of calls. Using X5 here triggers
+		// RAS pop-then-push behavior which is not correct for function calls.
+		// Please refer to section 2.5.1 of the RISC-V ISA
+		// (https://docs.riscv.org/reference/isa/unpriv/rv32.html#rashints) for details.
+		callClosure = regInfo{inputs: []regMask{gpspMask ^ regX5, regCtxt, 0}, clobbers: callerSave}
+		callInter   = regInfo{inputs: []regMask{gpMask ^ regX5}, clobbers: callerSave}
 	)
 
 	RISCV64ops := []opData{

src/cmd/compile/internal/ssa/_gen/divmod.rules

@@ -223,7 +223,7 @@
     (Rsh64Ux64 <typ.UInt64>
       (Avg64u
         (Lsh64x64 <typ.UInt64> (ZeroExt32to64 x) (Const64 <typ.UInt64> [32]))
-        (Mul64 <typ.UInt64> (ZeroExt32to64 x) (Const64 <typ.UInt32> [int64(umagic32(c).m)])))
+        (Mul64 <typ.UInt64> (ZeroExt32to64 x) (Const64 <typ.UInt64> [int64(umagic32(c).m)])))
       (Const64 <typ.UInt64> [32 + umagic32(c).s - 1])))
 (Div32u <t> x (Const32 [c])) && umagicOK32(c) && config.RegSize == 4 =>
   (Rsh32Ux64 <t>

src/cmd/compile/internal/ssa/_gen/generic.rules

@@ -1063,10 +1063,10 @@
 (Div64 <t> x (Const64 [-1<<63])) => (Rsh64Ux64 (And64 <t> x (Neg64 <t> x)) (Const64 <typ.UInt64> [63]))
 
 // Unsigned divide by power of 2.  Strength reduce to a shift.
-(Div8u  n (Const8  [c])) && isUnsignedPowerOfTwo(uint8(c)) => (Rsh8Ux64  n (Const64 <typ.UInt64> [log8u(uint8(c))]))
-(Div16u n (Const16 [c])) && isUnsignedPowerOfTwo(uint16(c)) => (Rsh16Ux64 n (Const64 <typ.UInt64> [log16u(uint16(c))]))
-(Div32u n (Const32 [c])) && isUnsignedPowerOfTwo(uint32(c)) => (Rsh32Ux64 n (Const64 <typ.UInt64> [log32u(uint32(c))]))
-(Div64u n (Const64 [c])) && isUnsignedPowerOfTwo(uint64(c)) => (Rsh64Ux64 n (Const64 <typ.UInt64> [log64u(uint64(c))]))
+(Div8u  n (Const8  [c])) && isPowerOfTwo(uint8(c)) => (Rsh8Ux64  n (Const64 <typ.UInt64> [log8u(uint8(c))]))
+(Div16u n (Const16 [c])) && isPowerOfTwo(uint16(c)) => (Rsh16Ux64 n (Const64 <typ.UInt64> [log16u(uint16(c))]))
+(Div32u n (Const32 [c])) && isPowerOfTwo(uint32(c)) => (Rsh32Ux64 n (Const64 <typ.UInt64> [log32u(uint32(c))]))
+(Div64u n (Const64 [c])) && isPowerOfTwo(uint64(c)) => (Rsh64Ux64 n (Const64 <typ.UInt64> [log64u(uint64(c))]))
 
 // Strength reduce multiplication by a power of two to a shift.
 // Excluded from early opt so that prove can recognize mod
@@ -1090,13 +1090,13 @@
   (Neg64 (Lsh64x64 <t> x (Const64 <typ.UInt64> [log64(-c)])))
 
 // Strength reduction of mod to div.
-// Strength reduction of div to mul is delayed to genericlateopt.rules.
+// Strength reduction of div to mul is delayed to divmod.rules.
 
 // Unsigned mod by power of 2 constant.
-(Mod8u  <t> n (Const8  [c])) && isUnsignedPowerOfTwo(uint8(c)) => (And8  n (Const8  <t> [c-1]))
-(Mod16u <t> n (Const16 [c])) && isUnsignedPowerOfTwo(uint16(c)) => (And16 n (Const16 <t> [c-1]))
-(Mod32u <t> n (Const32 [c])) && isUnsignedPowerOfTwo(uint32(c)) => (And32 n (Const32 <t> [c-1]))
-(Mod64u <t> n (Const64 [c])) && isUnsignedPowerOfTwo(uint64(c)) => (And64 n (Const64 <t> [c-1]))
+(Mod8u  <t> n (Const8  [c])) && isPowerOfTwo(uint8(c)) => (And8  n (Const8  <t> [c-1]))
+(Mod16u <t> n (Const16 [c])) && isPowerOfTwo(uint16(c)) => (And16 n (Const16 <t> [c-1]))
+(Mod32u <t> n (Const32 [c])) && isPowerOfTwo(uint32(c)) => (And32 n (Const32 <t> [c-1]))
+(Mod64u <t> n (Const64 [c])) && isPowerOfTwo(uint64(c)) => (And64 n (Const64 <t> [c-1]))
 
 // Signed non-negative mod by power of 2 constant.
 // TODO: Replace ModN with ModNu in prove.
@@ -1560,6 +1560,43 @@
 
 (MemEq p q _ _) && isSamePtr(p, q) => (ConstBool <typ.Bool> [true])
 
+// 3-32 bytes memeq (enabled only with support of unaligned loads and 8-byte max word size)
+
+(MemEq p q (Const64 [c]) mem)
+  && (c == 3 || c == 5 || c == 9 || c == 17)
+  && canLoadUnaligned(config)
+  && config.RegSize == 8
+  => (AndB (MemEq p q (Const64 <typ.Int64> [c-1]) mem)
+           (Eq8 (Load <typ.Int8> (OffPtr <p.Type> p [c-1]) mem) (Load <typ.Int8> (OffPtr <q.Type> q [c-1]) mem)))
+
+(MemEq p q (Const64 [c]) mem)
+  && (c == 6 || c == 10 || c == 18)
+  && canLoadUnaligned(config)
+  && config.RegSize == 8
+  => (AndB (MemEq p q (Const64 <typ.Int64> [c-2]) mem)
+           (Eq16 (Load <typ.Int16> (OffPtr <p.Type> p [c-2]) mem) (Load <typ.Int16> (OffPtr <q.Type> q [c-2]) mem)))
+
+(MemEq p q (Const64 [c]) mem)
+  && (c == 7 || c == 11 || c == 19 || c == 20)
+  && canLoadUnaligned(config)
+  && config.RegSize == 8
+  => (AndB (MemEq p q (Const64 <typ.Int64> [min(c-3,16)]) mem)
+           (Eq32 (Load <typ.Int32> (OffPtr <p.Type> p [c-4]) mem) (Load <typ.Int32> (OffPtr <q.Type> q [c-4]) mem)))
+
+(MemEq p q (Const64 [c]) mem)
+  && ((c >= 12 && c <= 16) || (c >= 21 && c <= 24))
+  && canLoadUnaligned(config)
+  && config.RegSize == 8
+  => (AndB (MemEq p q (Const64 <typ.Int64> [8 + int64(bool2int(c>16))*8]) mem)
+           (Eq64 (Load <typ.Int64> (OffPtr <p.Type> p [c-8]) mem) (Load <typ.Int64> (OffPtr <q.Type> q [c-8]) mem)))
+
+(MemEq p q (Const64 [c]) mem)
+  && c >= 25 && c <= 32
+  && canLoadUnaligned(config)
+  && config.RegSize == 8
+  => (AndB (MemEq p q (Const64 <typ.Int64> [16]) mem)
+           (MemEq (OffPtr <p.Type> p [16]) (OffPtr <q.Type> q [16]) (Const64 <typ.Int64> [c-16]) mem))
+
 // Turn known-size calls to memclrNoHeapPointers into a Zero.
 // Note that we are using types.Types[types.TUINT8] instead of sptr.Type.Elem() - see issue 55122 and CL 431496 for more details.
 (SelectN [0] call:(StaticCall {sym} sptr (Const(64|32) [c]) mem))

src/cmd/compile/internal/ssa/_gen/genericOps.go

@@ -715,6 +715,14 @@ var genericOps = []opData{
 
 	// Returns true if arg0 is all zero.
 	{name: "IsZeroVec", argLength: 1},
+
+	// Returns a mask indicating whether arg0's elements are NaN.
+	{name: "IsNaNFloat32x4", argLength: 1},
+	{name: "IsNaNFloat32x8", argLength: 1},
+	{name: "IsNaNFloat32x16", argLength: 1},
+	{name: "IsNaNFloat64x2", argLength: 1},
+	{name: "IsNaNFloat64x4", argLength: 1},
+	{name: "IsNaNFloat64x8", argLength: 1},
 }
 
 //     kind          controls          successors   implicit exit

src/cmd/compile/internal/ssa/_gen/simdAMD64.rules

@@ -1,4 +1,4 @@
-// Code generated by x/arch/internal/simdgen using 'go run . -xedPath $XED_PATH -o godefs -goroot $GOROOT go.yaml types.yaml categories.yaml'; DO NOT EDIT.
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
 
 (AESDecryptLastRoundUint8x16 ...) => (VAESDECLAST128 ...)
 (AESDecryptLastRoundUint8x32 ...) => (VAESDECLAST256 ...)
@@ -57,19 +57,19 @@
 (AddUint64x4 ...) => (VPADDQ256 ...)
 (AddUint64x8 ...) => (VPADDQ512 ...)
 (AddPairsFloat32x4 ...) => (VHADDPS128 ...)
-(AddPairsFloat32x8 ...) => (VHADDPS256 ...)
 (AddPairsFloat64x2 ...) => (VHADDPD128 ...)
-(AddPairsFloat64x4 ...) => (VHADDPD256 ...)
 (AddPairsInt16x8 ...) => (VPHADDW128 ...)
-(AddPairsInt16x16 ...) => (VPHADDW256 ...)
 (AddPairsInt32x4 ...) => (VPHADDD128 ...)
-(AddPairsInt32x8 ...) => (VPHADDD256 ...)
 (AddPairsUint16x8 ...) => (VPHADDW128 ...)
-(AddPairsUint16x16 ...) => (VPHADDW256 ...)
 (AddPairsUint32x4 ...) => (VPHADDD128 ...)
-(AddPairsUint32x8 ...) => (VPHADDD256 ...)
+(AddPairsGroupedFloat32x8 ...) => (VHADDPS256 ...)
+(AddPairsGroupedFloat64x4 ...) => (VHADDPD256 ...)
+(AddPairsGroupedInt16x16 ...) => (VPHADDW256 ...)
+(AddPairsGroupedInt32x8 ...) => (VPHADDD256 ...)
+(AddPairsGroupedUint16x16 ...) => (VPHADDW256 ...)
+(AddPairsGroupedUint32x8 ...) => (VPHADDD256 ...)
 (AddPairsSaturatedInt16x8 ...) => (VPHADDSW128 ...)
-(AddPairsSaturatedInt16x16 ...) => (VPHADDSW256 ...)
+(AddPairsSaturatedGroupedInt16x16 ...) => (VPHADDSW256 ...)
 (AddSaturatedInt8x16 ...) => (VPADDSB128 ...)
 (AddSaturatedInt8x32 ...) => (VPADDSB256 ...)
 (AddSaturatedInt8x64 ...) => (VPADDSB512 ...)
@@ -140,36 +140,36 @@
 (AverageUint16x8 ...) => (VPAVGW128 ...)
 (AverageUint16x16 ...) => (VPAVGW256 ...)
 (AverageUint16x32 ...) => (VPAVGW512 ...)
-(Broadcast128Float32x4 ...) => (VBROADCASTSS128 ...)
-(Broadcast128Float64x2 ...) => (VPBROADCASTQ128 ...)
-(Broadcast128Int8x16 ...) => (VPBROADCASTB128 ...)
-(Broadcast128Int16x8 ...) => (VPBROADCASTW128 ...)
-(Broadcast128Int32x4 ...) => (VPBROADCASTD128 ...)
-(Broadcast128Int64x2 ...) => (VPBROADCASTQ128 ...)
-(Broadcast128Uint8x16 ...) => (VPBROADCASTB128 ...)
-(Broadcast128Uint16x8 ...) => (VPBROADCASTW128 ...)
-(Broadcast128Uint32x4 ...) => (VPBROADCASTD128 ...)
-(Broadcast128Uint64x2 ...) => (VPBROADCASTQ128 ...)
-(Broadcast256Float32x4 ...) => (VBROADCASTSS256 ...)
-(Broadcast256Float64x2 ...) => (VBROADCASTSD256 ...)
-(Broadcast256Int8x16 ...) => (VPBROADCASTB256 ...)
-(Broadcast256Int16x8 ...) => (VPBROADCASTW256 ...)
-(Broadcast256Int32x4 ...) => (VPBROADCASTD256 ...)
-(Broadcast256Int64x2 ...) => (VPBROADCASTQ256 ...)
-(Broadcast256Uint8x16 ...) => (VPBROADCASTB256 ...)
-(Broadcast256Uint16x8 ...) => (VPBROADCASTW256 ...)
-(Broadcast256Uint32x4 ...) => (VPBROADCASTD256 ...)
-(Broadcast256Uint64x2 ...) => (VPBROADCASTQ256 ...)
-(Broadcast512Float32x4 ...) => (VBROADCASTSS512 ...)
-(Broadcast512Float64x2 ...) => (VBROADCASTSD512 ...)
-(Broadcast512Int8x16 ...) => (VPBROADCASTB512 ...)
-(Broadcast512Int16x8 ...) => (VPBROADCASTW512 ...)
-(Broadcast512Int32x4 ...) => (VPBROADCASTD512 ...)
-(Broadcast512Int64x2 ...) => (VPBROADCASTQ512 ...)
-(Broadcast512Uint8x16 ...) => (VPBROADCASTB512 ...)
-(Broadcast512Uint16x8 ...) => (VPBROADCASTW512 ...)
-(Broadcast512Uint32x4 ...) => (VPBROADCASTD512 ...)
-(Broadcast512Uint64x2 ...) => (VPBROADCASTQ512 ...)
+(Broadcast1To2Float64x2 ...) => (VPBROADCASTQ128 ...)
+(Broadcast1To2Int64x2 ...) => (VPBROADCASTQ128 ...)
+(Broadcast1To2Uint64x2 ...) => (VPBROADCASTQ128 ...)
+(Broadcast1To4Float32x4 ...) => (VBROADCASTSS128 ...)
+(Broadcast1To4Float64x2 ...) => (VBROADCASTSD256 ...)
+(Broadcast1To4Int32x4 ...) => (VPBROADCASTD128 ...)
+(Broadcast1To4Int64x2 ...) => (VPBROADCASTQ256 ...)
+(Broadcast1To4Uint32x4 ...) => (VPBROADCASTD128 ...)
+(Broadcast1To4Uint64x2 ...) => (VPBROADCASTQ256 ...)
+(Broadcast1To8Float32x4 ...) => (VBROADCASTSS256 ...)
+(Broadcast1To8Float64x2 ...) => (VBROADCASTSD512 ...)
+(Broadcast1To8Int16x8 ...) => (VPBROADCASTW128 ...)
+(Broadcast1To8Int32x4 ...) => (VPBROADCASTD256 ...)
+(Broadcast1To8Int64x2 ...) => (VPBROADCASTQ512 ...)
+(Broadcast1To8Uint16x8 ...) => (VPBROADCASTW128 ...)
+(Broadcast1To8Uint32x4 ...) => (VPBROADCASTD256 ...)
+(Broadcast1To8Uint64x2 ...) => (VPBROADCASTQ512 ...)
+(Broadcast1To16Float32x4 ...) => (VBROADCASTSS512 ...)
+(Broadcast1To16Int8x16 ...) => (VPBROADCASTB128 ...)
+(Broadcast1To16Int16x8 ...) => (VPBROADCASTW256 ...)
+(Broadcast1To16Int32x4 ...) => (VPBROADCASTD512 ...)
+(Broadcast1To16Uint8x16 ...) => (VPBROADCASTB128 ...)
+(Broadcast1To16Uint16x8 ...) => (VPBROADCASTW256 ...)
+(Broadcast1To16Uint32x4 ...) => (VPBROADCASTD512 ...)
+(Broadcast1To32Int8x16 ...) => (VPBROADCASTB256 ...)
+(Broadcast1To32Int16x8 ...) => (VPBROADCASTW512 ...)
+(Broadcast1To32Uint8x16 ...) => (VPBROADCASTB256 ...)
+(Broadcast1To32Uint16x8 ...) => (VPBROADCASTW512 ...)
+(Broadcast1To64Int8x16 ...) => (VPBROADCASTB512 ...)
+(Broadcast1To64Uint8x16 ...) => (VPBROADCASTB512 ...)
 (CeilFloat32x4 x) => (VROUNDPS128 [2] x)
 (CeilFloat32x8 x) => (VROUNDPS256 [2] x)
 (CeilFloat64x2 x) => (VROUNDPD128 [2] x)
@@ -316,12 +316,6 @@
 (DotProductPairsSaturatedUint8x16 ...) => (VPMADDUBSW128 ...)
 (DotProductPairsSaturatedUint8x32 ...) => (VPMADDUBSW256 ...)
 (DotProductPairsSaturatedUint8x64 ...) => (VPMADDUBSW512 ...)
-(DotProductQuadrupleInt32x4 ...) => (VPDPBUSD128 ...)
-(DotProductQuadrupleInt32x8 ...) => (VPDPBUSD256 ...)
-(DotProductQuadrupleInt32x16 ...) => (VPDPBUSD512 ...)
-(DotProductQuadrupleSaturatedInt32x4 ...) => (VPDPBUSDS128 ...)
-(DotProductQuadrupleSaturatedInt32x8 ...) => (VPDPBUSDS256 ...)
-(DotProductQuadrupleSaturatedInt32x16 ...) => (VPDPBUSDS512 ...)
 (EqualFloat32x4 x y) => (VCMPPS128 [0] x y)
 (EqualFloat32x8 x y) => (VCMPPS256 [0] x y)
 (EqualFloat32x16 x y) => (VPMOVMToVec32x16 (VCMPPS512 [0] x y))
@@ -382,26 +376,26 @@
 (ExpandUint64x2 x mask) => (VPEXPANDQMasked128 x (VPMOVVec64x2ToM <types.TypeMask> mask))
 (ExpandUint64x4 x mask) => (VPEXPANDQMasked256 x (VPMOVVec64x4ToM <types.TypeMask> mask))
 (ExpandUint64x8 x mask) => (VPEXPANDQMasked512 x (VPMOVVec64x8ToM <types.TypeMask> mask))
-(ExtendLo2ToInt64x2Int8x16 ...) => (VPMOVSXBQ128 ...)
-(ExtendLo2ToInt64x2Int16x8 ...) => (VPMOVSXWQ128 ...)
-(ExtendLo2ToInt64x2Int32x4 ...) => (VPMOVSXDQ128 ...)
-(ExtendLo2ToUint64x2Uint8x16 ...) => (VPMOVZXBQ128 ...)
-(ExtendLo2ToUint64x2Uint16x8 ...) => (VPMOVZXWQ128 ...)
-(ExtendLo2ToUint64x2Uint32x4 ...) => (VPMOVZXDQ128 ...)
-(ExtendLo4ToInt32x4Int8x16 ...) => (VPMOVSXBD128 ...)
-(ExtendLo4ToInt32x4Int16x8 ...) => (VPMOVSXWD128 ...)
-(ExtendLo4ToInt64x4Int8x16 ...) => (VPMOVSXBQ256 ...)
-(ExtendLo4ToInt64x4Int16x8 ...) => (VPMOVSXWQ256 ...)
-(ExtendLo4ToUint32x4Uint8x16 ...) => (VPMOVZXBD128 ...)
-(ExtendLo4ToUint32x4Uint16x8 ...) => (VPMOVZXWD128 ...)
-(ExtendLo4ToUint64x4Uint8x16 ...) => (VPMOVZXBQ256 ...)
-(ExtendLo4ToUint64x4Uint16x8 ...) => (VPMOVZXWQ256 ...)
-(ExtendLo8ToInt16x8Int8x16 ...) => (VPMOVSXBW128 ...)
-(ExtendLo8ToInt32x8Int8x16 ...) => (VPMOVSXBD256 ...)
-(ExtendLo8ToInt64x8Int8x16 ...) => (VPMOVSXBQ512 ...)
-(ExtendLo8ToUint16x8Uint8x16 ...) => (VPMOVZXBW128 ...)
-(ExtendLo8ToUint32x8Uint8x16 ...) => (VPMOVZXBD256 ...)
-(ExtendLo8ToUint64x8Uint8x16 ...) => (VPMOVZXBQ512 ...)
+(ExtendLo2ToInt64Int8x16 ...) => (VPMOVSXBQ128 ...)
+(ExtendLo2ToInt64Int16x8 ...) => (VPMOVSXWQ128 ...)
+(ExtendLo2ToInt64Int32x4 ...) => (VPMOVSXDQ128 ...)
+(ExtendLo2ToUint64Uint8x16 ...) => (VPMOVZXBQ128 ...)
+(ExtendLo2ToUint64Uint16x8 ...) => (VPMOVZXWQ128 ...)
+(ExtendLo2ToUint64Uint32x4 ...) => (VPMOVZXDQ128 ...)
+(ExtendLo4ToInt32Int8x16 ...) => (VPMOVSXBD128 ...)
+(ExtendLo4ToInt32Int16x8 ...) => (VPMOVSXWD128 ...)
+(ExtendLo4ToInt64Int8x16 ...) => (VPMOVSXBQ256 ...)
+(ExtendLo4ToInt64Int16x8 ...) => (VPMOVSXWQ256 ...)
+(ExtendLo4ToUint32Uint8x16 ...) => (VPMOVZXBD128 ...)
+(ExtendLo4ToUint32Uint16x8 ...) => (VPMOVZXWD128 ...)
+(ExtendLo4ToUint64Uint8x16 ...) => (VPMOVZXBQ256 ...)
+(ExtendLo4ToUint64Uint16x8 ...) => (VPMOVZXWQ256 ...)
+(ExtendLo8ToInt16Int8x16 ...) => (VPMOVSXBW128 ...)
+(ExtendLo8ToInt32Int8x16 ...) => (VPMOVSXBD256 ...)
+(ExtendLo8ToInt64Int8x16 ...) => (VPMOVSXBQ512 ...)
+(ExtendLo8ToUint16Uint8x16 ...) => (VPMOVZXBW128 ...)
+(ExtendLo8ToUint32Uint8x16 ...) => (VPMOVZXBD256 ...)
+(ExtendLo8ToUint64Uint8x16 ...) => (VPMOVZXBQ512 ...)
 (ExtendToInt16Int8x16 ...) => (VPMOVSXBW256 ...)
 (ExtendToInt16Int8x32 ...) => (VPMOVSXBW512 ...)
 (ExtendToInt32Int8x16 ...) => (VPMOVSXBD512 ...)
@@ -565,12 +559,6 @@
 (InterleaveLoGroupedUint32x16 ...) => (VPUNPCKLDQ512 ...)
 (InterleaveLoGroupedUint64x4 ...) => (VPUNPCKLQDQ256 ...)
 (InterleaveLoGroupedUint64x8 ...) => (VPUNPCKLQDQ512 ...)
-(IsNanFloat32x4 x y) => (VCMPPS128 [3] x y)
-(IsNanFloat32x8 x y) => (VCMPPS256 [3] x y)
-(IsNanFloat32x16 x y) => (VPMOVMToVec32x16 (VCMPPS512 [3] x y))
-(IsNanFloat64x2 x y) => (VCMPPD128 [3] x y)
-(IsNanFloat64x4 x y) => (VCMPPD256 [3] x y)
-(IsNanFloat64x8 x y) => (VPMOVMToVec64x8 (VCMPPD512 [3] x y))
 (LeadingZerosInt32x4 ...) => (VPLZCNTD128 ...)
 (LeadingZerosInt32x8 ...) => (VPLZCNTD256 ...)
 (LeadingZerosInt32x16 ...) => (VPLZCNTD512 ...)
@@ -914,29 +902,29 @@
 (SaturateToInt16Int64x4 ...) => (VPMOVSQW128_256 ...)
 (SaturateToInt16Int64x8 ...) => (VPMOVSQW128_512 ...)
 (SaturateToInt16ConcatInt32x4 ...) => (VPACKSSDW128 ...)
-(SaturateToInt16ConcatInt32x8 ...) => (VPACKSSDW256 ...)
-(SaturateToInt16ConcatInt32x16 ...) => (VPACKSSDW512 ...)
+(SaturateToInt16ConcatGroupedInt32x8 ...) => (VPACKSSDW256 ...)
+(SaturateToInt16ConcatGroupedInt32x16 ...) => (VPACKSSDW512 ...)
 (SaturateToInt32Int64x2 ...) => (VPMOVSQD128_128 ...)
 (SaturateToInt32Int64x4 ...) => (VPMOVSQD128_256 ...)
 (SaturateToInt32Int64x8 ...) => (VPMOVSQD256 ...)
-(SaturateToUint8Int16x8 ...) => (VPMOVSWB128_128 ...)
-(SaturateToUint8Int16x16 ...) => (VPMOVSWB128_256 ...)
-(SaturateToUint8Int32x4 ...) => (VPMOVSDB128_128 ...)
-(SaturateToUint8Int32x8 ...) => (VPMOVSDB128_256 ...)
-(SaturateToUint8Int32x16 ...) => (VPMOVSDB128_512 ...)
-(SaturateToUint8Int64x2 ...) => (VPMOVSQB128_128 ...)
-(SaturateToUint8Int64x4 ...) => (VPMOVSQB128_256 ...)
-(SaturateToUint8Int64x8 ...) => (VPMOVSQB128_512 ...)
+(SaturateToUint8Uint16x8 ...) => (VPMOVUSWB128_128 ...)
+(SaturateToUint8Uint16x16 ...) => (VPMOVUSWB128_256 ...)
 (SaturateToUint8Uint16x32 ...) => (VPMOVUSWB256 ...)
+(SaturateToUint8Uint32x4 ...) => (VPMOVUSDB128_128 ...)
+(SaturateToUint8Uint32x8 ...) => (VPMOVUSDB128_256 ...)
+(SaturateToUint8Uint32x16 ...) => (VPMOVUSDB128_512 ...)
+(SaturateToUint8Uint64x2 ...) => (VPMOVUSQB128_128 ...)
+(SaturateToUint8Uint64x4 ...) => (VPMOVUSQB128_256 ...)
+(SaturateToUint8Uint64x8 ...) => (VPMOVUSQB128_512 ...)
 (SaturateToUint16Uint32x4 ...) => (VPMOVUSDW128_128 ...)
 (SaturateToUint16Uint32x8 ...) => (VPMOVUSDW128_256 ...)
 (SaturateToUint16Uint32x16 ...) => (VPMOVUSDW256 ...)
 (SaturateToUint16Uint64x2 ...) => (VPMOVUSQW128_128 ...)
 (SaturateToUint16Uint64x4 ...) => (VPMOVUSQW128_256 ...)
 (SaturateToUint16Uint64x8 ...) => (VPMOVUSQW128_512 ...)
-(SaturateToUint16ConcatUint32x4 ...) => (VPACKUSDW128 ...)
-(SaturateToUint16ConcatUint32x8 ...) => (VPACKUSDW256 ...)
-(SaturateToUint16ConcatUint32x16 ...) => (VPACKUSDW512 ...)
+(SaturateToUint16ConcatInt32x4 ...) => (VPACKUSDW128 ...)
+(SaturateToUint16ConcatGroupedInt32x8 ...) => (VPACKUSDW256 ...)
+(SaturateToUint16ConcatGroupedInt32x16 ...) => (VPACKUSDW512 ...)
 (SaturateToUint32Uint64x2 ...) => (VPMOVUSQD128_128 ...)
 (SaturateToUint32Uint64x4 ...) => (VPMOVUSQD128_256 ...)
 (SaturateToUint32Uint64x8 ...) => (VPMOVUSQD256 ...)
@@ -1223,19 +1211,19 @@
 (SubUint64x4 ...) => (VPSUBQ256 ...)
 (SubUint64x8 ...) => (VPSUBQ512 ...)
 (SubPairsFloat32x4 ...) => (VHSUBPS128 ...)
-(SubPairsFloat32x8 ...) => (VHSUBPS256 ...)
 (SubPairsFloat64x2 ...) => (VHSUBPD128 ...)
-(SubPairsFloat64x4 ...) => (VHSUBPD256 ...)
 (SubPairsInt16x8 ...) => (VPHSUBW128 ...)
-(SubPairsInt16x16 ...) => (VPHSUBW256 ...)
 (SubPairsInt32x4 ...) => (VPHSUBD128 ...)
-(SubPairsInt32x8 ...) => (VPHSUBD256 ...)
 (SubPairsUint16x8 ...) => (VPHSUBW128 ...)
-(SubPairsUint16x16 ...) => (VPHSUBW256 ...)
 (SubPairsUint32x4 ...) => (VPHSUBD128 ...)
-(SubPairsUint32x8 ...) => (VPHSUBD256 ...)
+(SubPairsGroupedFloat32x8 ...) => (VHSUBPS256 ...)
+(SubPairsGroupedFloat64x4 ...) => (VHSUBPD256 ...)
+(SubPairsGroupedInt16x16 ...) => (VPHSUBW256 ...)
+(SubPairsGroupedInt32x8 ...) => (VPHSUBD256 ...)
+(SubPairsGroupedUint16x16 ...) => (VPHSUBW256 ...)
+(SubPairsGroupedUint32x8 ...) => (VPHSUBD256 ...)
 (SubPairsSaturatedInt16x8 ...) => (VPHSUBSW128 ...)
-(SubPairsSaturatedInt16x16 ...) => (VPHSUBSW256 ...)
+(SubPairsSaturatedGroupedInt16x16 ...) => (VPHSUBSW256 ...)
 (SubSaturatedInt8x16 ...) => (VPSUBSB128 ...)
 (SubSaturatedInt8x32 ...) => (VPSUBSB256 ...)
 (SubSaturatedInt8x64 ...) => (VPSUBSB512 ...)
@@ -1436,23 +1424,23 @@
 (VMOVDQU16Masked128 (VPAVGW128 x y) mask) => (VPAVGWMasked128 x y mask)
 (VMOVDQU16Masked256 (VPAVGW256 x y) mask) => (VPAVGWMasked256 x y mask)
 (VMOVDQU16Masked512 (VPAVGW512 x y) mask) => (VPAVGWMasked512 x y mask)
-(VMOVDQU32Masked128 (VBROADCASTSS128 x) mask) => (VBROADCASTSSMasked128 x mask)
 (VMOVDQU64Masked128 (VPBROADCASTQ128 x) mask) => (VPBROADCASTQMasked128 x mask)
-(VMOVDQU8Masked128 (VPBROADCASTB128 x) mask) => (VPBROADCASTBMasked128 x mask)
-(VMOVDQU16Masked128 (VPBROADCASTW128 x) mask) => (VPBROADCASTWMasked128 x mask)
-(VMOVDQU32Masked128 (VPBROADCASTD128 x) mask) => (VPBROADCASTDMasked128 x mask)
-(VMOVDQU32Masked256 (VBROADCASTSS256 x) mask) => (VBROADCASTSSMasked256 x mask)
+(VMOVDQU32Masked128 (VBROADCASTSS128 x) mask) => (VBROADCASTSSMasked128 x mask)
 (VMOVDQU64Masked256 (VBROADCASTSD256 x) mask) => (VBROADCASTSDMasked256 x mask)
-(VMOVDQU8Masked256 (VPBROADCASTB256 x) mask) => (VPBROADCASTBMasked256 x mask)
-(VMOVDQU16Masked256 (VPBROADCASTW256 x) mask) => (VPBROADCASTWMasked256 x mask)
-(VMOVDQU32Masked256 (VPBROADCASTD256 x) mask) => (VPBROADCASTDMasked256 x mask)
+(VMOVDQU32Masked128 (VPBROADCASTD128 x) mask) => (VPBROADCASTDMasked128 x mask)
 (VMOVDQU64Masked256 (VPBROADCASTQ256 x) mask) => (VPBROADCASTQMasked256 x mask)
-(VMOVDQU32Masked512 (VBROADCASTSS512 x) mask) => (VBROADCASTSSMasked512 x mask)
+(VMOVDQU32Masked256 (VBROADCASTSS256 x) mask) => (VBROADCASTSSMasked256 x mask)
 (VMOVDQU64Masked512 (VBROADCASTSD512 x) mask) => (VBROADCASTSDMasked512 x mask)
-(VMOVDQU8Masked512 (VPBROADCASTB512 x) mask) => (VPBROADCASTBMasked512 x mask)
-(VMOVDQU16Masked512 (VPBROADCASTW512 x) mask) => (VPBROADCASTWMasked512 x mask)
-(VMOVDQU32Masked512 (VPBROADCASTD512 x) mask) => (VPBROADCASTDMasked512 x mask)
+(VMOVDQU16Masked128 (VPBROADCASTW128 x) mask) => (VPBROADCASTWMasked128 x mask)
+(VMOVDQU32Masked256 (VPBROADCASTD256 x) mask) => (VPBROADCASTDMasked256 x mask)
 (VMOVDQU64Masked512 (VPBROADCASTQ512 x) mask) => (VPBROADCASTQMasked512 x mask)
+(VMOVDQU32Masked512 (VBROADCASTSS512 x) mask) => (VBROADCASTSSMasked512 x mask)
+(VMOVDQU8Masked128 (VPBROADCASTB128 x) mask) => (VPBROADCASTBMasked128 x mask)
+(VMOVDQU16Masked256 (VPBROADCASTW256 x) mask) => (VPBROADCASTWMasked256 x mask)
+(VMOVDQU32Masked512 (VPBROADCASTD512 x) mask) => (VPBROADCASTDMasked512 x mask)
+(VMOVDQU8Masked256 (VPBROADCASTB256 x) mask) => (VPBROADCASTBMasked256 x mask)
+(VMOVDQU16Masked512 (VPBROADCASTW512 x) mask) => (VPBROADCASTWMasked512 x mask)
+(VMOVDQU8Masked512 (VPBROADCASTB512 x) mask) => (VPBROADCASTBMasked512 x mask)
 (VMOVDQU32Masked128 (VRNDSCALEPS128 [a] x) mask) => (VRNDSCALEPSMasked128 [a] x mask)
 (VMOVDQU32Masked256 (VRNDSCALEPS256 [a] x) mask) => (VRNDSCALEPSMasked256 [a] x mask)
 (VMOVDQU32Masked512 (VRNDSCALEPS512 [a] x) mask) => (VRNDSCALEPSMasked512 [a] x mask)
@@ -1547,12 +1535,6 @@
 (VMOVDQU16Masked128 (VPMADDUBSW128 x y) mask) => (VPMADDUBSWMasked128 x y mask)
 (VMOVDQU16Masked256 (VPMADDUBSW256 x y) mask) => (VPMADDUBSWMasked256 x y mask)
 (VMOVDQU16Masked512 (VPMADDUBSW512 x y) mask) => (VPMADDUBSWMasked512 x y mask)
-(VMOVDQU32Masked128 (VPDPBUSD128 x y z) mask) => (VPDPBUSDMasked128 x y z mask)
-(VMOVDQU32Masked256 (VPDPBUSD256 x y z) mask) => (VPDPBUSDMasked256 x y z mask)
-(VMOVDQU32Masked512 (VPDPBUSD512 x y z) mask) => (VPDPBUSDMasked512 x y z mask)
-(VMOVDQU32Masked128 (VPDPBUSDS128 x y z) mask) => (VPDPBUSDSMasked128 x y z mask)
-(VMOVDQU32Masked256 (VPDPBUSDS256 x y z) mask) => (VPDPBUSDSMasked256 x y z mask)
-(VMOVDQU32Masked512 (VPDPBUSDS512 x y z) mask) => (VPDPBUSDSMasked512 x y z mask)
 (VMOVDQU8Masked128 (VPMOVSXBQ128 x) mask) => (VPMOVSXBQMasked128 x mask)
 (VMOVDQU16Masked128 (VPMOVSXWQ128 x) mask) => (VPMOVSXWQMasked128 x mask)
 (VMOVDQU32Masked128 (VPMOVSXDQ128 x) mask) => (VPMOVSXDQMasked128 x mask)
@@ -1775,9 +1757,9 @@
 (VMOVDQU64Masked128 (VPMOVSQB128_128 x) mask) => (VPMOVSQBMasked128_128 x mask)
 (VMOVDQU64Masked256 (VPMOVSQB128_256 x) mask) => (VPMOVSQBMasked128_256 x mask)
 (VMOVDQU64Masked512 (VPMOVSQB128_512 x) mask) => (VPMOVSQBMasked128_512 x mask)
-(VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) => (VPACKSSDWMasked128 x y mask)
 (VMOVDQU32Masked256 (VPACKSSDW256 x y) mask) => (VPACKSSDWMasked256 x y mask)
 (VMOVDQU32Masked512 (VPACKSSDW512 x y) mask) => (VPACKSSDWMasked512 x y mask)
+(VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) => (VPACKSSDWMasked128 x y mask)
 (VMOVDQU32Masked128 (VPMOVSDW128_128 x) mask) => (VPMOVSDWMasked128_128 x mask)
 (VMOVDQU32Masked256 (VPMOVSDW128_256 x) mask) => (VPMOVSDWMasked128_256 x mask)
 (VMOVDQU32Masked256 (VPMOVSDW256 x) mask) => (VPMOVSDWMasked256 x mask)
@@ -1787,10 +1769,18 @@
 (VMOVDQU64Masked128 (VPMOVSQD128_128 x) mask) => (VPMOVSQDMasked128_128 x mask)
 (VMOVDQU64Masked256 (VPMOVSQD128_256 x) mask) => (VPMOVSQDMasked128_256 x mask)
 (VMOVDQU64Masked256 (VPMOVSQD256 x) mask) => (VPMOVSQDMasked256 x mask)
+(VMOVDQU16Masked128 (VPMOVUSWB128_128 x) mask) => (VPMOVUSWBMasked128_128 x mask)
+(VMOVDQU16Masked256 (VPMOVUSWB128_256 x) mask) => (VPMOVUSWBMasked128_256 x mask)
 (VMOVDQU16Masked256 (VPMOVUSWB256 x) mask) => (VPMOVUSWBMasked256 x mask)
-(VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) => (VPACKUSDWMasked128 x y mask)
+(VMOVDQU32Masked128 (VPMOVUSDB128_128 x) mask) => (VPMOVUSDBMasked128_128 x mask)
+(VMOVDQU32Masked256 (VPMOVUSDB128_256 x) mask) => (VPMOVUSDBMasked128_256 x mask)
+(VMOVDQU32Masked512 (VPMOVUSDB128_512 x) mask) => (VPMOVUSDBMasked128_512 x mask)
+(VMOVDQU64Masked128 (VPMOVUSQB128_128 x) mask) => (VPMOVUSQBMasked128_128 x mask)
+(VMOVDQU64Masked256 (VPMOVUSQB128_256 x) mask) => (VPMOVUSQBMasked128_256 x mask)
+(VMOVDQU64Masked512 (VPMOVUSQB128_512 x) mask) => (VPMOVUSQBMasked128_512 x mask)
 (VMOVDQU32Masked256 (VPACKUSDW256 x y) mask) => (VPACKUSDWMasked256 x y mask)
 (VMOVDQU32Masked512 (VPACKUSDW512 x y) mask) => (VPACKUSDWMasked512 x y mask)
+(VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) => (VPACKUSDWMasked128 x y mask)
 (VMOVDQU32Masked128 (VPMOVUSDW128_128 x) mask) => (VPMOVUSDWMasked128_128 x mask)
 (VMOVDQU32Masked256 (VPMOVUSDW128_256 x) mask) => (VPMOVUSDWMasked128_256 x mask)
 (VMOVDQU32Masked256 (VPMOVUSDW256 x) mask) => (VPMOVUSDWMasked256 x mask)
@@ -2018,6 +2008,7 @@
 (VPBLENDMDMasked512 dst (VPMOVDW256 x) mask) => (VPMOVDWMasked256Merging dst x mask)
 (VPBLENDMDMasked512 dst (VPMOVSDB128_512 x) mask) => (VPMOVSDBMasked128_512Merging dst x mask)
 (VPBLENDMDMasked512 dst (VPMOVSDW256 x) mask) => (VPMOVSDWMasked256Merging dst x mask)
+(VPBLENDMDMasked512 dst (VPMOVUSDB128_512 x) mask) => (VPMOVUSDBMasked128_512Merging dst x mask)
 (VPBLENDMDMasked512 dst (VPMOVUSDW256 x) mask) => (VPMOVUSDWMasked256Merging dst x mask)
 (VPBLENDMDMasked512 dst (VPMULLD512 x y) mask) => (VPMULLDMasked512Merging dst x y mask)
 (VPBLENDMDMasked512 dst (VPOPCNTD512 x) mask) => (VPOPCNTDMasked512Merging dst x mask)
@@ -2071,6 +2062,7 @@
 (VPBLENDMQMasked512 dst (VPMOVSQB128_512 x) mask) => (VPMOVSQBMasked128_512Merging dst x mask)
 (VPBLENDMQMasked512 dst (VPMOVSQD256 x) mask) => (VPMOVSQDMasked256Merging dst x mask)
 (VPBLENDMQMasked512 dst (VPMOVSQW128_512 x) mask) => (VPMOVSQWMasked128_512Merging dst x mask)
+(VPBLENDMQMasked512 dst (VPMOVUSQB128_512 x) mask) => (VPMOVUSQBMasked128_512Merging dst x mask)
 (VPBLENDMQMasked512 dst (VPMOVUSQD256 x) mask) => (VPMOVUSQDMasked256Merging dst x mask)
 (VPBLENDMQMasked512 dst (VPMOVUSQW128_512 x) mask) => (VPMOVUSQWMasked128_512Merging dst x mask)
 (VPBLENDMQMasked512 dst (VPMULLQ512 x y) mask) => (VPMULLQMasked512Merging dst x y mask)
@@ -2235,9 +2227,12 @@
 (VPBLENDVB128 dst (VPMOVSXWQ128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXWQMasked128Merging dst x (VPMOVVec16x8ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVSXWQ256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXWQMasked256Merging dst x (VPMOVVec16x8ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVSXWQ512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXWQMasked512Merging dst x (VPMOVVec16x8ToM <types.TypeMask> mask))
+(VPBLENDVB128 dst (VPMOVUSDB128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSDBMasked128_128Merging dst x (VPMOVVec32x4ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVUSDW128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSDWMasked128_128Merging dst x (VPMOVVec32x4ToM <types.TypeMask> mask))
+(VPBLENDVB128 dst (VPMOVUSQB128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQBMasked128_128Merging dst x (VPMOVVec64x2ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVUSQD128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQDMasked128_128Merging dst x (VPMOVVec64x2ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVUSQW128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQWMasked128_128Merging dst x (VPMOVVec64x2ToM <types.TypeMask> mask))
+(VPBLENDVB128 dst (VPMOVUSWB128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSWBMasked128_128Merging dst x (VPMOVVec16x8ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVWB128_128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVWBMasked128_128Merging dst x (VPMOVVec16x8ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVZXBD128 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVZXBDMasked128Merging dst x (VPMOVVec8x16ToM <types.TypeMask> mask))
 (VPBLENDVB128 dst (VPMOVZXBD256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVZXBDMasked256Merging dst x (VPMOVVec8x16ToM <types.TypeMask> mask))
@@ -2396,9 +2391,12 @@
 (VPBLENDVB256 dst (VPMOVSXBW512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXBWMasked512Merging dst x (VPMOVVec8x32ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVSXDQ512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXDQMasked512Merging dst x (VPMOVVec32x8ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVSXWD512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVSXWDMasked512Merging dst x (VPMOVVec16x16ToM <types.TypeMask> mask))
+(VPBLENDVB256 dst (VPMOVUSDB128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSDBMasked128_256Merging dst x (VPMOVVec32x8ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVUSDW128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSDWMasked128_256Merging dst x (VPMOVVec32x8ToM <types.TypeMask> mask))
+(VPBLENDVB256 dst (VPMOVUSQB128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQBMasked128_256Merging dst x (VPMOVVec64x4ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVUSQD128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQDMasked128_256Merging dst x (VPMOVVec64x4ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVUSQW128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSQWMasked128_256Merging dst x (VPMOVVec64x4ToM <types.TypeMask> mask))
+(VPBLENDVB256 dst (VPMOVUSWB128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVUSWBMasked128_256Merging dst x (VPMOVVec16x16ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVWB128_256 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVWBMasked128_256Merging dst x (VPMOVVec16x16ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVZXBW512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVZXBWMasked512Merging dst x (VPMOVVec8x32ToM <types.TypeMask> mask))
 (VPBLENDVB256 dst (VPMOVZXDQ512 x) mask) && v.Block.CPUfeatures.hasFeature(CPUavx512) => (VPMOVZXDQMasked512Merging dst x (VPMOVVec32x8ToM <types.TypeMask> mask))
@@ -2511,30 +2509,30 @@
 (VPANDNQMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPANDNQMasked128load {sym} [off] x ptr mask mem)
 (VPANDNQMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPANDNQMasked256load {sym} [off] x ptr mask mem)
 (VPANDNQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPANDNQMasked512load {sym} [off] x ptr mask mem)
-(VRNDSCALEPS128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPS256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPS512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VRNDSCALEPSMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VRNDSCALEPSMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VRNDSCALEPSMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VRNDSCALEPDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VRNDSCALEPDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VRNDSCALEPDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPS128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPS256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPS512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VREDUCEPSMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPSMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPSMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VREDUCEPDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
+(VRNDSCALEPS128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPS256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPS512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPS512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VRNDSCALEPSMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VRNDSCALEPSMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VRNDSCALEPSMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPSMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VRNDSCALEPDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VRNDSCALEPDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VRNDSCALEPDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRNDSCALEPDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPS128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPS256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPS512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPS512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VREDUCEPSMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPSMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPSMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPSMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VREDUCEPDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VREDUCEPDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
 (VPERMI2PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPERMI2PS128load {sym} [off] x y ptr mem)
 (VPERMI2D128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPERMI2D128load {sym} [off] x y ptr mem)
 (VPERMI2PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPERMI2PS256load {sym} [off] x y ptr mem)
@@ -2655,54 +2653,46 @@
 (VDIVPDMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VDIVPDMasked128load {sym} [off] x ptr mask mem)
 (VDIVPDMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VDIVPDMasked256load {sym} [off] x ptr mask mem)
 (VDIVPDMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VDIVPDMasked512load {sym} [off] x ptr mask mem)
-(VPDPBUSD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSD512load {sym} [off] x y ptr mem)
-(VPDPBUSDMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDMasked128load {sym} [off] x y ptr mask mem)
-(VPDPBUSDMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDMasked256load {sym} [off] x y ptr mask mem)
-(VPDPBUSDMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDMasked512load {sym} [off] x y ptr mask mem)
-(VPDPBUSDS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDS512load {sym} [off] x y ptr mem)
-(VPDPBUSDSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDSMasked128load {sym} [off] x y ptr mask mem)
-(VPDPBUSDSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDSMasked256load {sym} [off] x y ptr mask mem)
-(VPDPBUSDSMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPDPBUSDSMasked512load {sym} [off] x y ptr mask mem)
 (VPCMPEQD512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPEQD512load {sym} [off] x ptr mem)
 (VPCMPEQQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPEQQ512load {sym} [off] x ptr mem)
-(VCMPPS512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VCMPPS512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
-(VCMPPD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VCMPPD512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
-(VCMPPSMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VCMPPSMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VCMPPSMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VCMPPDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VCMPPDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VCMPPDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPQMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPQMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPQMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUQMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUQMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VPCMPUQMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mask mem)
-(VGF2P8AFFINEQB128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEQB256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEQB512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEINVQB128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEINVQB256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEINVQB512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VGF2P8AFFINEINVQBMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VGF2P8AFFINEINVQBMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VGF2P8AFFINEINVQBMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VGF2P8AFFINEQBMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VGF2P8AFFINEQBMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VGF2P8AFFINEQBMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
+(VCMPPS512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VCMPPS512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
+(VCMPPD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VCMPPD512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
+(VCMPPSMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VCMPPSMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VCMPPSMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPSMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VCMPPDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VCMPPDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VCMPPDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VCMPPDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPQMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPQMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPQMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUDMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUDMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUDMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUQMasked128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUQMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VPCMPUQMasked512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mask mem)
+(VGF2P8AFFINEQB128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEQB256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEQB512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQB512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEINVQB128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEINVQB256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEINVQB512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQB512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VGF2P8AFFINEINVQBMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VGF2P8AFFINEINVQBMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VGF2P8AFFINEINVQBMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEINVQBMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VGF2P8AFFINEQBMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VGF2P8AFFINEQBMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VGF2P8AFFINEQBMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VGF2P8AFFINEQBMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
 (VPCMPGTD512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPGTD512load {sym} [off] x ptr mem)
 (VPCMPGTQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPGTQ512load {sym} [off] x ptr mem)
-(VPCMPUD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPUD512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
-(VPCMPUQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQ512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
-(VPCMPD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPD512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
-(VPCMPQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPQ512load {sym} [makeValAndOff(int32(int8(c)),off)] x ptr mem)
+(VPCMPUD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPUD512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
+(VPCMPUQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPUQ512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
+(VPCMPD512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPD512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
+(VPCMPQ512 [c] x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPCMPQ512load {sym} [makeValAndOff(int32(uint8(c)),off)] x ptr mem)
 (VPUNPCKHDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPUNPCKHDQ512load {sym} [off] x ptr mem)
 (VPUNPCKHQDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPUNPCKHQDQ512load {sym} [off] x ptr mem)
 (VPUNPCKLDQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPUNPCKLDQ512load {sym} [off] x ptr mem)
@@ -2781,11 +2771,7 @@
 (VPMULLQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ128load {sym} [off] x ptr mem)
 (VPMULLQ256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ256load {sym} [off] x ptr mem)
 (VPMULLQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ512load {sym} [off] x ptr mem)
-(VFMADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS128load {sym} [off] x y ptr mem)
-(VFMADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS256load {sym} [off] x y ptr mem)
 (VFMADD213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS512load {sym} [off] x y ptr mem)
-(VFMADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD128load {sym} [off] x y ptr mem)
-(VFMADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD256load {sym} [off] x y ptr mem)
 (VFMADD213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD512load {sym} [off] x y ptr mem)
 (VFMADD213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PSMasked128load {sym} [off] x y ptr mask mem)
 (VFMADD213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PSMasked256load {sym} [off] x y ptr mask mem)
@@ -2793,11 +2779,7 @@
 (VFMADD213PDMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked128load {sym} [off] x y ptr mask mem)
 (VFMADD213PDMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked256load {sym} [off] x y ptr mask mem)
 (VFMADD213PDMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked512load {sym} [off] x y ptr mask mem)
-(VFMADDSUB213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS128load {sym} [off] x y ptr mem)
-(VFMADDSUB213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS256load {sym} [off] x y ptr mem)
 (VFMADDSUB213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS512load {sym} [off] x y ptr mem)
-(VFMADDSUB213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD128load {sym} [off] x y ptr mem)
-(VFMADDSUB213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD256load {sym} [off] x y ptr mem)
 (VFMADDSUB213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD512load {sym} [off] x y ptr mem)
 (VFMADDSUB213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PSMasked128load {sym} [off] x y ptr mask mem)
 (VFMADDSUB213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PSMasked256load {sym} [off] x y ptr mask mem)
@@ -2817,11 +2799,7 @@
 (VPMULLQMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked128load {sym} [off] x ptr mask mem)
 (VPMULLQMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked256load {sym} [off] x ptr mask mem)
 (VPMULLQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked512load {sym} [off] x ptr mask mem)
-(VFMSUBADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS128load {sym} [off] x y ptr mem)
-(VFMSUBADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS256load {sym} [off] x y ptr mem)
 (VFMSUBADD213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS512load {sym} [off] x y ptr mem)
-(VFMSUBADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD128load {sym} [off] x y ptr mem)
-(VFMSUBADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD256load {sym} [off] x y ptr mem)
 (VFMSUBADD213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD512load {sym} [off] x y ptr mem)
 (VFMSUBADD213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PSMasked128load {sym} [off] x y ptr mask mem)
 (VFMSUBADD213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PSMasked256load {sym} [off] x y ptr mask mem)
@@ -2883,30 +2861,30 @@
 (VRSQRT14PDMasked128 l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRSQRT14PDMasked128load {sym} [off] ptr mask mem)
 (VRSQRT14PDMasked256 l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRSQRT14PDMasked256load {sym} [off] ptr mask mem)
 (VRSQRT14PDMasked512 l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VRSQRT14PDMasked512load {sym} [off] ptr mask mem)
-(VPROLD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLQ128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLQ256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLQ512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPROLDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPROLDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPROLDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPROLQMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPROLQMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPROLQMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORQ128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORQ256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORQ512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPRORDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORQMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORQMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPRORQMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
+(VPROLD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLQ128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLQ256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLQ512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLQ512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPROLDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPROLDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPROLDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPROLQMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPROLQMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPROLQMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPROLQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORD128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORD256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORQ128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORQ256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORQ512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPRORQ512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPRORDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORQMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORQMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPRORQMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
 (VPROLVD128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLVD128load {sym} [off] x ptr mem)
 (VPROLVD256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLVD256load {sym} [off] x ptr mem)
 (VPROLVD512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPROLVD512load {sym} [off] x ptr mem)
@@ -2932,13 +2910,13 @@
 (VPRORVQMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORVQMasked256load {sym} [off] x ptr mask mem)
 (VPRORVQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPRORVQMasked512load {sym} [off] x ptr mask mem)
 (VPACKSSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPACKSSDW512load {sym} [off] x ptr mem)
-(VPACKSSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKSSDWMasked128load {sym} [off] x ptr mask mem)
 (VPACKSSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKSSDWMasked256load {sym} [off] x ptr mask mem)
 (VPACKSSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKSSDWMasked512load {sym} [off] x ptr mask mem)
+(VPACKSSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKSSDWMasked128load {sym} [off] x ptr mask mem)
 (VPACKUSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPACKUSDW512load {sym} [off] x ptr mem)
-(VPACKUSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKUSDWMasked128load {sym} [off] x ptr mask mem)
 (VPACKUSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKUSDWMasked256load {sym} [off] x ptr mask mem)
 (VPACKUSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKUSDWMasked512load {sym} [off] x ptr mask mem)
+(VPACKUSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPACKUSDWMasked128load {sym} [off] x ptr mask mem)
 (VSCALEFPS128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPS128load {sym} [off] x ptr mem)
 (VSCALEFPS256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPS256load {sym} [off] x ptr mem)
 (VSCALEFPS512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPS512load {sym} [off] x ptr mem)
@@ -2951,30 +2929,30 @@
 (VSCALEFPDMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPDMasked128load {sym} [off] x ptr mask mem)
 (VSCALEFPDMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPDMasked256load {sym} [off] x ptr mask mem)
 (VSCALEFPDMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VSCALEFPDMasked512load {sym} [off] x ptr mask mem)
-(VPSHLDD128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDD256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDQ128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDQ256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDQ512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHLDDMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHLDDMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHLDDMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHLDQMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHLDQMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHLDQMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDD128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDD256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDQ128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDQ256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDQ512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHRDDMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDDMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDDMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDQMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDQMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
-(VPSHRDQMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mask mem)
+(VPSHLDD128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDD256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDQ128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDQ256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDQ512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQ512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHLDDMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHLDDMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHLDDMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHLDQMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHLDQMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHLDQMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHLDQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDD128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDD256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDQ128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDQ256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDQ512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQ512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHRDDMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDDMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDDMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDQMasked128 [c]  x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDQMasked256 [c]  x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
+(VPSHRDQMasked512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHRDQMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mask mem)
 (VPSLLVD512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLVD512load {sym} [off] x ptr mem)
 (VPSLLVQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLVQ512load {sym} [off] x ptr mem)
 (VPSHLDVD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHLDVD128load {sym} [off] x y ptr mem)
@@ -3059,41 +3037,41 @@
 (VPXORQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPXORQMasked512load {sym} [off] x ptr mask mem)
 (VPBLENDMDMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPBLENDMDMasked512load {sym} [off] x ptr mask mem)
 (VPBLENDMQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPBLENDMQMasked512load {sym} [off] x ptr mask mem)
-(VSHUFPS512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSHUFPS512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VSHUFPD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSHUFPD512load {sym} [makeValAndOff(int32(int8(c)),off)]  x ptr mem)
-(VPSHUFD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHUFD512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSHUFDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked256load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSHUFDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked512load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSHUFDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked128load {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLD512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSLLQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLQ512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSLLDMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLDMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLDMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSLLQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRLD512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRLQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRLQ512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRAD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAD512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRAQ128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRAQ256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRAQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mem)
-(VPSRLDMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLDMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLDMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRLQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRADMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRADMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRADMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRAQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked128constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRAQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked256constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPSRAQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked512constload {sym} [makeValAndOff(int32(int8(c)),off)]  ptr mask mem)
-(VPTERNLOGD128 [c]  x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD128load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
-(VPTERNLOGD256 [c]  x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD256load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
-(VPTERNLOGD512 [c]  x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD512load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
-(VPTERNLOGQ128 [c]  x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ128load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
-(VPTERNLOGQ256 [c]  x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ256load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
-(VPTERNLOGQ512 [c]  x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ512load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
+(VSHUFPS512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSHUFPS512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VSHUFPD512 [c]  x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VSHUFPD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x ptr mem)
+(VPSHUFD512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSHUFD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSHUFDMasked256 [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked256load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSHUFDMasked512 [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked512load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSHUFDMasked128 [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSHUFDMasked128load {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLD512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSLLQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSLLQ512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSLLDMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLDMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLDMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLDMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSLLQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSLLQMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRLD512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRLQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRLQ512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRAD512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAD512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRAQ128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRAQ256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRAQ512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPSRAQ512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mem)
+(VPSRLDMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLDMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLDMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLDMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRLQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRLQMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRADMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRADMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRADMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRADMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRAQMasked128const [c]  l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked128constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRAQMasked256const [c]  l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked256constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPSRAQMasked512const [c]  l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPSRAQMasked512constload {sym} [makeValAndOff(int32(uint8(c)),off)]  ptr mask mem)
+(VPTERNLOGD128 [c]  x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)
+(VPTERNLOGD256 [c]  x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)
+(VPTERNLOGD512 [c]  x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGD512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)
+(VPTERNLOGQ128 [c]  x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ128load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)
+(VPTERNLOGQ256 [c]  x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ256load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)
+(VPTERNLOGQ512 [c]  x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPTERNLOGQ512load {sym} [makeValAndOff(int32(uint8(c)),off)]  x y ptr mem)

src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go

@@ -1,4 +1,4 @@
-// Code generated by x/arch/internal/simdgen using 'go run . -xedPath $XED_PATH -o godefs -goroot $GOROOT go.yaml types.yaml categories.yaml'; DO NOT EDIT.
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
 
 package main
 
@@ -28,16 +28,16 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VADDSUBPS128", argLength: 2, reg: v21, asm: "VADDSUBPS", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VADDSUBPS256", argLength: 2, reg: v21, asm: "VADDSUBPS", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VAESDEC128", argLength: 2, reg: v21, asm: "VAESDEC", commutative: false, typ: "Vec128", resultInArg0: false},
-		{name: "VAESDEC256", argLength: 2, reg: w21, asm: "VAESDEC", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VAESDEC256", argLength: 2, reg: v21, asm: "VAESDEC", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VAESDEC512", argLength: 2, reg: w21, asm: "VAESDEC", commutative: false, typ: "Vec512", resultInArg0: false},
 		{name: "VAESDECLAST128", argLength: 2, reg: v21, asm: "VAESDECLAST", commutative: false, typ: "Vec128", resultInArg0: false},
-		{name: "VAESDECLAST256", argLength: 2, reg: w21, asm: "VAESDECLAST", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VAESDECLAST256", argLength: 2, reg: v21, asm: "VAESDECLAST", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VAESDECLAST512", argLength: 2, reg: w21, asm: "VAESDECLAST", commutative: false, typ: "Vec512", resultInArg0: false},
 		{name: "VAESENC128", argLength: 2, reg: v21, asm: "VAESENC", commutative: false, typ: "Vec128", resultInArg0: false},
-		{name: "VAESENC256", argLength: 2, reg: w21, asm: "VAESENC", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VAESENC256", argLength: 2, reg: v21, asm: "VAESENC", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VAESENC512", argLength: 2, reg: w21, asm: "VAESENC", commutative: false, typ: "Vec512", resultInArg0: false},
 		{name: "VAESENCLAST128", argLength: 2, reg: v21, asm: "VAESENCLAST", commutative: false, typ: "Vec128", resultInArg0: false},
-		{name: "VAESENCLAST256", argLength: 2, reg: w21, asm: "VAESENCLAST", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VAESENCLAST256", argLength: 2, reg: v21, asm: "VAESENCLAST", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VAESENCLAST512", argLength: 2, reg: w21, asm: "VAESENCLAST", commutative: false, typ: "Vec512", resultInArg0: false},
 		{name: "VAESIMC128", argLength: 1, reg: v11, asm: "VAESIMC", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VBROADCASTSD256", argLength: 1, reg: v11, asm: "VBROADCASTSD", commutative: false, typ: "Vec256", resultInArg0: false},
@@ -172,38 +172,38 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VEXPANDPSMasked128", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VEXPANDPSMasked256", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VEXPANDPSMasked512", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec512", resultInArg0: false},
-		{name: "VFMADD213PD128", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMADD213PD256", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMADD213PD128", argLength: 3, reg: v31, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMADD213PD256", argLength: 3, reg: v31, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADD213PD512", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMADD213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMADD213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADD213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VFMADD213PS128", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMADD213PS256", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMADD213PS128", argLength: 3, reg: v31, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMADD213PS256", argLength: 3, reg: v31, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADD213PS512", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMADD213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMADD213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADD213PSMasked512", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VFMADDSUB213PD128", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMADDSUB213PD256", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMADDSUB213PD128", argLength: 3, reg: v31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMADDSUB213PD256", argLength: 3, reg: v31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADDSUB213PD512", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VFMADDSUB213PS128", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMADDSUB213PS256", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMADDSUB213PS128", argLength: 3, reg: v31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMADDSUB213PS256", argLength: 3, reg: v31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADDSUB213PS512", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked512", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VFMSUBADD213PD128", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMSUBADD213PD256", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMSUBADD213PD128", argLength: 3, reg: v31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMSUBADD213PD256", argLength: 3, reg: v31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMSUBADD213PD512", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VFMSUBADD213PS128", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VFMSUBADD213PS256", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VFMSUBADD213PS128", argLength: 3, reg: v31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VFMSUBADD213PS256", argLength: 3, reg: v31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VFMSUBADD213PS512", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VFMSUBADD213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VFMSUBADD213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true},
@@ -452,18 +452,6 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VPCOMPRESSWMasked128", argLength: 2, reg: wkw, asm: "VPCOMPRESSW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPCOMPRESSWMasked256", argLength: 2, reg: wkw, asm: "VPCOMPRESSW", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VPCOMPRESSWMasked512", argLength: 2, reg: wkw, asm: "VPCOMPRESSW", commutative: false, typ: "Vec512", resultInArg0: false},
-		{name: "VPDPBUSD128", argLength: 3, reg: v31, asm: "VPDPBUSD", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VPDPBUSD256", argLength: 3, reg: v31, asm: "VPDPBUSD", commutative: false, typ: "Vec256", resultInArg0: true},
-		{name: "VPDPBUSD512", argLength: 3, reg: w31, asm: "VPDPBUSD", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VPDPBUSDMasked128", argLength: 4, reg: w3kw, asm: "VPDPBUSD", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VPDPBUSDMasked256", argLength: 4, reg: w3kw, asm: "VPDPBUSD", commutative: false, typ: "Vec256", resultInArg0: true},
-		{name: "VPDPBUSDMasked512", argLength: 4, reg: w3kw, asm: "VPDPBUSD", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VPDPBUSDS128", argLength: 3, reg: v31, asm: "VPDPBUSDS", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VPDPBUSDS256", argLength: 3, reg: v31, asm: "VPDPBUSDS", commutative: false, typ: "Vec256", resultInArg0: true},
-		{name: "VPDPBUSDS512", argLength: 3, reg: w31, asm: "VPDPBUSDS", commutative: false, typ: "Vec512", resultInArg0: true},
-		{name: "VPDPBUSDSMasked128", argLength: 4, reg: w3kw, asm: "VPDPBUSDS", commutative: false, typ: "Vec128", resultInArg0: true},
-		{name: "VPDPBUSDSMasked256", argLength: 4, reg: w3kw, asm: "VPDPBUSDS", commutative: false, typ: "Vec256", resultInArg0: true},
-		{name: "VPDPBUSDSMasked512", argLength: 4, reg: w3kw, asm: "VPDPBUSDS", commutative: false, typ: "Vec512", resultInArg0: true},
 		{name: "VPDPWSSD128", argLength: 3, reg: v31, asm: "VPDPWSSD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPDPWSSD256", argLength: 3, reg: v31, asm: "VPDPWSSD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VPDPWSSD512", argLength: 3, reg: w31, asm: "VPDPWSSD", commutative: false, typ: "Vec512", resultInArg0: true},
@@ -780,12 +768,24 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VPMOVSXWQMasked128", argLength: 2, reg: wkw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVSXWQMasked256", argLength: 2, reg: wkw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VPMOVSXWQMasked512", argLength: 2, reg: wkw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec512", resultInArg0: false},
+		{name: "VPMOVUSDB128_128", argLength: 1, reg: w11, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSDB128_256", argLength: 1, reg: w11, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSDB128_512", argLength: 1, reg: w11, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSDBMasked128_128", argLength: 2, reg: wkw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSDBMasked128_256", argLength: 2, reg: wkw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSDBMasked128_512", argLength: 2, reg: wkw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSDW128_128", argLength: 1, reg: w11, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSDW128_256", argLength: 1, reg: w11, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSDW256", argLength: 1, reg: w11, asm: "VPMOVUSDW", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VPMOVUSDWMasked128_128", argLength: 2, reg: wkw, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSDWMasked128_256", argLength: 2, reg: wkw, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSDWMasked256", argLength: 2, reg: wkw, asm: "VPMOVUSDW", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VPMOVUSQB128_128", argLength: 1, reg: w11, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSQB128_256", argLength: 1, reg: w11, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSQB128_512", argLength: 1, reg: w11, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSQBMasked128_128", argLength: 2, reg: wkw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSQBMasked128_256", argLength: 2, reg: wkw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSQBMasked128_512", argLength: 2, reg: wkw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSQD128_128", argLength: 1, reg: w11, asm: "VPMOVUSQD", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSQD128_256", argLength: 1, reg: w11, asm: "VPMOVUSQD", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSQD256", argLength: 1, reg: w11, asm: "VPMOVUSQD", commutative: false, typ: "Vec256", resultInArg0: false},
@@ -798,7 +798,11 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VPMOVUSQWMasked128_128", argLength: 2, reg: wkw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSQWMasked128_256", argLength: 2, reg: wkw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSQWMasked128_512", argLength: 2, reg: wkw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSWB128_128", argLength: 1, reg: w11, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSWB128_256", argLength: 1, reg: w11, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSWB256", argLength: 1, reg: w11, asm: "VPMOVUSWB", commutative: false, typ: "Vec256", resultInArg0: false},
+		{name: "VPMOVUSWBMasked128_128", argLength: 2, reg: wkw, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: false},
+		{name: "VPMOVUSWBMasked128_256", argLength: 2, reg: wkw, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVUSWBMasked256", argLength: 2, reg: wkw, asm: "VPMOVUSWB", commutative: false, typ: "Vec256", resultInArg0: false},
 		{name: "VPMOVWB128_128", argLength: 1, reg: w11, asm: "VPMOVWB", commutative: false, typ: "Vec128", resultInArg0: false},
 		{name: "VPMOVWB128_256", argLength: 1, reg: w11, asm: "VPMOVWB", commutative: false, typ: "Vec128", resultInArg0: false},
@@ -1590,38 +1594,26 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VDIVPSMasked128load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: false},
 		{name: "VDIVPSMasked256load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: false},
 		{name: "VDIVPSMasked512load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: false},
-		{name: "VFMADD213PD128load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADD213PD256load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PD512load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADD213PS128load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADD213PS256load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PS512load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADD213PSMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADDSUB213PD128load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADDSUB213PD256load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PD512load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADDSUB213PS128load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMADDSUB213PS256load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PS512load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMADDSUB213PSMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMSUBADD213PD128load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMSUBADD213PD256load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PD512load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMSUBADD213PS128load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VFMSUBADD213PS256load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PS512load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VFMSUBADD213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
@@ -1698,14 +1690,6 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VPCMPEQQ512load", argLength: 3, reg: w2kload, asm: "VPCMPEQQ", commutative: false, typ: "Mask", aux: "SymOff", symEffect: "Read", resultInArg0: false},
 		{name: "VPCMPGTD512load", argLength: 3, reg: w2kload, asm: "VPCMPGTD", commutative: false, typ: "Mask", aux: "SymOff", symEffect: "Read", resultInArg0: false},
 		{name: "VPCMPGTQ512load", argLength: 3, reg: w2kload, asm: "VPCMPGTQ", commutative: false, typ: "Mask", aux: "SymOff", symEffect: "Read", resultInArg0: false},
-		{name: "VPDPBUSD512load", argLength: 4, reg: w31load, asm: "VPDPBUSD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDMasked128load", argLength: 5, reg: w3kwload, asm: "VPDPBUSD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDMasked256load", argLength: 5, reg: w3kwload, asm: "VPDPBUSD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDMasked512load", argLength: 5, reg: w3kwload, asm: "VPDPBUSD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDS512load", argLength: 4, reg: w31load, asm: "VPDPBUSDS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDSMasked128load", argLength: 5, reg: w3kwload, asm: "VPDPBUSDS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDSMasked256load", argLength: 5, reg: w3kwload, asm: "VPDPBUSDS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
-		{name: "VPDPBUSDSMasked512load", argLength: 5, reg: w3kwload, asm: "VPDPBUSDS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VPDPWSSD512load", argLength: 4, reg: w31load, asm: "VPDPWSSD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VPDPWSSDMasked128load", argLength: 5, reg: w3kwload, asm: "VPDPWSSD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true},
 		{name: "VPDPWSSDMasked256load", argLength: 5, reg: w3kwload, asm: "VPDPWSSD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true},
@@ -2382,15 +2366,23 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf
 		{name: "VPMOVSXWQMasked128Merging", argLength: 3, reg: w2kw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVSXWQMasked256Merging", argLength: 3, reg: w2kw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VPMOVSXWQMasked512Merging", argLength: 3, reg: w2kw, asm: "VPMOVSXWQ", commutative: false, typ: "Vec512", resultInArg0: true},
+		{name: "VPMOVUSDBMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSDBMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSDBMasked128_512Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDB", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSDWMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSDWMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDW", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSDWMasked256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSDW", commutative: false, typ: "Vec256", resultInArg0: true},
+		{name: "VPMOVUSQBMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSQBMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSQBMasked128_512Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQB", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSQDMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSQDMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQD", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSQDMasked256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQD", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VPMOVUSQWMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSQWMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSQWMasked128_512Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSQW", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSWBMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: true},
+		{name: "VPMOVUSWBMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSWB", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVUSWBMasked256Merging", argLength: 3, reg: w2kw, asm: "VPMOVUSWB", commutative: false, typ: "Vec256", resultInArg0: true},
 		{name: "VPMOVWBMasked128_128Merging", argLength: 3, reg: w2kw, asm: "VPMOVWB", commutative: false, typ: "Vec128", resultInArg0: true},
 		{name: "VPMOVWBMasked128_256Merging", argLength: 3, reg: w2kw, asm: "VPMOVWB", commutative: false, typ: "Vec128", resultInArg0: true},

src/cmd/compile/internal/ssa/_gen/simdgenericOps.go

@@ -1,4 +1,4 @@
-// Code generated by x/arch/internal/simdgen using 'go run . -xedPath $XED_PATH -o godefs -goroot $GOROOT go.yaml types.yaml categories.yaml'; DO NOT EDIT.
+// Code generated by 'simdgen -o godefs -goroot $GOROOT -xedPath $XED_PATH go.yaml types.yaml categories.yaml'; DO NOT EDIT.
 
 package main
 
@@ -48,19 +48,19 @@ func simdGenericOps() []opData {
 		{name: "AddInt64x4", argLength: 2, commutative: true},
 		{name: "AddInt64x8", argLength: 2, commutative: true},
 		{name: "AddPairsFloat32x4", argLength: 2, commutative: false},
-		{name: "AddPairsFloat32x8", argLength: 2, commutative: false},
 		{name: "AddPairsFloat64x2", argLength: 2, commutative: false},
-		{name: "AddPairsFloat64x4", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedFloat32x8", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedFloat64x4", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedInt16x16", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedInt32x8", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedUint16x16", argLength: 2, commutative: false},
+		{name: "AddPairsGroupedUint32x8", argLength: 2, commutative: false},
 		{name: "AddPairsInt16x8", argLength: 2, commutative: false},
-		{name: "AddPairsInt16x16", argLength: 2, commutative: false},
 		{name: "AddPairsInt32x4", argLength: 2, commutative: false},
-		{name: "AddPairsInt32x8", argLength: 2, commutative: false},
+		{name: "AddPairsSaturatedGroupedInt16x16", argLength: 2, commutative: false},
 		{name: "AddPairsSaturatedInt16x8", argLength: 2, commutative: false},
-		{name: "AddPairsSaturatedInt16x16", argLength: 2, commutative: false},
 		{name: "AddPairsUint16x8", argLength: 2, commutative: false},
-		{name: "AddPairsUint16x16", argLength: 2, commutative: false},
 		{name: "AddPairsUint32x4", argLength: 2, commutative: false},
-		{name: "AddPairsUint32x8", argLength: 2, commutative: false},
 		{name: "AddSaturatedInt8x16", argLength: 2, commutative: true},
 		{name: "AddSaturatedInt8x32", argLength: 2, commutative: true},
 		{name: "AddSaturatedInt8x64", argLength: 2, commutative: true},
@@ -143,36 +143,36 @@ func simdGenericOps() []opData {
 		{name: "AverageUint16x8", argLength: 2, commutative: true},
 		{name: "AverageUint16x16", argLength: 2, commutative: true},
 		{name: "AverageUint16x32", argLength: 2, commutative: true},
-		{name: "Broadcast128Float32x4", argLength: 1, commutative: false},
-		{name: "Broadcast128Float64x2", argLength: 1, commutative: false},
-		{name: "Broadcast128Int8x16", argLength: 1, commutative: false},
-		{name: "Broadcast128Int16x8", argLength: 1, commutative: false},
-		{name: "Broadcast128Int32x4", argLength: 1, commutative: false},
-		{name: "Broadcast128Int64x2", argLength: 1, commutative: false},
-		{name: "Broadcast128Uint8x16", argLength: 1, commutative: false},
-		{name: "Broadcast128Uint16x8", argLength: 1, commutative: false},
-		{name: "Broadcast128Uint32x4", argLength: 1, commutative: false},
-		{name: "Broadcast128Uint64x2", argLength: 1, commutative: false},
-		{name: "Broadcast256Float32x4", argLength: 1, commutative: false},
-		{name: "Broadcast256Float64x2", argLength: 1, commutative: false},
-		{name: "Broadcast256Int8x16", argLength: 1, commutative: false},
-		{name: "Broadcast256Int16x8", argLength: 1, commutative: false},
-		{name: "Broadcast256Int32x4", argLength: 1, commutative: false},
-		{name: "Broadcast256Int64x2", argLength: 1, commutative: false},
-		{name: "Broadcast256Uint8x16", argLength: 1, commutative: false},
-		{name: "Broadcast256Uint16x8", argLength: 1, commutative: false},
-		{name: "Broadcast256Uint32x4", argLength: 1, commutative: false},
-		{name: "Broadcast256Uint64x2", argLength: 1, commutative: false},
-		{name: "Broadcast512Float32x4", argLength: 1, commutative: false},
-		{name: "Broadcast512Float64x2", argLength: 1, commutative: false},
-		{name: "Broadcast512Int8x16", argLength: 1, commutative: false},
-		{name: "Broadcast512Int16x8", argLength: 1, commutative: false},
-		{name: "Broadcast512Int32x4", argLength: 1, commutative: false},
-		{name: "Broadcast512Int64x2", argLength: 1, commutative: false},
-		{name: "Broadcast512Uint8x16", argLength: 1, commutative: false},
-		{name: "Broadcast512Uint16x8", argLength: 1, commutative: false},
-		{name: "Broadcast512Uint32x4", argLength: 1, commutative: false},
-		{name: "Broadcast512Uint64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To2Float64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To2Int64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To2Uint64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Float32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Float64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Int32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Int64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Uint32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To4Uint64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Float32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Float64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Int16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Int32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Int64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Uint16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Uint32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To8Uint64x2", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Float32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Int8x16", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Int16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Int32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Uint8x16", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Uint16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To16Uint32x4", argLength: 1, commutative: false},
+		{name: "Broadcast1To32Int8x16", argLength: 1, commutative: false},
+		{name: "Broadcast1To32Int16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To32Uint8x16", argLength: 1, commutative: false},
+		{name: "Broadcast1To32Uint16x8", argLength: 1, commutative: false},
+		{name: "Broadcast1To64Int8x16", argLength: 1, commutative: false},
+		{name: "Broadcast1To64Uint8x16", argLength: 1, commutative: false},
 		{name: "CeilFloat32x4", argLength: 1, commutative: false},
 		{name: "CeilFloat32x8", argLength: 1, commutative: false},
 		{name: "CeilFloat64x2", argLength: 1, commutative: false},
@@ -304,12 +304,6 @@ func simdGenericOps() []opData {
 		{name: "DotProductPairsSaturatedUint8x16", argLength: 2, commutative: false},
 		{name: "DotProductPairsSaturatedUint8x32", argLength: 2, commutative: false},
 		{name: "DotProductPairsSaturatedUint8x64", argLength: 2, commutative: false},
-		{name: "DotProductQuadrupleInt32x4", argLength: 3, commutative: false},
-		{name: "DotProductQuadrupleInt32x8", argLength: 3, commutative: false},
-		{name: "DotProductQuadrupleInt32x16", argLength: 3, commutative: false},
-		{name: "DotProductQuadrupleSaturatedInt32x4", argLength: 3, commutative: false},
-		{name: "DotProductQuadrupleSaturatedInt32x8", argLength: 3, commutative: false},
-		{name: "DotProductQuadrupleSaturatedInt32x16", argLength: 3, commutative: false},
 		{name: "EqualFloat32x4", argLength: 2, commutative: true},
 		{name: "EqualFloat32x8", argLength: 2, commutative: true},
 		{name: "EqualFloat32x16", argLength: 2, commutative: true},
@@ -370,26 +364,26 @@ func simdGenericOps() []opData {
 		{name: "ExpandUint64x2", argLength: 2, commutative: false},
 		{name: "ExpandUint64x4", argLength: 2, commutative: false},
 		{name: "ExpandUint64x8", argLength: 2, commutative: false},
-		{name: "ExtendLo2ToInt64x2Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo2ToInt64x2Int16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo2ToInt64x2Int32x4", argLength: 1, commutative: false},
-		{name: "ExtendLo2ToUint64x2Uint8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo2ToUint64x2Uint16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo2ToUint64x2Uint32x4", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToInt32x4Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToInt32x4Int16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToInt64x4Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToInt64x4Int16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToUint32x4Uint8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToUint32x4Uint16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToUint64x4Uint8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo4ToUint64x4Uint16x8", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToInt16x8Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToInt32x8Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToInt64x8Int8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToUint16x8Uint8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToUint32x8Uint8x16", argLength: 1, commutative: false},
-		{name: "ExtendLo8ToUint64x8Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToInt64Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToInt64Int16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToInt64Int32x4", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToUint64Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToUint64Uint16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo2ToUint64Uint32x4", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToInt32Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToInt32Int16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToInt64Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToInt64Int16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToUint32Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToUint32Uint16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToUint64Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo4ToUint64Uint16x8", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToInt16Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToInt32Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToInt64Int8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToUint16Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToUint32Uint8x16", argLength: 1, commutative: false},
+		{name: "ExtendLo8ToUint64Uint8x16", argLength: 1, commutative: false},
 		{name: "ExtendToInt16Int8x16", argLength: 1, commutative: false},
 		{name: "ExtendToInt16Int8x32", argLength: 1, commutative: false},
 		{name: "ExtendToInt32Int8x16", argLength: 1, commutative: false},
@@ -525,12 +519,6 @@ func simdGenericOps() []opData {
 		{name: "InterleaveLoUint16x8", argLength: 2, commutative: false},
 		{name: "InterleaveLoUint32x4", argLength: 2, commutative: false},
 		{name: "InterleaveLoUint64x2", argLength: 2, commutative: false},
-		{name: "IsNanFloat32x4", argLength: 2, commutative: true},
-		{name: "IsNanFloat32x8", argLength: 2, commutative: true},
-		{name: "IsNanFloat32x16", argLength: 2, commutative: true},
-		{name: "IsNanFloat64x2", argLength: 2, commutative: true},
-		{name: "IsNanFloat64x4", argLength: 2, commutative: true},
-		{name: "IsNanFloat64x8", argLength: 2, commutative: true},
 		{name: "LeadingZerosInt32x4", argLength: 1, commutative: false},
 		{name: "LeadingZerosInt32x8", argLength: 1, commutative: false},
 		{name: "LeadingZerosInt32x16", argLength: 1, commutative: false},
@@ -830,9 +818,9 @@ func simdGenericOps() []opData {
 		{name: "SaturateToInt8Int64x2", argLength: 1, commutative: false},
 		{name: "SaturateToInt8Int64x4", argLength: 1, commutative: false},
 		{name: "SaturateToInt8Int64x8", argLength: 1, commutative: false},
+		{name: "SaturateToInt16ConcatGroupedInt32x8", argLength: 2, commutative: false},
+		{name: "SaturateToInt16ConcatGroupedInt32x16", argLength: 2, commutative: false},
 		{name: "SaturateToInt16ConcatInt32x4", argLength: 2, commutative: false},
-		{name: "SaturateToInt16ConcatInt32x8", argLength: 2, commutative: false},
-		{name: "SaturateToInt16ConcatInt32x16", argLength: 2, commutative: false},
 		{name: "SaturateToInt16Int32x4", argLength: 1, commutative: false},
 		{name: "SaturateToInt16Int32x8", argLength: 1, commutative: false},
 		{name: "SaturateToInt16Int32x16", argLength: 1, commutative: false},
@@ -842,18 +830,18 @@ func simdGenericOps() []opData {
 		{name: "SaturateToInt32Int64x2", argLength: 1, commutative: false},
 		{name: "SaturateToInt32Int64x4", argLength: 1, commutative: false},
 		{name: "SaturateToInt32Int64x8", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int16x8", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int16x16", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int32x4", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int32x8", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int32x16", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int64x2", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int64x4", argLength: 1, commutative: false},
-		{name: "SaturateToUint8Int64x8", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint16x8", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint16x16", argLength: 1, commutative: false},
 		{name: "SaturateToUint8Uint16x32", argLength: 1, commutative: false},
-		{name: "SaturateToUint16ConcatUint32x4", argLength: 2, commutative: false},
-		{name: "SaturateToUint16ConcatUint32x8", argLength: 2, commutative: false},
-		{name: "SaturateToUint16ConcatUint32x16", argLength: 2, commutative: false},
+		{name: "SaturateToUint8Uint32x4", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint32x8", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint32x16", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint64x2", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint64x4", argLength: 1, commutative: false},
+		{name: "SaturateToUint8Uint64x8", argLength: 1, commutative: false},
+		{name: "SaturateToUint16ConcatGroupedInt32x8", argLength: 2, commutative: false},
+		{name: "SaturateToUint16ConcatGroupedInt32x16", argLength: 2, commutative: false},
+		{name: "SaturateToUint16ConcatInt32x4", argLength: 2, commutative: false},
 		{name: "SaturateToUint16Uint32x4", argLength: 1, commutative: false},
 		{name: "SaturateToUint16Uint32x8", argLength: 1, commutative: false},
 		{name: "SaturateToUint16Uint32x16", argLength: 1, commutative: false},
@@ -1042,19 +1030,19 @@ func simdGenericOps() []opData {
 		{name: "SubInt64x4", argLength: 2, commutative: false},
 		{name: "SubInt64x8", argLength: 2, commutative: false},
 		{name: "SubPairsFloat32x4", argLength: 2, commutative: false},
-		{name: "SubPairsFloat32x8", argLength: 2, commutative: false},
 		{name: "SubPairsFloat64x2", argLength: 2, commutative: false},
-		{name: "SubPairsFloat64x4", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedFloat32x8", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedFloat64x4", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedInt16x16", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedInt32x8", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedUint16x16", argLength: 2, commutative: false},
+		{name: "SubPairsGroupedUint32x8", argLength: 2, commutative: false},
 		{name: "SubPairsInt16x8", argLength: 2, commutative: false},
-		{name: "SubPairsInt16x16", argLength: 2, commutative: false},
 		{name: "SubPairsInt32x4", argLength: 2, commutative: false},
-		{name: "SubPairsInt32x8", argLength: 2, commutative: false},
+		{name: "SubPairsSaturatedGroupedInt16x16", argLength: 2, commutative: false},
 		{name: "SubPairsSaturatedInt16x8", argLength: 2, commutative: false},
-		{name: "SubPairsSaturatedInt16x16", argLength: 2, commutative: false},
 		{name: "SubPairsUint16x8", argLength: 2, commutative: false},
-		{name: "SubPairsUint16x16", argLength: 2, commutative: false},
 		{name: "SubPairsUint32x4", argLength: 2, commutative: false},
-		{name: "SubPairsUint32x8", argLength: 2, commutative: false},
 		{name: "SubSaturatedInt8x16", argLength: 2, commutative: false},
 		{name: "SubSaturatedInt8x32", argLength: 2, commutative: false},
 		{name: "SubSaturatedInt8x64", argLength: 2, commutative: false},

src/cmd/compile/internal/ssa/check.go

@@ -345,6 +345,47 @@ func checkFunc(f *Func) {
 						v.Op, v.Args[1].Type.String())
 				}
 			}
+			// Check size of args.
+			// This list isn't exhaustive, just the common ops.
+			// It also can't handle ops with args of different types, like shifts.
+			var argSize int64
+			switch v.Op {
+			case OpAdd8, OpSub8, OpMul8, OpDiv8, OpDiv8u, OpMod8, OpMod8u,
+				OpAnd8, OpOr8, OpXor8,
+				OpEq8, OpNeq8, OpLess8, OpLeq8,
+				OpNeg8, OpCom8,
+				OpSignExt8to16, OpSignExt8to32, OpSignExt8to64,
+				OpZeroExt8to16, OpZeroExt8to32, OpZeroExt8to64:
+				argSize = 1
+			case OpAdd16, OpSub16, OpMul16, OpDiv16, OpDiv16u, OpMod16, OpMod16u,
+				OpAnd16, OpOr16, OpXor16,
+				OpEq16, OpNeq16, OpLess16, OpLeq16,
+				OpNeg16, OpCom16,
+				OpSignExt16to32, OpSignExt16to64,
+				OpZeroExt16to32, OpZeroExt16to64,
+				OpTrunc16to8:
+				argSize = 2
+			case OpAdd32, OpSub32, OpMul32, OpDiv32, OpDiv32u, OpMod32, OpMod32u,
+				OpAnd32, OpOr32, OpXor32,
+				OpEq32, OpNeq32, OpLess32, OpLeq32,
+				OpNeg32, OpCom32,
+				OpSignExt32to64, OpZeroExt32to64,
+				OpTrunc32to8, OpTrunc32to16:
+				argSize = 4
+			case OpAdd64, OpSub64, OpMul64, OpDiv64, OpDiv64u, OpMod64, OpMod64u,
+				OpAnd64, OpOr64, OpXor64,
+				OpEq64, OpNeq64, OpLess64, OpLeq64,
+				OpNeg64, OpCom64,
+				OpTrunc64to8, OpTrunc64to16, OpTrunc64to32:
+				argSize = 8
+			}
+			if argSize != 0 {
+				for i, arg := range v.Args {
+					if arg.Type.Size() != argSize {
+						f.Fatalf("arg %d to %s (%v) should be %d bytes in size, it is %s", i, v.Op, v, argSize, arg.Type.String())
+					}
+				}
+			}
 
 			// TODO: check for cycles in values
 		}

src/cmd/compile/internal/ssa/config.go

@@ -304,6 +304,9 @@ func NewConfig(arch string, types Types, ctxt *obj.Link, optimize, softfloat boo
 		c.LinkReg = linkRegLOONG64
 		c.hasGReg = true
 		c.unalignedOK = true
+		c.haveBswap64 = true
+		c.haveBswap32 = true
+		c.haveBswap16 = true
 		c.haveCondSelect = true
 	case "s390x":
 		c.PtrSize = 8

src/cmd/compile/internal/ssa/prove.go

@@ -250,6 +250,10 @@ func fitsInBitsU(x uint64, b uint) bool {
 	return x>>b == 0
 }
 
+func noLimit() limit {
+	return noLimitForBitsize(64)
+}
+
 func noLimitForBitsize(bitsize uint) limit {
 	return limit{min: -(1 << (bitsize - 1)), max: 1<<(bitsize-1) - 1, umin: 0, umax: 1<<bitsize - 1}
 }
@@ -269,6 +273,13 @@ func convertIntWithBitsize[Target uint64 | int64, Source uint64 | int64](x Sourc
 	}
 }
 
+func (l limit) unsignedFixedLeadingBits() (fixed uint64, count uint) {
+	varying := uint(bits.Len64(l.umin ^ l.umax))
+	count = uint(bits.LeadingZeros64(l.umin ^ l.umax))
+	fixed = l.umin &^ (1<<varying - 1)
+	return
+}
+
 // add returns the limit obtained by adding a value with limit l
 // to a value with limit l2. The result must fit in b bits.
 func (l limit) add(l2 limit, b uint) limit {
@@ -295,7 +306,7 @@ func (l limit) add(l2 limit, b uint) limit {
 		return limit{min: int64r, max: int64r, umin: r, umax: r}
 	}
 
-	r := noLimit
+	r := noLimit()
 	min, minOk := safeAdd(l.min, l2.min, b)
 	max, maxOk := safeAdd(l.max, l2.max, b)
 	if minOk && maxOk {
@@ -313,7 +324,7 @@ func (l limit) add(l2 limit, b uint) limit {
 
 // same as add but for subtraction.
 func (l limit) sub(l2 limit, b uint) limit {
-	r := noLimit
+	r := noLimit()
 	min, minOk := safeSub(l.min, l2.max, b)
 	max, maxOk := safeSub(l.max, l2.min, b)
 	if minOk && maxOk {
@@ -331,7 +342,7 @@ func (l limit) sub(l2 limit, b uint) limit {
 
 // same as add but for multiplication.
 func (l limit) mul(l2 limit, b uint) limit {
-	r := noLimit
+	r := noLimit()
 	umaxhi, umaxlo := bits.Mul64(l.umax, l2.umax)
 	if umaxhi == 0 && fitsInBitsU(umaxlo, b) {
 		r.umax = umaxlo
@@ -353,7 +364,7 @@ func (l limit) mul(l2 limit, b uint) limit {
 
 // Similar to add, but compute 1 << l if it fits without overflow in b bits.
 func (l limit) exp2(b uint) limit {
-	r := noLimit
+	r := noLimit()
 	if l.umax < uint64(b) {
 		r.umin = 1 << l.umin
 		r.umax = 1 << l.umax
@@ -404,7 +415,53 @@ func (l limit) neg(b uint) limit {
 	return l.com(b).add(limit{min: 1, max: 1, umin: 1, umax: 1}, b)
 }
 
-var noLimit = limit{math.MinInt64, math.MaxInt64, 0, math.MaxUint64}
+// Similar to add, but computes the TrailingZeros of the limit for bitsize b.
+func (l limit) ctz(b uint) limit {
+	fixed, fixedCount := l.unsignedFixedLeadingBits()
+	if fixedCount == 64 {
+		constResult := min(uint(bits.TrailingZeros64(fixed)), b)
+		return limit{min: int64(constResult), max: int64(constResult), umin: uint64(constResult), umax: uint64(constResult)}
+	}
+
+	varying := 64 - fixedCount
+	if l.umin&((1<<varying)-1) != 0 {
+		// there will always be at least one non-zero bit in the varying part
+		varying--
+		return noLimit().unsignedMax(uint64(varying))
+	}
+	return noLimit().unsignedMax(uint64(min(uint(bits.TrailingZeros64(fixed)), b)))
+}
+
+// Similar to add, but computes the Len of the limit for bitsize b.
+func (l limit) bitlen(b uint) limit {
+	return noLimit().unsignedMinMax(
+		uint64(bits.Len64(l.umin)),
+		uint64(bits.Len64(l.umax)),
+	)
+}
+
+// Similar to add, but computes the PopCount of the limit for bitsize b.
+func (l limit) popcount(b uint) limit {
+	fixed, fixedCount := l.unsignedFixedLeadingBits()
+	varying := 64 - fixedCount
+	fixedContribution := uint64(bits.OnesCount64(fixed))
+
+	min := fixedContribution
+	max := fixedContribution + uint64(varying)
+
+	varyingMask := uint64(1)<<varying - 1
+
+	if varyingPartOfUmax := l.umax & varyingMask; uint(bits.OnesCount64(varyingPartOfUmax)) != varying {
+		// there is at least one zero bit in the varying part
+		max--
+	}
+	if varyingPartOfUmin := l.umin & varyingMask; varyingPartOfUmin != 0 {
+		// there is at least one non-zero bit in the varying part
+		min++
+	}
+
+	return noLimit().unsignedMinMax(min, max)
+}
 
 // a limitFact is a limit known for a particular value.
 type limitFact struct {
@@ -556,7 +613,7 @@ func (ft *factsTable) pointerNil(v *Value) {
 	ft.newLimit(v, limit{min: 0, max: 0, umin: 0, umax: 0})
 }
 func (ft *factsTable) pointerNonNil(v *Value) {
-	l := noLimit
+	l := noLimit()
 	l.umin = 1
 	ft.newLimit(v, l)
 }
@@ -1788,15 +1845,15 @@ func initLimit(v *Value) limit {
 		case OpConstNil:
 			return limit{min: 0, max: 0, umin: 0, umax: 0}
 		case OpAddr, OpLocalAddr: // TODO: others?
-			l := noLimit
+			l := noLimit()
 			l.umin = 1
 			return l
 		default:
-			return noLimit
+			return noLimit()
 		}
 	}
 	if !v.Type.IsInteger() {
-		return noLimit
+		return noLimit()
 	}
 
 	// Default limits based on type.
@@ -1904,55 +1961,20 @@ func (ft *factsTable) flowLimit(v *Value) {
 		}
 
 	// math/bits
-	case OpCtz64:
-		a := ft.limits[v.Args[0].ID]
-		if a.nonzero() {
-			ft.unsignedMax(v, uint64(bits.Len64(a.umax)-1))
-		}
-	case OpCtz32:
-		a := ft.limits[v.Args[0].ID]
-		if a.nonzero() {
-			ft.unsignedMax(v, uint64(bits.Len32(uint32(a.umax))-1))
-		}
-	case OpCtz16:
-		a := ft.limits[v.Args[0].ID]
-		if a.nonzero() {
-			ft.unsignedMax(v, uint64(bits.Len16(uint16(a.umax))-1))
-		}
-	case OpCtz8:
-		a := ft.limits[v.Args[0].ID]
-		if a.nonzero() {
-			ft.unsignedMax(v, uint64(bits.Len8(uint8(a.umax))-1))
-		}
+	case OpCtz64, OpCtz32, OpCtz16, OpCtz8:
+		a := v.Args[0]
+		al := ft.limits[a.ID]
+		ft.newLimit(v, al.ctz(uint(a.Type.Size())*8))
 
 	case OpPopCount64, OpPopCount32, OpPopCount16, OpPopCount8:
-		a := ft.limits[v.Args[0].ID]
-		changingBitsCount := uint64(bits.Len64(a.umax ^ a.umin))
-		sharedLeadingMask := ^(uint64(1)<<changingBitsCount - 1)
-		fixedBits := a.umax & sharedLeadingMask
-		min := uint64(bits.OnesCount64(fixedBits))
-		ft.unsignedMinMax(v, min, min+changingBitsCount)
+		a := v.Args[0]
+		al := ft.limits[a.ID]
+		ft.newLimit(v, al.popcount(uint(a.Type.Size())*8))
 
-	case OpBitLen64:
-		a := ft.limits[v.Args[0].ID]
-		ft.unsignedMinMax(v,
-			uint64(bits.Len64(a.umin)),
-			uint64(bits.Len64(a.umax)))
-	case OpBitLen32:
-		a := ft.limits[v.Args[0].ID]
-		ft.unsignedMinMax(v,
-			uint64(bits.Len32(uint32(a.umin))),
-			uint64(bits.Len32(uint32(a.umax))))
-	case OpBitLen16:
-		a := ft.limits[v.Args[0].ID]
-		ft.unsignedMinMax(v,
-			uint64(bits.Len16(uint16(a.umin))),
-			uint64(bits.Len16(uint16(a.umax))))
-	case OpBitLen8:
-		a := ft.limits[v.Args[0].ID]
-		ft.unsignedMinMax(v,
-			uint64(bits.Len8(uint8(a.umin))),
-			uint64(bits.Len8(uint8(a.umax))))
+	case OpBitLen64, OpBitLen32, OpBitLen16, OpBitLen8:
+		a := v.Args[0]
+		al := ft.limits[a.ID]
+		ft.newLimit(v, al.bitlen(uint(a.Type.Size())*8))
 
 	// Masks.
 
@@ -2043,7 +2065,7 @@ func (ft *factsTable) flowLimit(v *Value) {
 	case OpDiv64u, OpDiv32u, OpDiv16u, OpDiv8u:
 		a := ft.limits[v.Args[0].ID]
 		b := ft.limits[v.Args[1].ID]
-		lim := noLimit
+		lim := noLimit()
 		if b.umax > 0 {
 			lim = lim.unsignedMin(a.umin / b.umax)
 		}
@@ -2165,24 +2187,22 @@ func (ft *factsTable) detectSubRelations(v *Value) {
 		return // x-y might overflow
 	}
 
-	// Subtracting a positive number only makes
-	// things smaller.
-	if yLim.min >= 0 {
+	// Subtracting a positive non-zero number only makes
+	// things smaller. If it's positive or zero, it might
+	// also do nothing (x-0 == v).
+	if yLim.min > 0 {
+		ft.update(v.Block, v, x, signed, lt)
+	} else if yLim.min == 0 {
 		ft.update(v.Block, v, x, signed, lt|eq)
-		// TODO: is this worth it?
-		//if yLim.min > 0 {
-		//	ft.update(v.Block, v, x, signed, lt)
-		//}
 	}
 
 	// Subtracting a number from a bigger one
-	// can't go below 0.
-	if ft.orderS.OrderedOrEqual(y, x) {
+	// can't go below 1. If the numbers might be
+	// equal, then it can't go below 0.
+	if ft.orderS.Ordered(y, x) {
+		ft.signedMin(v, 1)
+	} else if ft.orderS.OrderedOrEqual(y, x) {
 		ft.setNonNegative(v)
-		// TODO: is this worth it?
-		//if ft.orderS.Ordered(y, x) {
-		//	ft.signedMin(v, 1)
-		//}
 	}
 }
 
@@ -2888,9 +2908,9 @@ func simplifyBlock(sdom SparseTree, ft *factsTable, b *Block) {
 			xl := ft.limits[x.ID]
 			y := v.Args[1]
 			yl := ft.limits[y.ID]
-			if xl.umin == xl.umax && isUnsignedPowerOfTwo(xl.umin) ||
+			if xl.umin == xl.umax && isPowerOfTwo(xl.umin) ||
 				xl.min == xl.max && isPowerOfTwo(xl.min) ||
-				yl.umin == yl.umax && isUnsignedPowerOfTwo(yl.umin) ||
+				yl.umin == yl.umax && isPowerOfTwo(yl.umin) ||
 				yl.min == yl.max && isPowerOfTwo(yl.min) {
 				// 0,1 * a power of two is better done as a shift
 				break

src/cmd/compile/internal/ssa/prove_test.go

@@ -6,11 +6,11 @@ package ssa
 
 import (
 	"math"
+	"math/bits"
 	"testing"
 )
 
-func testLimitUnaryOpSigned8(t *testing.T, opName string, op func(l limit, bitsize uint) limit, opImpl func(int8) int8) {
-	sizeLimit := noLimitForBitsize(8)
+func testLimitUnaryOpSigned8(t *testing.T, opName string, initLimit limit, op func(l limit, bitsize uint) limit, opImpl func(int8) int8) {
 	for min := math.MinInt8; min <= math.MaxInt8; min++ {
 		for max := min; max <= math.MaxInt8; max++ {
 			realSmallest, realBiggest := int8(math.MaxInt8), int8(math.MinInt8)
@@ -26,7 +26,7 @@ func testLimitUnaryOpSigned8(t *testing.T, opName string, op func(l limit, bitsi
 
 			l := limit{int64(min), int64(max), 0, math.MaxUint64}
 			l = op(l, 8)
-			l = l.intersect(sizeLimit) // We assume this is gonna be used by newLimit which is seeded by the op size already.
+			l = l.intersect(initLimit) // We assume this is gonna be used by newLimit which is seeded by the op size already.
 
 			if l.min != int64(realSmallest) || l.max != int64(realBiggest) {
 				t.Errorf("%s(%d..%d) = %d..%d; want %d..%d", opName, min, max, l.min, l.max, realSmallest, realBiggest)
@@ -35,8 +35,7 @@ func testLimitUnaryOpSigned8(t *testing.T, opName string, op func(l limit, bitsi
 	}
 }
 
-func testLimitUnaryOpUnsigned8(t *testing.T, opName string, op func(l limit, bitsize uint) limit, opImpl func(uint8) uint8) {
-	sizeLimit := noLimitForBitsize(8)
+func testLimitUnaryOpUnsigned8(t *testing.T, opName string, initLimit limit, op func(l limit, bitsize uint) limit, opImpl func(uint8) uint8) {
 	for min := 0; min <= math.MaxUint8; min++ {
 		for max := min; max <= math.MaxUint8; max++ {
 			realSmallest, realBiggest := uint8(math.MaxUint8), uint8(0)
@@ -52,7 +51,7 @@ func testLimitUnaryOpUnsigned8(t *testing.T, opName string, op func(l limit, bit
 
 			l := limit{math.MinInt64, math.MaxInt64, uint64(min), uint64(max)}
 			l = op(l, 8)
-			l = l.intersect(sizeLimit) // We assume this is gonna be used by newLimit which is seeded by the op size already.
+			l = l.intersect(initLimit) // We assume this is gonna be used by newLimit which is seeded by the op size already.
 
 			if l.umin != uint64(realSmallest) || l.umax != uint64(realBiggest) {
 				t.Errorf("%s(%d..%d) = %d..%d; want %d..%d", opName, min, max, l.umin, l.umax, realSmallest, realBiggest)
@@ -62,15 +61,27 @@ func testLimitUnaryOpUnsigned8(t *testing.T, opName string, op func(l limit, bit
 }
 
 func TestLimitNegSigned(t *testing.T) {
-	testLimitUnaryOpSigned8(t, "neg", limit.neg, func(x int8) int8 { return -x })
+	testLimitUnaryOpSigned8(t, "neg", noLimitForBitsize(8), limit.neg, func(x int8) int8 { return -x })
 }
 func TestLimitNegUnsigned(t *testing.T) {
-	testLimitUnaryOpUnsigned8(t, "neg", limit.neg, func(x uint8) uint8 { return -x })
+	testLimitUnaryOpUnsigned8(t, "neg", noLimitForBitsize(8), limit.neg, func(x uint8) uint8 { return -x })
 }
 
 func TestLimitComSigned(t *testing.T) {
-	testLimitUnaryOpSigned8(t, "com", limit.com, func(x int8) int8 { return ^x })
+	testLimitUnaryOpSigned8(t, "com", noLimitForBitsize(8), limit.com, func(x int8) int8 { return ^x })
 }
 func TestLimitComUnsigned(t *testing.T) {
-	testLimitUnaryOpUnsigned8(t, "com", limit.com, func(x uint8) uint8 { return ^x })
+	testLimitUnaryOpUnsigned8(t, "com", noLimitForBitsize(8), limit.com, func(x uint8) uint8 { return ^x })
+}
+
+func TestLimitCtzUnsigned(t *testing.T) {
+	testLimitUnaryOpUnsigned8(t, "ctz", limit{-128, 127, 0, 8}, limit.ctz, func(x uint8) uint8 { return uint8(bits.TrailingZeros8(x)) })
+}
+
+func TestLimitBitlenUnsigned(t *testing.T) {
+	testLimitUnaryOpUnsigned8(t, "bitlen", limit{-128, 127, 0, 8}, limit.bitlen, func(x uint8) uint8 { return uint8(bits.Len8(x)) })
+}
+
+func TestLimitPopcountUnsigned(t *testing.T) {
+	testLimitUnaryOpUnsigned8(t, "popcount", limit{-128, 127, 0, 8}, limit.popcount, func(x uint8) uint8 { return uint8(bits.OnesCount8(x)) })
 }

src/cmd/compile/internal/ssa/rewrite.go

@@ -480,7 +480,7 @@ func isSpecializedMalloc(aux Aux) bool {
 	name := fn.String()
 	return strings.HasPrefix(name, "runtime.mallocgcSmallNoScanSC") ||
 		strings.HasPrefix(name, "runtime.mallocgcSmallScanNoHeaderSC") ||
-		strings.HasPrefix(name, "runtime.mallocTiny")
+		strings.HasPrefix(name, "runtime.mallocgcTinySize")
 }
 
 // canLoadUnaligned reports if the architecture supports unaligned load operations.
@@ -518,22 +518,17 @@ func log32(n int32) int64 { return log32u(uint32(n)) }
 func log64(n int64) int64 { return log64u(uint64(n)) }
 
 // logXu returns the logarithm of n base 2.
-// n must be a power of 2 (isUnsignedPowerOfTwo returns true)
+// n must be a power of 2 (isPowerOfTwo returns true)
 func log8u(n uint8) int64   { return int64(bits.Len8(n)) - 1 }
 func log16u(n uint16) int64 { return int64(bits.Len16(n)) - 1 }
 func log32u(n uint32) int64 { return int64(bits.Len32(n)) - 1 }
 func log64u(n uint64) int64 { return int64(bits.Len64(n)) - 1 }
 
 // isPowerOfTwoX functions report whether n is a power of 2.
-func isPowerOfTwo[T int8 | int16 | int32 | int64](n T) bool {
+func isPowerOfTwo[T int8 | int16 | int32 | int64 | uint8 | uint16 | uint32 | uint64](n T) bool {
 	return n > 0 && n&(n-1) == 0
 }
 
-// isUnsignedPowerOfTwo reports whether n is an unsigned power of 2.
-func isUnsignedPowerOfTwo[T uint8 | uint16 | uint32 | uint64](n T) bool {
-	return n != 0 && n&(n-1) == 0
-}
-
 // is32Bit reports whether n can be represented as a signed 32 bit integer.
 func is32Bit(n int64) bool {
 	return n == int64(int32(n))
@@ -1356,7 +1351,7 @@ func overlap(offset1, size1, offset2, size2 int64) bool {
 // check if value zeroes out upper 32-bit of 64-bit register.
 // depth limits recursion depth. In AMD64.rules 3 is used as limit,
 // because it catches same amount of cases as 4.
-func zeroUpper32Bits(x *Value, depth int) bool {
+func ZeroUpper32Bits(x *Value, depth int) bool {
 	if x.Type.IsSigned() && x.Type.Size() < 8 {
 		// If the value is signed, it might get re-sign-extended
 		// during spill and restore. See issue 68227.
@@ -1373,6 +1368,8 @@ func zeroUpper32Bits(x *Value, depth int) bool {
 		OpAMD64SHRL, OpAMD64SHRLconst, OpAMD64SARL, OpAMD64SARLconst,
 		OpAMD64SHLL, OpAMD64SHLLconst:
 		return true
+	case OpAMD64MOVQconst:
+		return uint64(uint32(x.AuxInt)) == uint64(x.AuxInt)
 	case OpARM64REV16W, OpARM64REVW, OpARM64RBITW, OpARM64CLZW, OpARM64EXTRWconst,
 		OpARM64MULW, OpARM64MNEGW, OpARM64UDIVW, OpARM64DIVW, OpARM64UMODW,
 		OpARM64MADDW, OpARM64MSUBW, OpARM64RORW, OpARM64RORWconst:
@@ -1388,7 +1385,7 @@ func zeroUpper32Bits(x *Value, depth int) bool {
 			return false
 		}
 		for i := range x.Args {
-			if !zeroUpper32Bits(x.Args[i], depth-1) {
+			if !ZeroUpper32Bits(x.Args[i], depth-1) {
 				return false
 			}
 		}
@@ -1398,14 +1395,16 @@ func zeroUpper32Bits(x *Value, depth int) bool {
 	return false
 }
 
-// zeroUpper48Bits is similar to zeroUpper32Bits, but for upper 48 bits.
-func zeroUpper48Bits(x *Value, depth int) bool {
+// ZeroUpper48Bits is similar to ZeroUpper32Bits, but for upper 48 bits.
+func ZeroUpper48Bits(x *Value, depth int) bool {
 	if x.Type.IsSigned() && x.Type.Size() < 8 {
 		return false
 	}
 	switch x.Op {
 	case OpAMD64MOVWQZX, OpAMD64MOVWload, OpAMD64MOVWloadidx1, OpAMD64MOVWloadidx2:
 		return true
+	case OpAMD64MOVQconst, OpAMD64MOVLconst:
+		return uint64(uint16(x.AuxInt)) == uint64(x.AuxInt)
 	case OpArg: // note: but not ArgIntReg
 		return x.Type.Size() == 2 && x.Block.Func.Config.arch == "amd64"
 	case OpPhi, OpSelect0, OpSelect1:
@@ -1415,7 +1414,7 @@ func zeroUpper48Bits(x *Value, depth int) bool {
 			return false
 		}
 		for i := range x.Args {
-			if !zeroUpper48Bits(x.Args[i], depth-1) {
+			if !ZeroUpper48Bits(x.Args[i], depth-1) {
 				return false
 			}
 		}
@@ -1425,14 +1424,16 @@ func zeroUpper48Bits(x *Value, depth int) bool {
 	return false
 }
 
-// zeroUpper56Bits is similar to zeroUpper32Bits, but for upper 56 bits.
-func zeroUpper56Bits(x *Value, depth int) bool {
+// ZeroUpper56Bits is similar to ZeroUpper32Bits, but for upper 56 bits.
+func ZeroUpper56Bits(x *Value, depth int) bool {
 	if x.Type.IsSigned() && x.Type.Size() < 8 {
 		return false
 	}
 	switch x.Op {
 	case OpAMD64MOVBQZX, OpAMD64MOVBload, OpAMD64MOVBloadidx1:
 		return true
+	case OpAMD64MOVQconst, OpAMD64MOVLconst:
+		return uint64(uint8(x.AuxInt)) == uint64(x.AuxInt)
 	case OpArg: // note: but not ArgIntReg
 		return x.Type.Size() == 1 && x.Block.Func.Config.arch == "amd64"
 	case OpPhi, OpSelect0, OpSelect1:
@@ -1442,7 +1443,7 @@ func zeroUpper56Bits(x *Value, depth int) bool {
 			return false
 		}
 		for i := range x.Args {
-			if !zeroUpper56Bits(x.Args[i], depth-1) {
+			if !ZeroUpper56Bits(x.Args[i], depth-1) {
 				return false
 			}
 		}
@@ -2786,3 +2787,12 @@ func imakeOfStructMake(v *Value) *Value {
 	}
 	return v.Block.NewValue2(v.Pos, OpIMake, v.Type, v.Args[0], arg)
 }
+
+// bool2int converts bool to int: true to 1, false to 0
+func bool2int(x bool) int {
+	var b int
+	if x {
+		b = 1
+	}
+	return b
+}