all: REVERSE MERGE dev.simd (c456ab7) into master

This commit is a REVERSE MERGE.
It merges dev.simd back into its parent branch, master.
This marks the end of development on dev.simd.

Merge List:

+ 2025-12-08 c456ab7 [dev.simd] all: merge master (a33bbf1) into dev.simd
+ 2025-12-08 1d8711e [dev.simd] internal/buildcfg: don't enable SIMD experiment by default
+ 2025-12-08 f38e968 [dev.simd] cmd/compile: zero only low 128-bit of X15
+ 2025-12-08 144cf17 [dev.simd] simd, cmd/compile: move "simd" to "simd/archsimd"
+ 2025-12-08 3417b48 [dev.simd] simd: add carryless multiply
+ 2025-12-05 f51ee08 [dev.simd] simd: replace checking loops with call to slice-checker
+ 2025-12-03 2b91d96 [dev.simd] internal/buildcfg: turn GOEXPERIMENT=simd back on

Change-Id: Ife3f2ca4f6d8ce131335c0f868358db6a6a1a534

Author: cherrymui

src/cmd/compile/internal/amd64/simdssa.go

@@ -18,7 +18,6 @@ import (
 	"cmd/internal/obj"
 	"cmd/internal/obj/x86"
 	"internal/abi"
-	"internal/buildcfg"
 )
 
 // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
@@ -1718,7 +1717,15 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 	case ssa.OpAMD64VZEROUPPER, ssa.OpAMD64VZEROALL:
 		s.Prog(v.Op.Asm())
 
-	case ssa.OpAMD64Zero128, ssa.OpAMD64Zero256, ssa.OpAMD64Zero512: // no code emitted
+	case ssa.OpAMD64Zero128: // no code emitted
+
+	case ssa.OpAMD64Zero256, ssa.OpAMD64Zero512:
+		p := s.Prog(v.Op.Asm())
+		p.From.Type = obj.TYPE_REG
+		p.From.Reg = simdReg(v)
+		p.AddRestSourceReg(simdReg(v))
+		p.To.Type = obj.TYPE_REG
+		p.To.Reg = simdReg(v)
 
 	case ssa.OpAMD64VMOVSSf2v, ssa.OpAMD64VMOVSDf2v:
 		// These are for initializing the least 32/64 bits of a SIMD register from a "float".
@@ -1871,34 +1878,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 
 // zeroX15 zeroes the X15 register.
 func zeroX15(s *ssagen.State) {
-	if !buildcfg.Experiment.SIMD {
-		opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
-		return
-	}
-	vxorps := func(s *ssagen.State) {
-		p := s.Prog(x86.AVXORPS)
-		p.From.Type = obj.TYPE_REG
-		p.From.Reg = x86.REG_X15
-		p.AddRestSourceReg(x86.REG_X15)
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = x86.REG_X15
-	}
-	if buildcfg.GOAMD64 >= 3 {
-		vxorps(s)
-		return
-	}
-	// AVX may not be available, check before zeroing the high bits.
-	p := s.Prog(x86.ACMPB)
-	p.From.Type = obj.TYPE_MEM
-	p.From.Name = obj.NAME_EXTERN
-	p.From.Sym = ir.Syms.X86HasAVX
-	p.To.Type = obj.TYPE_CONST
-	p.To.Offset = 1
-	jmp := s.Prog(x86.AJNE)
-	jmp.To.Type = obj.TYPE_BRANCH
-	vxorps(s)
-	sse := opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
-	jmp.To.SetTarget(sse)
+	opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
 }
 
 // Example instruction: VRSQRTPS X1, X1

src/cmd/compile/internal/amd64/ssa.go

@@ -445,7 +445,7 @@ type hairyVisitor struct {
 
 func isDebugFn(fn *ir.Func) bool {
 	// if n := fn.Nname; n != nil {
-	// 	if n.Sym().Name == "Int32x8.Transpose8" && n.Sym().Pkg.Path == "simd" {
+	// 	if n.Sym().Name == "Int32x8.Transpose8" && n.Sym().Pkg.Path == "simd/archsimd" {
 	// 		fmt.Printf("isDebugFn '%s' DOT '%s'\n", n.Sym().Pkg.Path, n.Sym().Name)
 	// 		return true
 	// 	}

src/cmd/compile/internal/inline/inl.go

@@ -214,6 +214,7 @@ func init() {
 		vloadk  = regInfo{inputs: []regMask{gpspsb, mask, 0}, outputs: vonly}
 		vstorek = regInfo{inputs: []regMask{gpspsb, mask, v, 0}}
 
+		v01     = regInfo{inputs: nil, outputs: vonly}
 		v11     = regInfo{inputs: vonly, outputs: vonly}            // used in resultInArg0 ops, arg0 must not be x15
 		v21     = regInfo{inputs: []regMask{v, vz}, outputs: vonly} // used in resultInArg0 ops, arg0 must not be x15
 		vk      = regInfo{inputs: vzonly, outputs: maskonly}
@@ -232,6 +233,7 @@ func init() {
 		gpv     = regInfo{inputs: []regMask{gp}, outputs: vonly}
 		v2flags = regInfo{inputs: []regMask{vz, vz}}
 
+		w01   = regInfo{inputs: nil, outputs: wonly}
 		w11   = regInfo{inputs: wonly, outputs: wonly} // used in resultInArg0 ops, arg0 must not be x15
 		w21   = regInfo{inputs: []regMask{wz, wz}, outputs: wonly}
 		wk    = regInfo{inputs: wzonly, outputs: maskonly}
@@ -1398,12 +1400,15 @@ func init() {
 		{name: "VPMOVVec64x4ToM", argLength: 1, reg: vk, asm: "VPMOVQ2M"},
 		{name: "VPMOVVec64x8ToM", argLength: 1, reg: wk, asm: "VPMOVQ2M"},
 
+		// X15 is the zero register up to 128-bit. For larger values, we zero it on the fly.
 		{name: "Zero128", argLength: 0, reg: x15only, zeroWidth: true, fixedReg: true},
-		{name: "Zero256", argLength: 0, reg: x15only, zeroWidth: true, fixedReg: true},
-		{name: "Zero512", argLength: 0, reg: x15only, zeroWidth: true, fixedReg: true},
+		{name: "Zero256", argLength: 0, reg: v01, asm: "VPXOR"},
+		{name: "Zero512", argLength: 0, reg: w01, asm: "VPXORQ"},
 
+		// Move a 32/64 bit float to a 128-bit SIMD register.
 		{name: "VMOVSDf2v", argLength: 1, reg: fpv, asm: "VMOVSD"},
 		{name: "VMOVSSf2v", argLength: 1, reg: fpv, asm: "VMOVSS"},
+
 		{name: "VMOVQ", argLength: 1, reg: gpv, asm: "VMOVQ"},
 		{name: "VMOVD", argLength: 1, reg: gpv, asm: "VMOVD"},
 

src/cmd/compile/internal/ssa/_gen/AMD64Ops.go

@@ -1333,6 +1333,9 @@
 (blendMaskedInt16x32 x y mask) => (VPBLENDMWMasked512 x y (VPMOVVec16x32ToM <types.TypeMask> mask))
 (blendMaskedInt32x16 x y mask) => (VPBLENDMDMasked512 x y (VPMOVVec32x16ToM <types.TypeMask> mask))
 (blendMaskedInt64x8 x y mask) => (VPBLENDMQMasked512 x y (VPMOVVec64x8ToM <types.TypeMask> mask))
+(carrylessMultiplyUint64x2 ...) => (VPCLMULQDQ128 ...)
+(carrylessMultiplyUint64x4 ...) => (VPCLMULQDQ256 ...)
+(carrylessMultiplyUint64x8 ...) => (VPCLMULQDQ512 ...)
 (concatSelectedConstantFloat32x4 ...) => (VSHUFPS128 ...)
 (concatSelectedConstantFloat64x2 ...) => (VSHUFPD128 ...)
 (concatSelectedConstantInt32x4 ...) => (VSHUFPS128 ...)

src/cmd/compile/internal/ssa/_gen/simdAMD64.rules

@@ -1644,7 +1644,7 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
 		// Only enable intrinsics, if SIMD experiment.
 		simdIntrinsics(addF)
 
-		addF("simd", "ClearAVXUpperBits",
+		addF(simdPackage, "ClearAVXUpperBits",
 			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
 				s.vars[memVar] = s.newValue1(ssa.OpAMD64VZEROUPPER, types.TypeMem, s.mem())
 				return nil
@@ -1668,15 +1668,18 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
 		addF(simdPackage, "Uint32x8.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
 		addF(simdPackage, "Uint64x4.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
 
+		// sfp4 is intrinsic-if-constant, but otherwise it's complicated enough to just implement in Go.
 		sfp4 := func(method string, hwop ssa.Op, vectype *types.Type) {
-			addF("simd", method,
+			addF(simdPackage, method,
 				func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
 					x, a, b, c, d, y := args[0], args[1], args[2], args[3], args[4], args[5]
 					if a.Op == ssa.OpConst8 && b.Op == ssa.OpConst8 && c.Op == ssa.OpConst8 && d.Op == ssa.OpConst8 {
-						return select4FromPair(x, a, b, c, d, y, s, hwop, vectype)
-					} else {
-						return s.callResult(n, callNormal)
+						z := select4FromPair(x, a, b, c, d, y, s, hwop, vectype)
+						if z != nil {
+							return z
+						}
 					}
+					return s.callResult(n, callNormal)
 				},
 				sys.AMD64)
 		}
@@ -1693,15 +1696,18 @@ func initIntrinsics(cfg *intrinsicBuildConfig) {
 		sfp4("Uint32x16.SelectFromPairGrouped", ssa.OpconcatSelectedConstantGroupedUint32x16, types.TypeVec512)
 		sfp4("Float32x16.SelectFromPairGrouped", ssa.OpconcatSelectedConstantGroupedFloat32x16, types.TypeVec512)
 
+		// sfp2 is intrinsic-if-constant, but otherwise it's complicated enough to just implement in Go.
 		sfp2 := func(method string, hwop ssa.Op, vectype *types.Type, cscimm func(i, j uint8) int64) {
-			addF("simd", method,
+			addF(simdPackage, method,
 				func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
 					x, a, b, y := args[0], args[1], args[2], args[3]
 					if a.Op == ssa.OpConst8 && b.Op == ssa.OpConst8 {
-						return select2FromPair(x, a, b, y, s, hwop, vectype, cscimm)
-					} else {
-						return s.callResult(n, callNormal)
+						z := select2FromPair(x, a, b, y, s, hwop, vectype, cscimm)
+						if z != nil {
+							return z
+						}
 					}
+					return s.callResult(n, callNormal)
 				},
 				sys.AMD64)
 		}
@@ -1767,6 +1773,9 @@ const (
 
 func select2FromPair(x, _a, _b, y *ssa.Value, s *state, op ssa.Op, t *types.Type, csc func(a, b uint8) int64) *ssa.Value {
 	a, b := uint8(_a.AuxInt8()), uint8(_b.AuxInt8())
+	if a > 3 || b > 3 {
+		return nil
+	}
 	pattern := (a&2)>>1 + (b & 2)
 	a, b = a&1, b&1
 
@@ -1785,6 +1794,9 @@ func select2FromPair(x, _a, _b, y *ssa.Value, s *state, op ssa.Op, t *types.Type
 
 func select4FromPair(x, _a, _b, _c, _d, y *ssa.Value, s *state, op ssa.Op, t *types.Type) *ssa.Value {
 	a, b, c, d := uint8(_a.AuxInt8()), uint8(_b.AuxInt8()), uint8(_c.AuxInt8()), uint8(_d.AuxInt8())
+	if a > 7 || b > 7 || c > 7 || d > 7 {
+		return nil
+	}
 	pattern := a>>2 + (b&4)>>1 + (c & 4) + (d&4)<<1
 
 	a, b, c, d = a&3, b&3, c&3, d&3
@@ -2154,7 +2166,7 @@ func findIntrinsic(sym *types.Sym) intrinsicBuilder {
 	fn := sym.Name
 	if ssa.IntrinsicsDisable {
 		if pkg == "internal/runtime/sys" && (fn == "GetCallerPC" || fn == "GrtCallerSP" || fn == "GetClosurePtr") ||
-			pkg == "internal/simd" || pkg == "simd" { // TODO after simd has been moved to package simd, remove internal/simd
+			pkg == simdPackage {
 			// These runtime functions don't have definitions, must be intrinsics.
 		} else {
 			return nil