[pull] master from golang:master

src/bytes/bytes.go

@@ -1109,10 +1109,7 @@ func trimRightUnicode(s []byte, cutset string) []byte {
 
 func trimSpaceUnicode(s []byte) []byte {
 	for len(s) > 0 {
-		r, n := rune(s[0]), 1
-		if r >= utf8.RuneSelf {
-			r, n = utf8.DecodeRune(s)
-		}
+		r, n := utf8.DecodeRune(s)
 		if !stringslite.IsSpace(r) {
 			break
 		}

src/cmd/compile/internal/amd64/ssa.go

@@ -892,24 +892,34 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.From.Reg = v.Args[2].Reg()
 		memIdx(&p.To, v)
 		ssagen.AddAux(&p.To, v)
-	case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify:
+	case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify,
+		ssa.OpAMD64ADDWconstmodify, ssa.OpAMD64ADDBconstmodify:
 		sc := v.AuxValAndOff()
 		off := sc.Off64()
 		val := sc.Val()
 		if val == 1 || val == -1 {
 			var asm obj.As
-			if v.Op == ssa.OpAMD64ADDQconstmodify {
-				if val == 1 {
-					asm = x86.AINCQ
-				} else {
+			switch v.Op {
+			case ssa.OpAMD64ADDQconstmodify:
+				asm = x86.AINCQ
+				if val == -1 {
 					asm = x86.ADECQ
 				}
-			} else {
-				if val == 1 {
-					asm = x86.AINCL
-				} else {
+			case ssa.OpAMD64ADDLconstmodify:
+				asm = x86.AINCL
+				if val == -1 {
 					asm = x86.ADECL
 				}
+			case ssa.OpAMD64ADDWconstmodify:
+				asm = x86.AINCW
+				if val == -1 {
+					asm = x86.ADECW
+				}
+			default:
+				asm = x86.AINCB
+				if val == -1 {
+					asm = x86.ADECB
+				}
 			}
 			p := s.Prog(asm)
 			p.To.Type = obj.TYPE_MEM
@@ -920,6 +930,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		fallthrough
 	case ssa.OpAMD64ANDQconstmodify, ssa.OpAMD64ANDLconstmodify, ssa.OpAMD64ORQconstmodify, ssa.OpAMD64ORLconstmodify,
 		ssa.OpAMD64XORQconstmodify, ssa.OpAMD64XORLconstmodify,
+		ssa.OpAMD64ANDWconstmodify, ssa.OpAMD64ANDBconstmodify, ssa.OpAMD64ORWconstmodify, ssa.OpAMD64ORBconstmodify,
+		ssa.OpAMD64XORWconstmodify, ssa.OpAMD64XORBconstmodify,
 		ssa.OpAMD64BTSQconstmodify, ssa.OpAMD64BTRQconstmodify, ssa.OpAMD64BTCQconstmodify:
 		sc := v.AuxValAndOff()
 		off := sc.Off64()
@@ -960,7 +972,11 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpAMD64ADDLconstmodifyidx1, ssa.OpAMD64ADDLconstmodifyidx4, ssa.OpAMD64ADDLconstmodifyidx8, ssa.OpAMD64ADDQconstmodifyidx1, ssa.OpAMD64ADDQconstmodifyidx8,
 		ssa.OpAMD64ANDLconstmodifyidx1, ssa.OpAMD64ANDLconstmodifyidx4, ssa.OpAMD64ANDLconstmodifyidx8, ssa.OpAMD64ANDQconstmodifyidx1, ssa.OpAMD64ANDQconstmodifyidx8,
 		ssa.OpAMD64ORLconstmodifyidx1, ssa.OpAMD64ORLconstmodifyidx4, ssa.OpAMD64ORLconstmodifyidx8, ssa.OpAMD64ORQconstmodifyidx1, ssa.OpAMD64ORQconstmodifyidx8,
-		ssa.OpAMD64XORLconstmodifyidx1, ssa.OpAMD64XORLconstmodifyidx4, ssa.OpAMD64XORLconstmodifyidx8, ssa.OpAMD64XORQconstmodifyidx1, ssa.OpAMD64XORQconstmodifyidx8:
+		ssa.OpAMD64XORLconstmodifyidx1, ssa.OpAMD64XORLconstmodifyidx4, ssa.OpAMD64XORLconstmodifyidx8, ssa.OpAMD64XORQconstmodifyidx1, ssa.OpAMD64XORQconstmodifyidx8,
+		ssa.OpAMD64ADDWconstmodifyidx1, ssa.OpAMD64ADDWconstmodifyidx2, ssa.OpAMD64ADDBconstmodifyidx1,
+		ssa.OpAMD64ANDWconstmodifyidx1, ssa.OpAMD64ANDWconstmodifyidx2, ssa.OpAMD64ANDBconstmodifyidx1,
+		ssa.OpAMD64ORWconstmodifyidx1, ssa.OpAMD64ORWconstmodifyidx2, ssa.OpAMD64ORBconstmodifyidx1,
+		ssa.OpAMD64XORWconstmodifyidx1, ssa.OpAMD64XORWconstmodifyidx2, ssa.OpAMD64XORBconstmodifyidx1:
 		p := s.Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_CONST
 		sc := v.AuxValAndOff()
@@ -978,6 +994,18 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		case p.As == x86.AADDL && p.From.Offset == -1:
 			p.As = x86.ADECL
 			p.From.Type = obj.TYPE_NONE
+		case p.As == x86.AADDW && p.From.Offset == 1:
+			p.As = x86.AINCW
+			p.From.Type = obj.TYPE_NONE
+		case p.As == x86.AADDW && p.From.Offset == -1:
+			p.As = x86.ADECW
+			p.From.Type = obj.TYPE_NONE
+		case p.As == x86.AADDB && p.From.Offset == 1:
+			p.As = x86.AINCB
+			p.From.Type = obj.TYPE_NONE
+		case p.As == x86.AADDB && p.From.Offset == -1:
+			p.As = x86.ADECB
+			p.From.Type = obj.TYPE_NONE
 		}
 		memIdx(&p.To, v)
 		ssagen.AddAux2(&p.To, v, sc.Off64())
@@ -1251,7 +1279,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		}
 		if x != y {
 			width := v.Type.Size()
-			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg, 3) {
+			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg) {
 				// The source was naturally zext-ed from 32 to 64 bits,
 				// but we are asked to do a full 64-bit copy.
 				// Save the REX prefix byte in I-CACHE by using a 32-bit move,
@@ -1288,6 +1316,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.From.Reg = r
 		ssagen.AddrAuto(&p.To, v)
 	case ssa.OpAMD64LoweredHasCPUFeature:
+		// If this load changes width, update zeroUpperBits in AMD64Ops.go.
 		p := s.Prog(x86.AMOVBLZX)
 		p.From.Type = obj.TYPE_MEM
 		ssagen.AddAux(&p.From, v)
@@ -1669,6 +1698,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against
 		// JNE loop
 		// : result in AX
+		//
+		// If the width written to AX changes, update zeroUpperBits in AMD64Ops.go.
 		mov := x86.AMOVQ
 		op := x86.AANDQ
 		cmpxchg := x86.ACMPXCHGQ

src/cmd/compile/internal/arm64/ssa.go

@@ -1042,6 +1042,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		// LDAXR	(Rarg0), Rout
 		// STLXR	Rarg1, (Rarg0), Rtmp
 		// CBNZ		Rtmp, -2(PC)
+		//
+		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
 		var ld, st obj.As
 		switch v.Op {
 		case ssa.OpARM64LoweredAtomicExchange8:
@@ -1076,6 +1078,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 	case ssa.OpARM64LoweredAtomicExchange64Variant,
 		ssa.OpARM64LoweredAtomicExchange32Variant,
 		ssa.OpARM64LoweredAtomicExchange8Variant:
+		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
 		var swap obj.As
 		switch v.Op {
 		case ssa.OpARM64LoweredAtomicExchange8Variant:
@@ -1163,6 +1166,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		// STLXR	Rarg2, (Rarg0), Rtmp
 		// CBNZ		Rtmp, -4(PC)
 		// CSET		EQ, Rout
+		//
+		// If Rout stops being written only by CSET, update zeroUpperBits in ARM64Ops.go.
 		ld := arm64.ALDAXR
 		st := arm64.ASTLXR
 		cmp := arm64.ACMP
@@ -1212,6 +1217,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		// CASAL	Rtmp, (Rarg0), Rarg2
 		// CMP  	Rarg1, Rtmp
 		// CSET 	EQ, Rout
+		//
+		// If Rout stops being written only by CSET, update zeroUpperBits in ARM64Ops.go.
 		cas := arm64.ACASALD
 		cmp := arm64.ACMP
 		mov := arm64.AMOVD
@@ -1263,6 +1270,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		// AND/OR	Rarg1, Rout, tmp1
 		// STLXR[BW] tmp1, (Rarg0), Rtmp
 		// CBNZ		Rtmp, -3(PC)
+		//
+		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
 		ld := arm64.ALDAXR
 		st := arm64.ASTLXR
 		if v.Op == ssa.OpARM64LoweredAtomicAnd32 || v.Op == ssa.OpARM64LoweredAtomicOr32 {
@@ -1303,6 +1312,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 	case ssa.OpARM64LoweredAtomicAnd8Variant,
 		ssa.OpARM64LoweredAtomicAnd32Variant,
 		ssa.OpARM64LoweredAtomicAnd64Variant:
+		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
 		atomic_clear := arm64.ALDCLRALD
 		if v.Op == ssa.OpARM64LoweredAtomicAnd32Variant {
 			atomic_clear = arm64.ALDCLRALW
@@ -1332,6 +1342,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 	case ssa.OpARM64LoweredAtomicOr8Variant,
 		ssa.OpARM64LoweredAtomicOr32Variant,
 		ssa.OpARM64LoweredAtomicOr64Variant:
+		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
 		atomic_or := arm64.ALDORALD
 		if v.Op == ssa.OpARM64LoweredAtomicOr32Variant {
 			atomic_or = arm64.ALDORALW
@@ -1870,6 +1881,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		ssa.OpARM64LessThanNoov,
 		ssa.OpARM64GreaterEqualNoov:
 		// generate boolean values using CSET
+		//
+		// If the result stops being a 0/1-producing CSET, update zeroUpperBits in ARM64Ops.go.
 		p := s.Prog(arm64.ACSET)
 		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
 		condCode := condBits[v.Op]

src/cmd/compile/internal/ssa/_gen/AMD64.rules

@@ -1019,6 +1019,10 @@
 	((ADD|AND|OR|XOR)Qconstmodify [ValAndOff(valoff1).addOffset32(off2)] {sym} base mem)
 ((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym} (ADDQconst [off2] base) mem) && ValAndOff(valoff1).canAdd32(off2) =>
 	((ADD|AND|OR|XOR)Lconstmodify [ValAndOff(valoff1).addOffset32(off2)] {sym} base mem)
+((ADD|AND|OR|XOR)Wconstmodify [valoff1] {sym} (ADDQconst [off2] base) mem) && ValAndOff(valoff1).canAdd32(off2) =>
+	((ADD|AND|OR|XOR)Wconstmodify [ValAndOff(valoff1).addOffset32(off2)] {sym} base mem)
+((ADD|AND|OR|XOR)Bconstmodify [valoff1] {sym} (ADDQconst [off2] base) mem) && ValAndOff(valoff1).canAdd32(off2) =>
+	((ADD|AND|OR|XOR)Bconstmodify [ValAndOff(valoff1).addOffset32(off2)] {sym} base mem)
 ((ADD|SUB|AND|OR|XOR)Qmodify [off1] {sym} (ADDQconst [off2] base) val mem) && is32Bit(int64(off1)+int64(off2)) =>
 	((ADD|SUB|AND|OR|XOR)Qmodify [off1+off2] {sym} base val mem)
 ((ADD|SUB|AND|OR|XOR)Lmodify [off1] {sym} (ADDQconst [off2] base) val mem) && is32Bit(int64(off1)+int64(off2)) =>
@@ -1076,6 +1080,12 @@
 ((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym1} (LEAQ [off2] {sym2} base) mem)
 	&& ValAndOff(valoff1).canAdd32(off2) && canMergeSym(sym1, sym2) =>
 	((ADD|AND|OR|XOR)Lconstmodify [ValAndOff(valoff1).addOffset32(off2)] {mergeSym(sym1,sym2)} base mem)
+((ADD|AND|OR|XOR)Wconstmodify [valoff1] {sym1} (LEAQ [off2] {sym2} base) mem)
+	&& ValAndOff(valoff1).canAdd32(off2) && canMergeSym(sym1, sym2) =>
+	((ADD|AND|OR|XOR)Wconstmodify [ValAndOff(valoff1).addOffset32(off2)] {mergeSym(sym1,sym2)} base mem)
+((ADD|AND|OR|XOR)Bconstmodify [valoff1] {sym1} (LEAQ [off2] {sym2} base) mem)
+	&& ValAndOff(valoff1).canAdd32(off2) && canMergeSym(sym1, sym2) =>
+	((ADD|AND|OR|XOR)Bconstmodify [ValAndOff(valoff1).addOffset32(off2)] {mergeSym(sym1,sym2)} base mem)
 ((ADD|SUB|AND|OR|XOR)Qmodify [off1] {sym1} (LEAQ [off2] {sym2} base) val mem)
 	&& is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
 	((ADD|SUB|AND|OR|XOR)Qmodify [off1+off2] {mergeSym(sym1,sym2)} base val mem)
@@ -1474,6 +1484,17 @@
 (MOVLstore [off] {sym} ptr a:((ADD|AND|OR|XOR)Lconst [c] l:(MOVLload [off] {sym} ptr2 mem)) mem)
 	&& isSamePtr(ptr, ptr2) && a.Uses == 1 && l.Uses == 1 && clobber(l, a) =>
 	((ADD|AND|OR|XOR)Lconstmodify {sym} [makeValAndOff(int32(c),off)] ptr mem)
+// Sub-word read-modify-write. The narrow store keeps only the low 8/16 bits of
+// the 32-bit result, so ADDB/ADDW etc. with the same constant is equivalent.
+// The constant is restricted to an 8-bit immediate: it keeps every form we emit
+// encodable as imm8 (or INC/DEC), which also avoids the 16-bit immediates that
+// cause length-changing-prefix stalls.
+(MOVWstore [off] {sym} ptr a:((ADD|AND|OR|XOR)Lconst [c] l:(MOVWload [off] {sym} ptr2 mem)) mem)
+	&& isSamePtr(ptr, ptr2) && a.Uses == 1 && l.Uses == 1 && c == int32(int8(c)) && clobber(l, a) =>
+	((ADD|AND|OR|XOR)Wconstmodify {sym} [makeValAndOff(int32(c),off)] ptr mem)
+(MOVBstore [off] {sym} ptr a:((ADD|AND|OR|XOR)Lconst [c] l:(MOVBload [off] {sym} ptr2 mem)) mem)
+	&& isSamePtr(ptr, ptr2) && a.Uses == 1 && l.Uses == 1 && c == int32(int8(c)) && clobber(l, a) =>
+	((ADD|AND|OR|XOR)Bconstmodify {sym} [makeValAndOff(int32(c),off)] ptr mem)
 
 // float <-> int register moves, with no conversion.
 // These come up when compiling math.{Float{32,64}bits,Float{32,64}frombits}.